Problem with CPI1466 - Job holds large number of locks

Help!

I have a nightly routine that reads a file with designated library names to be saved to save files while active.

It seems to be keeping other critical processes from running.

0001.00 /*-------------------------------------------------------------------*/
0002.00 /*   SAVLIBS - Nightly Backup Procedure CL                           */
0003.00 /*-------------------------------------------------------------------*/
0004.00              PGM                                                        
0005.00 /* DECLARE VARIABLES AND FILE USED */                                  
0006.00              DCLF       FILE(BACKUP)                                    
0007.00              DCL        VAR(&SLT) TYPE(*CHAR) LEN(50) VALUE('BUFRE +    
0008.00                           *EQ "X"')                                    
0009.00              DCL        VAR(&STIMA) TYPE(*CHAR) LEN(6)                  
0010.00              DCL        VAR(&FCODE) TYPE(*CHAR) LEN(1)                  
0011.00              DCL        VAR(&DEV) TYPE(*CHAR) LEN(10)                  
0012.00              DCL        VAR(&YORN) TYPE(*CHAR) LEN(1)                  
0013.00              DCL        VAR(&PWRDWN) TYPE(*CHAR) LEN(1)                
0014.00              DCL        VAR(&BACK) TYPE(*CHAR) LEN(7) VALUE('BACKUP ')  
0015.00              DCL        VAR(&BULIB) TYPE(*CHAR) LEN(10)                
0016.00              DCL        VAR(&BUFRE) TYPE(*CHAR) LEN(1)                  
0017.00              DCL        VAR(&RPT) TYPE(*CHAR) LEN(2) VALUE('@X')        
0018.00              DCL        VAR(&CHAR6) TYPE(*CHAR) LEN(6)                  
0019.00              DCL        VAR(&RTNDTA) TYPE(*CHAR) LEN(150)              
0020.00              MONMSG     MSGID(CPF0000) EXEC(GOTO CMDLBL(ERROR))        
0021.00                                                                        
0022.00 /* OVERRIDE PRINTER, CHANGE LOGGING LEVEL, AND DELAY JOB */            
0023.00              OVRPRTF    FILE(QPDSPSBJ) HOLD(*YES)                      
0024.00              CHGJOB     LOG(4 0 *SECLVL) LOGCLPGM(*YES)                
0025.00                                                                        
0026.00  GETQSYSOPR: ALCOBJ     OBJ((QSYSOPR *MSGQ *EXCL)) WAIT(5)              
0027.00              MONMSG     MSGID(CPF1002) EXEC(DO)                        
0028.00              CALL       PGM(CHKMSGQ) PARM('QSYSOPR' 'CHK' &RTNDTA)      
0029.00 /*  If jobs have message queue, cancel them                          */
0030.00              IF         COND(&RTNDTA = Y) THEN(DO)                      
0031.00              DLYJOB     DLY(60)                                        
0032.00              CALL       PGM(CHKMSGQ) PARM('QSYSOPR' 'CNL' &RTNDTA)      
0033.00              GOTO       CMDLBL(GETQSYSOPR)                              
0034.00              ENDDO                                                  
0035.00              ENDDO                                                  
0036.00              CHGMSGQ    MSGQ(QSYSOPR) DLVRY(*DFT)                  
0037.00 /* DELETE REPORT LOOP */                                            
0038.00  LOOP3:      DLTSPLF    FILE(QPDSPSBJ) SPLNBR(*LAST)                
0039.00              MONMSG     MSGID(CPF3300) EXEC(GOTO CMDLBL(LOOPEND))  
0040.00              GOTO       CMDLBL(LOOP3)                              
0041.00  LOOPEND:    DLTOVR     FILE(*ALL)                                  
0042.00                                                                    
0043.00              CHGVAR     VAR(&FCODE) VALUE('D')                      
0044.00                                                                    
0045.00              CHGVAR     VAR(&SLT) VALUE('BUFRE *EQ "' *CAT &FCODE +
0046.00                           *CAT '"')                                
0047.00              CHGVAR     VAR(%SST(&BACK 7 1)) VALUE(&FCODE)          
0048.00                                                                    
0049.00                                                                    
0050.00              OVRDBF     FILE(BACKUP) SHARE(*YES)                    
0051.00              OPNQRYF    FILE((BACKUP)) OPTION(*INP *UPD) QRYSLT(&SLT) +
0052.00                           KEYFLD(*FILE)                                
0053.00  AGAIN:      RCVF                                                      
0054.00              MONMSG     MSGID(CPF0864) EXEC(GOTO CMDLBL(EOF))          
0055.00 /* BACKUP BASED ON FREQUENCY CODE ENTERED */                            
0056.00              CHKOBJ     OBJ(&BULIB) OBJTYPE(*LIB)                      
0057.00              MONMSG     MSGID(CPF0000) EXEC(GOTO CMDLBL(AGAIN))  

(Here's where I added the save to save file)    

0058.00 /* PDK & ASSOCIATES CHANGES MADE 10/12/2005                          */
0059.00 /* CHECK FOR SAVE FILE, IF NOT AVAILABLE-CREATE IT                   */
0060.00 /* IF AVAILABLE-CLEAR IT.                                            */
0061.00              CHKOBJ     OBJ(FTPLIB/&BULIB) OBJTYPE(*FILE)              
0062.00              MONMSG     MSGID(CPF9801) EXEC(DO)                        
0063.00              CRTSAVF    FILE(FTPLIB/&BULIB) AUT(*ALL)                  
0064.00              GOTO NOCLEAR                                              
0065.00              ENDDO                                                      
0066.00              CLRSAVF FILE(FTPLIB/&BULIB)                                
0067.00 NOCLEAR:                                                                
0068.00 /* SAVE ALL LIBRARY OBJECTS TO A SAVE FILE, THEN FTP IT OVER TO      */
0069.00 /* PC SERVER     */
 
(Here's where the save actually occurs)

0070.00              SAVOBJ     OBJ(*ALL) LIB(&BULIB) DEV(*SAVF) +              
0071.00                           SAVF(FTPLIB/&BULIB) SAVACT(*SYSDFN) +        
0072.00                           SAVACTWAIT(0) DTACPR(*YES)                    
0073.00              MONMSG     MSGID(CPF3770)                                  
0074.00 /* SUBMIT XFERLIB CL PROGRAM TO PASS SAVE FILE TO SERVER             */
0075.00  

(This is where I submit the job to push the savf file over to the server via FTP)  
                                                                     
0076.00              SBMJOB     CMD(CALL PGM(XFERLIB) PARM(&BULIB)) +          
0077.00                           JOB(XFERLIB) JOBD(PDK/PDK) JOBQ(QGPL/ICC)    
0078.00                                                                        
0079.00 /* END CHANGES PDK & ASSOCIATES                                      */
0080.00              CALL       PGM(BACKUPDT) PARM(&BULIB &BUFRE)  

(Then the process repeats till all libraries are saved)
           
0081.00              GOTO       CMDLBL(AGAIN)                                  
0082.00 EOF:         DLTOVR     FILE(*ALL)                                      
0083.00              CLOF       OPNID(BACKUP)                                  
0084.00 /* PRINT REQUESTED REPORTS BY FREQUENCY CODE */                        
0085.00 /*           CHGVAR     VAR(%SST(&RPT 2 1)) VALUE(&FCODE)            */
0086.00 /*           CALL       PGM(&RPT)                                    */
0087.00 /*           MONMSG     MSGID(CPF0000)                               */
0088.00 /*           CALL       PGM(CLRDEVMSGQ)                              */
0089.00  ERROR:      CHGMSGQ    MSGQ(QSYSOPR) DLVRY(*HOLD)                    
0090.00              MONMSG     MSGID(CPF0000)                                
0091.00              ENDPGM        


How do I release all the object locks?
Is there something I need to do after the save to the *savf?

Urgent! Please help!                                          
Phillip KnoxSenior Systems AnalystAsked:
Who is Participating?
I wear a lot of hats...

"The solutions and answers provided on Experts Exchange have been extremely helpful to me over the last few years. I wear a lot of hats - Developer, Database Administrator, Help Desk, etc., so I know a lot of things but not a lot about one thing. Experts Exchange gives me answers from people who do know a lot about one thing, in a easy to use platform." -Todd S.

_b_hCommented:

The message in the QSYSOPR message queue indicating that the job holds a large number of locks is not likely the problem. A save operation can hold a large number of locks if it is saving a large number of objects.

When the save of each library completes, it should release the locks on those objects.

How are the other jobs impacted? Is CPU percentage very high at the same time?
I notice that the save command uses COMPRESS(*YES). This can be a very CPU intensive option. Please check the compression is great enough to justify the CPU expenditure.

Barry
0
tliottaCommented:
COMPRESS(*YES) may indeed be the real problem. Rather than submitting jobs that simply CALL PGM(XFERLIB), those jobs might also do the individual SAVOBJ commands.

They could be submitted to a multi-threaded jobq so a number of them could run concurrently. Each one could put a shared lock on some object. This job could attempt to allocate the same object exclusively with a very long wait time so it wouldn't end until all submitted jobs ended.

The submitted jobs could be named according to the library being saved. If the QSYSOPR message still appeared, the job name could help determine which library was involved. Further, the locks would be distributed across multiple jobs, reducing any possible chance of hitting limits (though limits doesn't seem to be a problem really).

Distributing the work among multiple jobs should raise the efficiency some. Tuning could be done to the subsystem that ran those jobs.

Tom
0

Experts Exchange Solution brought to you by

Your issues matter to us.

Facing a tech roadblock? Get the help and guidance you need from experienced professionals who care. Ask your question anytime, anywhere, with no hassle.

Start your 7-day free trial
Phillip KnoxSenior Systems AnalystAuthor Commented:
Tom,

Could you cite some example of a multithresded jobque?

Phil
0
Cloud Class® Course: Amazon Web Services - Basic

Are you thinking about creating an Amazon Web Services account for your business? Not sure where to start? In this course you’ll get an overview of the history of AWS and take a tour of their user interface.

tliottaCommented:
Phil:

 ==>  crtsbsd  mylib/mysbsd  pools(( 1 *shrpool1 )) MAXJOBS( 4 )
 ==>  crtjobq  qgpl/myjobq
 ==>  addjobqe  mylib/mysbsd  qgpl/myjobq  MAXACT(4)  seqnbr(nnn)

That's the basics of it; there are additional details you might want to review by prompting though you can probably ignore most of it.

You have some subsystem that allows multiple jobs to be active concurrently. Then you attach a jobq to it and specify how many jobs may be started from that jobq. In this case, if you send ten long-running jobs to QGPL/MYJOBQ, you'll see four of them start up immediately. The other six will wait on the jobq until one of the active jobs finishes.

If you create your own subsystem, you'll have one to play with. Naturally, you'll have to make choices on all of the possible attributes, but most of those can be copied from QBATCH. You might choose to create your own class descriptions (*CLS) or use existing ones for example. The more you create yourself, the more you can customize with minimal effect on anything else. OTOH, the management effort can rise accordingly.

A multi-threaded jobq is useful for jobs that do _not_ interfere with nor depend on one another.

Tom
0
tliottaCommented:
Phile:

Use WRKSBS to review attributes of your active subsystems. As you look through them, you'll see that there are already multi-threaded queues. You _might_ use one of those, but I try to stay away from them in production stuff. No need to use them when it's so easy to create your own.

Note that authorities, etc., need to be attended to. You don't necessarily want various users cramming jobs through your jobq but you do need _your_ submitted jobs to have authority.

Tom
0
_b_hCommented:
Can you clarify "It seems to be keeping other critical processes from running"?
Does that mean the other processes are running at the same time as the backup or or delayed by the backup?
Are the other jobs getting error messages, like "unable to allocate"?

The easiest change is to remove the compression and see if that does what you need. To leave in the compression and run multiple jobs at the same time could create a bigger performance problem.

Barry
0
tliottaCommented:
Yeah, running general backups during production runs is going to be a problem regardless. It may depend on what the constraints are. If I/O is the constraint, then extra CPU may be freely available. If CPU is the constraint, then I/O may be freely available.

If neither is available, then the system is sized too small. If either is available, then multiple jobs can interleave and make better use. Tuning gives control.

Tom
0
Phillip KnoxSenior Systems AnalystAuthor Commented:
Barry,

Yes, there are other jobs running for update that are showing pending for update and it's sort of creating a bottleneck.

There really aren't any constraints as far as CPU, but would taking off the compress constraint make the savf job go faster?

I like the idea of shooting off the xferlib jobs with the savf stuff in it.

Phil
0
_b_hCommented:
Removing the compression will make it go faster, but the save files may be bigger.

Use WRKSYSACT if you can to confirm that CPU is not an issue, as some tasks will not show up in WRKACTJOB.

You can remove the compression and run multiple jobs, but I would do one of the two first and then test to see what happens.

Barry
0
tliottaCommented:
Barry's _probably_ correct that jobs will run faster without DTACPR(*YES). Count on cutting 20% off your run time of individual saves (though I have seen it actually take less wall-clock time) by dropping that. Of course, also count on other times increasing -- you'll be FTPing bigger savefiles for example (not to mention simply outputting more records to the savefile when building them).

But also count on the saves taking much less overall time by running concurrently. That's almost the entire point of having multi-user, multi-tasking systems in the first place. This can cut wall-time down far more than 20%. You can almost figure two saves will run in half the time, three saves in a third the time, etc. (Not quite, but often not far from it.)

If you don't have WRKSYSACT, it's because you don't have Performance Tools installed. It can be far more useful than WRKACTJOB.

All of it is always a cost/benefit balancing act.

Tom
0
Phillip KnoxSenior Systems AnalystAuthor Commented:
Thanks Tom, that worked great!
0
It's more than this solution.Get answers and train to solve all your tech problems - anytime, anywhere.Try it for free Edge Out The Competitionfor your dream job with proven skills and certifications.Get started today Stand Outas the employee with proven skills.Start learning today for free Move Your Career Forwardwith certification training in the latest technologies.Start your trial today
IBM System i

From novice to tech pro — start learning today.

Question has a verified solution.

Are you are experiencing a similar issue? Get a personalized answer when you ask a related question.

Have a better answer? Share it in a comment.