Problem with CPI1466 - Job holds large number of locks

Posted on 2006-04-13
Medium Priority
Last Modified: 2010-05-18

I have a nightly routine that reads a file with designated library names to be saved to save files while active.

It seems to be keeping other critical processes from running.

0001.00 /*-------------------------------------------------------------------*/
0002.00 /*   SAVLIBS - Nightly Backup Procedure CL                           */
0003.00 /*-------------------------------------------------------------------*/
0004.00              PGM                                                        
0005.00 /* DECLARE VARIABLES AND FILE USED */                                  
0006.00              DCLF       FILE(BACKUP)                                    
0007.00              DCL        VAR(&SLT) TYPE(*CHAR) LEN(50) VALUE('BUFRE +    
0008.00                           *EQ "X"')                                    
0009.00              DCL        VAR(&STIMA) TYPE(*CHAR) LEN(6)                  
0010.00              DCL        VAR(&FCODE) TYPE(*CHAR) LEN(1)                  
0011.00              DCL        VAR(&DEV) TYPE(*CHAR) LEN(10)                  
0012.00              DCL        VAR(&YORN) TYPE(*CHAR) LEN(1)                  
0013.00              DCL        VAR(&PWRDWN) TYPE(*CHAR) LEN(1)                
0014.00              DCL        VAR(&BACK) TYPE(*CHAR) LEN(7) VALUE('BACKUP ')  
0015.00              DCL        VAR(&BULIB) TYPE(*CHAR) LEN(10)                
0016.00              DCL        VAR(&BUFRE) TYPE(*CHAR) LEN(1)                  
0017.00              DCL        VAR(&RPT) TYPE(*CHAR) LEN(2) VALUE('@X')        
0018.00              DCL        VAR(&CHAR6) TYPE(*CHAR) LEN(6)                  
0019.00              DCL        VAR(&RTNDTA) TYPE(*CHAR) LEN(150)              
0020.00              MONMSG     MSGID(CPF0000) EXEC(GOTO CMDLBL(ERROR))        
0023.00              OVRPRTF    FILE(QPDSPSBJ) HOLD(*YES)                      
0024.00              CHGJOB     LOG(4 0 *SECLVL) LOGCLPGM(*YES)                
0026.00  GETQSYSOPR: ALCOBJ     OBJ((QSYSOPR *MSGQ *EXCL)) WAIT(5)              
0027.00              MONMSG     MSGID(CPF1002) EXEC(DO)                        
0028.00              CALL       PGM(CHKMSGQ) PARM('QSYSOPR' 'CHK' &RTNDTA)      
0029.00 /*  If jobs have message queue, cancel them                          */
0030.00              IF         COND(&RTNDTA = Y) THEN(DO)                      
0031.00              DLYJOB     DLY(60)                                        
0032.00              CALL       PGM(CHKMSGQ) PARM('QSYSOPR' 'CNL' &RTNDTA)      
0033.00              GOTO       CMDLBL(GETQSYSOPR)                              
0034.00              ENDDO                                                  
0035.00              ENDDO                                                  
0036.00              CHGMSGQ    MSGQ(QSYSOPR) DLVRY(*DFT)                  
0037.00 /* DELETE REPORT LOOP */                                            
0038.00  LOOP3:      DLTSPLF    FILE(QPDSPSBJ) SPLNBR(*LAST)                
0039.00              MONMSG     MSGID(CPF3300) EXEC(GOTO CMDLBL(LOOPEND))  
0040.00              GOTO       CMDLBL(LOOP3)                              
0041.00  LOOPEND:    DLTOVR     FILE(*ALL)                                  
0043.00              CHGVAR     VAR(&FCODE) VALUE('D')                      
0045.00              CHGVAR     VAR(&SLT) VALUE('BUFRE *EQ "' *CAT &FCODE +
0046.00                           *CAT '"')                                
0047.00              CHGVAR     VAR(%SST(&BACK 7 1)) VALUE(&FCODE)          
0050.00              OVRDBF     FILE(BACKUP) SHARE(*YES)                    
0051.00              OPNQRYF    FILE((BACKUP)) OPTION(*INP *UPD) QRYSLT(&SLT) +
0052.00                           KEYFLD(*FILE)                                
0053.00  AGAIN:      RCVF                                                      
0054.00              MONMSG     MSGID(CPF0864) EXEC(GOTO CMDLBL(EOF))          
0055.00 /* BACKUP BASED ON FREQUENCY CODE ENTERED */                            
0056.00              CHKOBJ     OBJ(&BULIB) OBJTYPE(*LIB)                      
0057.00              MONMSG     MSGID(CPF0000) EXEC(GOTO CMDLBL(AGAIN))  

(Here's where I added the save to save file)    

0058.00 /* PDK & ASSOCIATES CHANGES MADE 10/12/2005                          */
0059.00 /* CHECK FOR SAVE FILE, IF NOT AVAILABLE-CREATE IT                   */
0060.00 /* IF AVAILABLE-CLEAR IT.                                            */
0061.00              CHKOBJ     OBJ(FTPLIB/&BULIB) OBJTYPE(*FILE)              
0062.00              MONMSG     MSGID(CPF9801) EXEC(DO)                        
0063.00              CRTSAVF    FILE(FTPLIB/&BULIB) AUT(*ALL)                  
0064.00              GOTO NOCLEAR                                              
0065.00              ENDDO                                                      
0066.00              CLRSAVF FILE(FTPLIB/&BULIB)                                
0067.00 NOCLEAR:                                                                
0069.00 /* PC SERVER     */
(Here's where the save actually occurs)

0070.00              SAVOBJ     OBJ(*ALL) LIB(&BULIB) DEV(*SAVF) +              
0071.00                           SAVF(FTPLIB/&BULIB) SAVACT(*SYSDFN) +        
0072.00                           SAVACTWAIT(0) DTACPR(*YES)                    
0073.00              MONMSG     MSGID(CPF3770)                                  

(This is where I submit the job to push the savf file over to the server via FTP)  
0076.00              SBMJOB     CMD(CALL PGM(XFERLIB) PARM(&BULIB)) +          
0077.00                           JOB(XFERLIB) JOBD(PDK/PDK) JOBQ(QGPL/ICC)    
0079.00 /* END CHANGES PDK & ASSOCIATES                                      */
0080.00              CALL       PGM(BACKUPDT) PARM(&BULIB &BUFRE)  

(Then the process repeats till all libraries are saved)
0081.00              GOTO       CMDLBL(AGAIN)                                  
0082.00 EOF:         DLTOVR     FILE(*ALL)                                      
0083.00              CLOF       OPNID(BACKUP)                                  
0084.00 /* PRINT REQUESTED REPORTS BY FREQUENCY CODE */                        
0085.00 /*           CHGVAR     VAR(%SST(&RPT 2 1)) VALUE(&FCODE)            */
0086.00 /*           CALL       PGM(&RPT)                                    */
0087.00 /*           MONMSG     MSGID(CPF0000)                               */
0088.00 /*           CALL       PGM(CLRDEVMSGQ)                              */
0089.00  ERROR:      CHGMSGQ    MSGQ(QSYSOPR) DLVRY(*HOLD)                    
0090.00              MONMSG     MSGID(CPF0000)                                
0091.00              ENDPGM        

How do I release all the object locks?
Is there something I need to do after the save to the *savf?

Urgent! Please help!                                          
Question by:pipster1
  • 5
  • 3
  • 3
LVL 13

Expert Comment

ID: 16449651

The message in the QSYSOPR message queue indicating that the job holds a large number of locks is not likely the problem. A save operation can hold a large number of locks if it is saving a large number of objects.

When the save of each library completes, it should release the locks on those objects.

How are the other jobs impacted? Is CPU percentage very high at the same time?
I notice that the save command uses COMPRESS(*YES). This can be a very CPU intensive option. Please check the compression is great enough to justify the CPU expenditure.

LVL 27

Accepted Solution

tliotta earned 2000 total points
ID: 16456056
COMPRESS(*YES) may indeed be the real problem. Rather than submitting jobs that simply CALL PGM(XFERLIB), those jobs might also do the individual SAVOBJ commands.

They could be submitted to a multi-threaded jobq so a number of them could run concurrently. Each one could put a shared lock on some object. This job could attempt to allocate the same object exclusively with a very long wait time so it wouldn't end until all submitted jobs ended.

The submitted jobs could be named according to the library being saved. If the QSYSOPR message still appeared, the job name could help determine which library was involved. Further, the locks would be distributed across multiple jobs, reducing any possible chance of hitting limits (though limits doesn't seem to be a problem really).

Distributing the work among multiple jobs should raise the efficiency some. Tuning could be done to the subsystem that ran those jobs.


Author Comment

ID: 16456129

Could you cite some example of a multithresded jobque?

Nothing ever in the clear!

This technical paper will help you implement VMware’s VM encryption as well as implement Veeam encryption which together will achieve the nothing ever in the clear goal. If a bad guy steals VMs, backups or traffic they get nothing.

LVL 27

Expert Comment

ID: 16456305

 ==>  crtsbsd  mylib/mysbsd  pools(( 1 *shrpool1 )) MAXJOBS( 4 )
 ==>  crtjobq  qgpl/myjobq
 ==>  addjobqe  mylib/mysbsd  qgpl/myjobq  MAXACT(4)  seqnbr(nnn)

That's the basics of it; there are additional details you might want to review by prompting though you can probably ignore most of it.

You have some subsystem that allows multiple jobs to be active concurrently. Then you attach a jobq to it and specify how many jobs may be started from that jobq. In this case, if you send ten long-running jobs to QGPL/MYJOBQ, you'll see four of them start up immediately. The other six will wait on the jobq until one of the active jobs finishes.

If you create your own subsystem, you'll have one to play with. Naturally, you'll have to make choices on all of the possible attributes, but most of those can be copied from QBATCH. You might choose to create your own class descriptions (*CLS) or use existing ones for example. The more you create yourself, the more you can customize with minimal effect on anything else. OTOH, the management effort can rise accordingly.

A multi-threaded jobq is useful for jobs that do _not_ interfere with nor depend on one another.

LVL 27

Expert Comment

ID: 16456336

Use WRKSBS to review attributes of your active subsystems. As you look through them, you'll see that there are already multi-threaded queues. You _might_ use one of those, but I try to stay away from them in production stuff. No need to use them when it's so easy to create your own.

Note that authorities, etc., need to be attended to. You don't necessarily want various users cramming jobs through your jobq but you do need _your_ submitted jobs to have authority.

LVL 13

Expert Comment

ID: 16457330
Can you clarify "It seems to be keeping other critical processes from running"?
Does that mean the other processes are running at the same time as the backup or or delayed by the backup?
Are the other jobs getting error messages, like "unable to allocate"?

The easiest change is to remove the compression and see if that does what you need. To leave in the compression and run multiple jobs at the same time could create a bigger performance problem.

LVL 27

Expert Comment

ID: 16457445
Yeah, running general backups during production runs is going to be a problem regardless. It may depend on what the constraints are. If I/O is the constraint, then extra CPU may be freely available. If CPU is the constraint, then I/O may be freely available.

If neither is available, then the system is sized too small. If either is available, then multiple jobs can interleave and make better use. Tuning gives control.


Author Comment

ID: 16457686

Yes, there are other jobs running for update that are showing pending for update and it's sort of creating a bottleneck.

There really aren't any constraints as far as CPU, but would taking off the compress constraint make the savf job go faster?

I like the idea of shooting off the xferlib jobs with the savf stuff in it.

LVL 13

Expert Comment

ID: 16457728
Removing the compression will make it go faster, but the save files may be bigger.

Use WRKSYSACT if you can to confirm that CPU is not an issue, as some tasks will not show up in WRKACTJOB.

You can remove the compression and run multiple jobs, but I would do one of the two first and then test to see what happens.

LVL 27

Expert Comment

ID: 16457962
Barry's _probably_ correct that jobs will run faster without DTACPR(*YES). Count on cutting 20% off your run time of individual saves (though I have seen it actually take less wall-clock time) by dropping that. Of course, also count on other times increasing -- you'll be FTPing bigger savefiles for example (not to mention simply outputting more records to the savefile when building them).

But also count on the saves taking much less overall time by running concurrently. That's almost the entire point of having multi-user, multi-tasking systems in the first place. This can cut wall-time down far more than 20%. You can almost figure two saves will run in half the time, three saves in a third the time, etc. (Not quite, but often not far from it.)

If you don't have WRKSYSACT, it's because you don't have Performance Tools installed. It can be far more useful than WRKACTJOB.

All of it is always a cost/benefit balancing act.


Author Comment

ID: 16526745
Thanks Tom, that worked great!

Featured Post

Free Tool: Path Explorer

An intuitive utility to help find the CSS path to UI elements on a webpage. These paths are used frequently in a variety of front-end development and QA automation tasks.

One of a set of tools we're offering as a way of saying thank you for being a part of the community.

Question has a verified solution.

If you are experiencing a similar issue, please ask a related question

Microsoft Office Picture Manager was included in Office 2003, 2007, and 2010, but not in Office 2013. Users had hopes that it would be in Office 2016/Office 365, but it is not. Fortunately, the same zero-cost technique that works to install it with …
Often times it's very very easy to extend a volume on a Linux instance in AWS, but impossible to shrink it. I wanted to contribute to the experts-exchange community a way of providing a procedure that works on an AWS instance. It can also be used on…
Are you ready to place your question in front of subject-matter experts for more timely responses? With the release of Priority Question, Premium Members, Team Accounts and Qualified Experts can now identify the emergent level of their issue, signal…
Look below the covers at a subform control , and the form that is inside it. Explore properties and see how easy it is to aggregate, get statistics, and synchronize results for your data. A Microsoft Access subform is used to show relevant calcul…
Suggested Courses

850 members asked questions and received personalized solutions in the past 7 days.

Join the community of 500,000 technology professionals and ask your questions.

Join & Ask a Question