Go Premium for a chance to win a PS4. Enter to Win

x
?
Solved

AIX oracle  memory issues

Posted on 2014-02-15
2
Medium Priority
?
3,665 Views
Last Modified: 2014-03-13
This Thread is related to the last 2 Q's
we have run into a major problem after increasing Oracle SGA from 60GB to 160GB at our prod server.

7.1
11g R2 oarcle
no large pages.

the difference is as follows


Original prod lpar where we had the issue

root@1


:/root >vmo -aF
             ame_cpus_per_pool = n/a
               ame_maxfree_mem = n/a
           ame_min_ucpool_size = n/a
               ame_minfree_mem = n/a
               ams_loan_policy = n/a
  enhanced_affinity_affin_time = 1
enhanced_affinity_vmpool_limit = 10
           force_relalias_lite = 0
             kernel_heap_psize = 65536
                  lgpg_regions = 128
                     lgpg_size = 16777216
               low_ps_handling = 1
                       maxfree = 3200
                       maxperm = 67382967
                        maxpin = 70624564
                       maxpin% = 90
                 memory_frames = 78643184
                 memplace_data = 0
          memplace_mapped_file = 0
        memplace_shm_anonymous = 0
            memplace_shm_named = 0
                memplace_stack = 0
                 memplace_text = 0
        memplace_unmapped_file = 0
                       minfree = 1500
                       minperm = 3743495
                      minperm% = 5
                     nokilluid = 0
                       npskill = 125696
                       npswarn = 502784
                     numpsblks = 16089088
               pinnable_frames = 70700261
           relalias_percentage = 0
                         scrub = 0
                      v_pinshm = 1
              vmm_default_pspa = 0
            wlm_memlimit_nonpg = 1
##Restricted tunables
               ame_sys_memview = n/a
                cpu_scale_memp = 8
         data_stagger_interval = 161
                         defps = 1
enhanced_affinity_attach_limit = 100
     enhanced_affinity_balance = 100
     enhanced_affinity_private = 40
      enhanced_memory_affinity = 1
                esid_allocator = 1
                     framesets = 2
                     htabscale = n/a
                  kernel_psize = 65536
          large_page_heap_size = 0
             lru_poll_interval = 10
                     lrubucket = 131072
                    maxclient% = 90
                      maxperm% = 90
               mbuf_heap_psize = 65536
               memory_affinity = 1
                 munmap_npages = 65536
                     npsrpgmax = 1005568
                     npsrpgmin = 754176
                   npsscrubmax = 1005568
                   npsscrubmin = 754176
            num_sem_undo_lists = 0
              num_spec_dataseg = 0
             page_steal_method = 1
          psm_timeout_interval = 20000
             relalias_lockmode = 0
                      rpgclean = 0
                    rpgcontrol = 2
                    scrubclean = 0
                shm_1tb_shared = 12
           shm_1tb_unsh_enable = 1
              shm_1tb_unshared = 256
         soft_min_lgpgs_vmpool = 0
              spec_dataseg_int = 512
              strict_maxclient = 1
                strict_maxperm = 0
          vm_modlist_threshold = -1
               vmm_fork_policy = 1
                vmm_klock_mode = 2
            vmm_mpsize_support = 2
               vmm_vmap_policy = 0
                  vtiol_avg_ms = 200
                  vtiol_minreq = 25
            vtiol_minth_active = 1
                    vtiol_mode = 0
               vtiol_pgin_mode = 2
              vtiol_pgout_mode = 2
               vtiol_q_cpu_pct = 2500
          vtiol_thread_cpu_pct = 5000

 ========================================================================================================================        
      Another test lpar where we were able to reproduce the issue    
root@2

:/root >vmo -aF
             ame_cpus_per_pool = n/a
               ame_maxfree_mem = n/a
           ame_min_ucpool_size = n/a
               ame_minfree_mem = n/a
               ams_loan_policy = n/a
  enhanced_affinity_affin_time = 1
enhanced_affinity_vmpool_limit = 10
                esid_allocator = 1
           force_relalias_lite = 0
             kernel_heap_psize = 65536
                  lgpg_regions = 128
                     lgpg_size = 16777216
               low_ps_handling = 1
                       maxfree = 1088
                       maxperm = 67587160
                        maxpin = 70606996
                       maxpin% = 90
                 memory_frames = 78643200
                 memplace_data = 0
          memplace_mapped_file = 0
        memplace_shm_anonymous = 0
            memplace_shm_named = 0
                memplace_stack = 0
                 memplace_text = 0
        memplace_unmapped_file = 0
                       minfree = 960
                       minperm = 3754838
                      minperm% = 5
                     nokilluid = 0
                       npskill = 167936
                       npswarn = 671744
           num_locks_per_semid = 1
                     numpsblks = 21495808
               pinnable_frames = 69879077
           relalias_percentage = 0
                         scrub = 0
                      v_pinshm = 1
              vmm_default_pspa = 0
                vmm_klock_mode = 2
            wlm_memlimit_nonpg = 1
##Restricted tunables
               ame_sys_memview = n/a
                cpu_scale_memp = 8
         data_stagger_interval = 161
                         defps = 1
enhanced_affinity_attach_limit = 100
     enhanced_affinity_balance = 100
     enhanced_affinity_private = 40
      enhanced_memory_affinity = 1
                     framesets = 2
                     htabscale = n/a
                  kernel_psize = 65536
          large_page_heap_size = 0
             lru_poll_interval = 10
                     lrubucket = 131072
                    maxclient% = 90
                      maxperm% = 90
               mbuf_heap_psize = 65536
               memory_affinity = 1
          multiple_semid_lists = 0
                 munmap_npages = 16384
                     npsrpgmax = 1343488
                     npsrpgmin = 1007616
                   npsscrubmax = 1343488
                   npsscrubmin = 1007616
            num_sem_undo_lists = 0
             num_sems_per_lock = 1
              num_spec_dataseg = 0
             page_steal_method = 1
          psm_timeout_interval = 20000
             relalias_lockmode = 1
                      rpgclean = 0
                    rpgcontrol = 2
                    scrubclean = 0
                shm_1tb_shared = 12
           shm_1tb_unsh_enable = 1
              shm_1tb_unshared = 256
         soft_min_lgpgs_vmpool = 0
              spec_dataseg_int = 512
              strict_maxclient = 1
                strict_maxperm = 0
                   sync_npages = 0
               vm_mmap_areload = 0
          vm_modlist_threshold = -1
              vm_pvlist_dohard = 0
              vm_pvlist_szpcnt = 0
               vmm_fork_policy = 1
            vmm_mpsize_support = 2
               vmm_vmap_policy = 0
                  vtiol_avg_ms = 200
                  vtiol_minreq = 25
            vtiol_minth_active = 1
                    vtiol_mode = 0
               vtiol_pgin_mode = 2
              vtiol_pgout_mode = 2
               vtiol_q_cpu_pct = 2500
          vtiol_thread_cpu_pct = 5000






==================
root@3

a 3rd lpar with higher OSLEVEL where we could not reproduce the issue and it was fine.

:/root >vmo -aF
             ame_cpus_per_pool = n/a
               ame_maxfree_mem = n/a
           ame_min_ucpool_size = n/a
               ame_minfree_mem = n/a
               ams_loan_policy = n/a
  enhanced_affinity_affin_time = 1
enhanced_affinity_vmpool_limit = 10
                esid_allocator = 0
           force_relalias_lite = 0
             kernel_heap_psize = 65536
                  lgpg_regions = 0
                     lgpg_size = 0
               low_ps_handling = 1
                       maxfree = 1088
                       maxperm = 447945
                        maxpin = 423142
                       maxpin% = 80
                 memory_frames = 524288
                 memplace_data = 2
          memplace_mapped_file = 2
        memplace_shm_anonymous = 2
            memplace_shm_named = 2
                memplace_stack = 2
                 memplace_text = 2
        memplace_unmapped_file = 2
                       minfree = 960
                       minperm = 14931
                      minperm% = 3
                     nokilluid = 0
                       npskill = 1024
                       npswarn = 4096
           num_locks_per_semid = 1
                     numpsblks = 131072
               pinnable_frames = 364227
           relalias_percentage = 0
                         scrub = 0
                      v_pinshm = 0
              vmm_default_pspa = 0
                vmm_klock_mode = 0
            wlm_memlimit_nonpg = 1
##Restricted tunables
                  ame_hw_accel = n/a
               ame_sys_memview = n/a
                     batch_tlb = 1
                cpu_scale_memp = 8
         data_stagger_interval = 161
                         defps = 1
enhanced_affinity_attach_limit = 100
     enhanced_affinity_balance = 100
     enhanced_affinity_private = 40
      enhanced_memory_affinity = 0
                     framesets = 2
                     htabscale = n/a
                  kernel_psize = 65536
          large_page_heap_size = 0
               lru_file_repage = 0
             lru_poll_interval = 10
                     lrubucket = 131072
                    maxclient% = 90
                      maxperm% = 90
               mbuf_heap_psize = 65536
               memory_affinity = 1
          multiple_semid_lists = 0
                 munmap_npages = 16384
                     npsrpgmax = 8192
                     npsrpgmin = 6144
                   npsscrubmax = 8192
                   npsscrubmin = 6144
            num_sem_undo_lists = 0
             num_sems_per_lock = 1
              num_spec_dataseg = 0
                numperm_global = 1
             page_steal_method = 1
          psm_timeout_interval = 20000
             relalias_lockmode = 1
                      rpgclean = 0
                    rpgcontrol = 2
                    scrubclean = 0
                shm_1tb_shared = 44
           shm_1tb_unsh_enable = 0
              shm_1tb_unshared = 256
         soft_min_lgpgs_vmpool = 0
              spec_dataseg_int = 512
              strict_maxclient = 1
                strict_maxperm = 0
                   sync_npages = 4096
                 thrpgio_inval = 1024
                thrpgio_npages = 1024
                vm_16m_lruopts = 0
               vm_mmap_areload = 0
          vm_modlist_threshold = -1
              vm_pvlist_dohard = 0
             vm_pvlist_szpcnt = 0
               vmm_fork_policy = 1
            vmm_mpsize_support = 3
               vmm_vmap_policy = 0
                  vtiol_avg_ms = 200
                  vtiol_minreq = 25
            vtiol_minth_active = 1
                    vtiol_mode = 0
               vtiol_pgin_mode = 2
              vtiol_pgout_mode = 2
               vtiol_q_cpu_pct = 2500
          vtiol_thread_cpu_pct = 5000
               wlm_rmem_filter = 0
0
Comment
Question by:it-rex
2 Comments
 
LVL 68

Accepted Solution

by:
woolmilkporc earned 2000 total points
ID: 39862652
Well,

I see some inconsistencies.

You said you don't use large pages.

I assume you didn't set Oracle's "lock_sga" to "true", correct? Please check this, and examine your Oracle user's capabilities for CAP_BYPASS_RAC_VMM and CAP_PROPAGATE:

lsuser -a capabilities oracle_user

Why then did you set aside 128 16MB pages on LPARs 1 and 2?
("lgpg_size = 16777216" and "lgpg_regions = 128")
LPAR 3 doesn't reserve any memory for large pages which is the default behaviour.

Next, you have enabled pinning of shared memory on LPARs 1 and 2 ("v_pinshm = 1").
LPAR 3 has "v_pinshm = 0" (the default).
Further, on LPARs 1 and 2  you raised " maxpin%" (maximum percentage of real memory that can be pinned) from the default "80%" to "90%". LPAR 3 still has the default of "80%", yet this is meaningless there.

As far as I know Oracle doesn't pin SGA memory without "lock_sga = true", but with "v_pinshm = 1" other processes might pin memory pages, namely "aioserver".
How many aioservers do you run ("aioo -a")?

v_pinshm together with the wasted large page regions could actually lead to storage issues.

Some checks you could run at the moment you're reproducing the issue:

Start "nmon" and type "m".
What's the value for "Pinned"? Do the other values look OK?

What does "vmstat -l" report on large pages (last two columns, "alp" and "flp")?

Run as Oracle user:
ps -elf | grep [o]ra_smon_${ORACLE_SID} | awk '{print $4}') | egrep "Vsid|shmat"

What do you see under "Pin"?

What is the "pin" value you get with "svmon -G"? Which is the page size distribution?

Additionally, run

ps -elf | grep [a]ioserver | awk '{print $4}') | egrep "Vsid|shmat"

Pinned memory? How much?

Anyway, I'd suggest changing "v_pinshm" to "0" on the affected LPARs, as well as "lgpg_regions" and "lgpg_size". Done that, restart Oracle and look what happens.

vmo -o v_pinshm=1
vmo -o lgpg_regions=0
vmo -o lgpg_size=0

All parameters are "dynamic", i. e. can be changed on the fly, but to make them last across reboots you must run:

vmo -p -o v_pinshm=1
vmo -p -o lgpg_regions=0
vmo -p -o lgpg_size=0

and run "bosboot -a" against your ipldevice (and possibly mirrored boot disks):

bosboot -a -d /dev/ipldevice

wmp
0
 
LVL 11

Author Closing Comment

by:it-rex
ID: 39927639
thanks
0

Featured Post

How to Use the Help Bell

Need to boost the visibility of your question for solutions? Use the Experts Exchange Help Bell to confirm priority levels and contact subject-matter experts for question attention.  Check out this how-to article for more information.

Question has a verified solution.

If you are experiencing a similar issue, please ask a related question

In tuning file systems on the Solaris Operating System, changing some parameters of a file system usually destroys the data on it. For instance, changing the cache segment block size in the volume of a T3 requires that you delete the existing volu…
I promised to write further about my project, and here I am.  First, I needed to setup the Primary Server.  You can read how in this article: Setup FreeBSD Server with full HDD encryption (http://www.experts-exchange.com/OS/Unix/BSD/FreeBSD/A_3660-S…
Learn how to find files with the shell using the find and locate commands. Use locate to find a needle in a haystack.: With locate, check if the file still exists.: Use find to get the actual location of the file.:
This video shows how to set up a shell script to accept a positional parameter when called, pass that to a SQL script, accept the output from the statement back and then manipulate it in the Shell.
Suggested Courses
Course of the Month11 days, 1 hour left to enroll

886 members asked questions and received personalized solutions in the past 7 days.

Join the community of 500,000 technology professionals and ask your questions.

Join & Ask a Question