Solved

AIX oracle  memory issues

Posted on 2014-02-15
2
3,342 Views
Last Modified: 2014-03-13
This Thread is related to the last 2 Q's
we have run into a major problem after increasing Oracle SGA from 60GB to 160GB at our prod server.

7.1
11g R2 oarcle
no large pages.

the difference is as follows


Original prod lpar where we had the issue

root@1


:/root >vmo -aF
             ame_cpus_per_pool = n/a
               ame_maxfree_mem = n/a
           ame_min_ucpool_size = n/a
               ame_minfree_mem = n/a
               ams_loan_policy = n/a
  enhanced_affinity_affin_time = 1
enhanced_affinity_vmpool_limit = 10
           force_relalias_lite = 0
             kernel_heap_psize = 65536
                  lgpg_regions = 128
                     lgpg_size = 16777216
               low_ps_handling = 1
                       maxfree = 3200
                       maxperm = 67382967
                        maxpin = 70624564
                       maxpin% = 90
                 memory_frames = 78643184
                 memplace_data = 0
          memplace_mapped_file = 0
        memplace_shm_anonymous = 0
            memplace_shm_named = 0
                memplace_stack = 0
                 memplace_text = 0
        memplace_unmapped_file = 0
                       minfree = 1500
                       minperm = 3743495
                      minperm% = 5
                     nokilluid = 0
                       npskill = 125696
                       npswarn = 502784
                     numpsblks = 16089088
               pinnable_frames = 70700261
           relalias_percentage = 0
                         scrub = 0
                      v_pinshm = 1
              vmm_default_pspa = 0
            wlm_memlimit_nonpg = 1
##Restricted tunables
               ame_sys_memview = n/a
                cpu_scale_memp = 8
         data_stagger_interval = 161
                         defps = 1
enhanced_affinity_attach_limit = 100
     enhanced_affinity_balance = 100
     enhanced_affinity_private = 40
      enhanced_memory_affinity = 1
                esid_allocator = 1
                     framesets = 2
                     htabscale = n/a
                  kernel_psize = 65536
          large_page_heap_size = 0
             lru_poll_interval = 10
                     lrubucket = 131072
                    maxclient% = 90
                      maxperm% = 90
               mbuf_heap_psize = 65536
               memory_affinity = 1
                 munmap_npages = 65536
                     npsrpgmax = 1005568
                     npsrpgmin = 754176
                   npsscrubmax = 1005568
                   npsscrubmin = 754176
            num_sem_undo_lists = 0
              num_spec_dataseg = 0
             page_steal_method = 1
          psm_timeout_interval = 20000
             relalias_lockmode = 0
                      rpgclean = 0
                    rpgcontrol = 2
                    scrubclean = 0
                shm_1tb_shared = 12
           shm_1tb_unsh_enable = 1
              shm_1tb_unshared = 256
         soft_min_lgpgs_vmpool = 0
              spec_dataseg_int = 512
              strict_maxclient = 1
                strict_maxperm = 0
          vm_modlist_threshold = -1
               vmm_fork_policy = 1
                vmm_klock_mode = 2
            vmm_mpsize_support = 2
               vmm_vmap_policy = 0
                  vtiol_avg_ms = 200
                  vtiol_minreq = 25
            vtiol_minth_active = 1
                    vtiol_mode = 0
               vtiol_pgin_mode = 2
              vtiol_pgout_mode = 2
               vtiol_q_cpu_pct = 2500
          vtiol_thread_cpu_pct = 5000

 ========================================================================================================================        
      Another test lpar where we were able to reproduce the issue    
root@2

:/root >vmo -aF
             ame_cpus_per_pool = n/a
               ame_maxfree_mem = n/a
           ame_min_ucpool_size = n/a
               ame_minfree_mem = n/a
               ams_loan_policy = n/a
  enhanced_affinity_affin_time = 1
enhanced_affinity_vmpool_limit = 10
                esid_allocator = 1
           force_relalias_lite = 0
             kernel_heap_psize = 65536
                  lgpg_regions = 128
                     lgpg_size = 16777216
               low_ps_handling = 1
                       maxfree = 1088
                       maxperm = 67587160
                        maxpin = 70606996
                       maxpin% = 90
                 memory_frames = 78643200
                 memplace_data = 0
          memplace_mapped_file = 0
        memplace_shm_anonymous = 0
            memplace_shm_named = 0
                memplace_stack = 0
                 memplace_text = 0
        memplace_unmapped_file = 0
                       minfree = 960
                       minperm = 3754838
                      minperm% = 5
                     nokilluid = 0
                       npskill = 167936
                       npswarn = 671744
           num_locks_per_semid = 1
                     numpsblks = 21495808
               pinnable_frames = 69879077
           relalias_percentage = 0
                         scrub = 0
                      v_pinshm = 1
              vmm_default_pspa = 0
                vmm_klock_mode = 2
            wlm_memlimit_nonpg = 1
##Restricted tunables
               ame_sys_memview = n/a
                cpu_scale_memp = 8
         data_stagger_interval = 161
                         defps = 1
enhanced_affinity_attach_limit = 100
     enhanced_affinity_balance = 100
     enhanced_affinity_private = 40
      enhanced_memory_affinity = 1
                     framesets = 2
                     htabscale = n/a
                  kernel_psize = 65536
          large_page_heap_size = 0
             lru_poll_interval = 10
                     lrubucket = 131072
                    maxclient% = 90
                      maxperm% = 90
               mbuf_heap_psize = 65536
               memory_affinity = 1
          multiple_semid_lists = 0
                 munmap_npages = 16384
                     npsrpgmax = 1343488
                     npsrpgmin = 1007616
                   npsscrubmax = 1343488
                   npsscrubmin = 1007616
            num_sem_undo_lists = 0
             num_sems_per_lock = 1
              num_spec_dataseg = 0
             page_steal_method = 1
          psm_timeout_interval = 20000
             relalias_lockmode = 1
                      rpgclean = 0
                    rpgcontrol = 2
                    scrubclean = 0
                shm_1tb_shared = 12
           shm_1tb_unsh_enable = 1
              shm_1tb_unshared = 256
         soft_min_lgpgs_vmpool = 0
              spec_dataseg_int = 512
              strict_maxclient = 1
                strict_maxperm = 0
                   sync_npages = 0
               vm_mmap_areload = 0
          vm_modlist_threshold = -1
              vm_pvlist_dohard = 0
              vm_pvlist_szpcnt = 0
               vmm_fork_policy = 1
            vmm_mpsize_support = 2
               vmm_vmap_policy = 0
                  vtiol_avg_ms = 200
                  vtiol_minreq = 25
            vtiol_minth_active = 1
                    vtiol_mode = 0
               vtiol_pgin_mode = 2
              vtiol_pgout_mode = 2
               vtiol_q_cpu_pct = 2500
          vtiol_thread_cpu_pct = 5000






==================
root@3

a 3rd lpar with higher OSLEVEL where we could not reproduce the issue and it was fine.

:/root >vmo -aF
             ame_cpus_per_pool = n/a
               ame_maxfree_mem = n/a
           ame_min_ucpool_size = n/a
               ame_minfree_mem = n/a
               ams_loan_policy = n/a
  enhanced_affinity_affin_time = 1
enhanced_affinity_vmpool_limit = 10
                esid_allocator = 0
           force_relalias_lite = 0
             kernel_heap_psize = 65536
                  lgpg_regions = 0
                     lgpg_size = 0
               low_ps_handling = 1
                       maxfree = 1088
                       maxperm = 447945
                        maxpin = 423142
                       maxpin% = 80
                 memory_frames = 524288
                 memplace_data = 2
          memplace_mapped_file = 2
        memplace_shm_anonymous = 2
            memplace_shm_named = 2
                memplace_stack = 2
                 memplace_text = 2
        memplace_unmapped_file = 2
                       minfree = 960
                       minperm = 14931
                      minperm% = 3
                     nokilluid = 0
                       npskill = 1024
                       npswarn = 4096
           num_locks_per_semid = 1
                     numpsblks = 131072
               pinnable_frames = 364227
           relalias_percentage = 0
                         scrub = 0
                      v_pinshm = 0
              vmm_default_pspa = 0
                vmm_klock_mode = 0
            wlm_memlimit_nonpg = 1
##Restricted tunables
                  ame_hw_accel = n/a
               ame_sys_memview = n/a
                     batch_tlb = 1
                cpu_scale_memp = 8
         data_stagger_interval = 161
                         defps = 1
enhanced_affinity_attach_limit = 100
     enhanced_affinity_balance = 100
     enhanced_affinity_private = 40
      enhanced_memory_affinity = 0
                     framesets = 2
                     htabscale = n/a
                  kernel_psize = 65536
          large_page_heap_size = 0
               lru_file_repage = 0
             lru_poll_interval = 10
                     lrubucket = 131072
                    maxclient% = 90
                      maxperm% = 90
               mbuf_heap_psize = 65536
               memory_affinity = 1
          multiple_semid_lists = 0
                 munmap_npages = 16384
                     npsrpgmax = 8192
                     npsrpgmin = 6144
                   npsscrubmax = 8192
                   npsscrubmin = 6144
            num_sem_undo_lists = 0
             num_sems_per_lock = 1
              num_spec_dataseg = 0
                numperm_global = 1
             page_steal_method = 1
          psm_timeout_interval = 20000
             relalias_lockmode = 1
                      rpgclean = 0
                    rpgcontrol = 2
                    scrubclean = 0
                shm_1tb_shared = 44
           shm_1tb_unsh_enable = 0
              shm_1tb_unshared = 256
         soft_min_lgpgs_vmpool = 0
              spec_dataseg_int = 512
              strict_maxclient = 1
                strict_maxperm = 0
                   sync_npages = 4096
                 thrpgio_inval = 1024
                thrpgio_npages = 1024
                vm_16m_lruopts = 0
               vm_mmap_areload = 0
          vm_modlist_threshold = -1
              vm_pvlist_dohard = 0
             vm_pvlist_szpcnt = 0
               vmm_fork_policy = 1
            vmm_mpsize_support = 3
               vmm_vmap_policy = 0
                  vtiol_avg_ms = 200
                  vtiol_minreq = 25
            vtiol_minth_active = 1
                    vtiol_mode = 0
               vtiol_pgin_mode = 2
              vtiol_pgout_mode = 2
               vtiol_q_cpu_pct = 2500
          vtiol_thread_cpu_pct = 5000
               wlm_rmem_filter = 0
0
Comment
Question by:it-rex
[X]
Welcome to Experts Exchange

Add your voice to the tech community where 5M+ people just like you are talking about what matters.

  • Help others & share knowledge
  • Earn cash & points
  • Learn & ask questions
2 Comments
 
LVL 68

Accepted Solution

by:
woolmilkporc earned 500 total points
ID: 39862652
Well,

I see some inconsistencies.

You said you don't use large pages.

I assume you didn't set Oracle's "lock_sga" to "true", correct? Please check this, and examine your Oracle user's capabilities for CAP_BYPASS_RAC_VMM and CAP_PROPAGATE:

lsuser -a capabilities oracle_user

Why then did you set aside 128 16MB pages on LPARs 1 and 2?
("lgpg_size = 16777216" and "lgpg_regions = 128")
LPAR 3 doesn't reserve any memory for large pages which is the default behaviour.

Next, you have enabled pinning of shared memory on LPARs 1 and 2 ("v_pinshm = 1").
LPAR 3 has "v_pinshm = 0" (the default).
Further, on LPARs 1 and 2  you raised " maxpin%" (maximum percentage of real memory that can be pinned) from the default "80%" to "90%". LPAR 3 still has the default of "80%", yet this is meaningless there.

As far as I know Oracle doesn't pin SGA memory without "lock_sga = true", but with "v_pinshm = 1" other processes might pin memory pages, namely "aioserver".
How many aioservers do you run ("aioo -a")?

v_pinshm together with the wasted large page regions could actually lead to storage issues.

Some checks you could run at the moment you're reproducing the issue:

Start "nmon" and type "m".
What's the value for "Pinned"? Do the other values look OK?

What does "vmstat -l" report on large pages (last two columns, "alp" and "flp")?

Run as Oracle user:
ps -elf | grep [o]ra_smon_${ORACLE_SID} | awk '{print $4}') | egrep "Vsid|shmat"

What do you see under "Pin"?

What is the "pin" value you get with "svmon -G"? Which is the page size distribution?

Additionally, run

ps -elf | grep [a]ioserver | awk '{print $4}') | egrep "Vsid|shmat"

Pinned memory? How much?

Anyway, I'd suggest changing "v_pinshm" to "0" on the affected LPARs, as well as "lgpg_regions" and "lgpg_size". Done that, restart Oracle and look what happens.

vmo -o v_pinshm=1
vmo -o lgpg_regions=0
vmo -o lgpg_size=0

All parameters are "dynamic", i. e. can be changed on the fly, but to make them last across reboots you must run:

vmo -p -o v_pinshm=1
vmo -p -o lgpg_regions=0
vmo -p -o lgpg_size=0

and run "bosboot -a" against your ipldevice (and possibly mirrored boot disks):

bosboot -a -d /dev/ipldevice

wmp
0
 
LVL 11

Author Closing Comment

by:it-rex
ID: 39927639
thanks
0

Featured Post

Free Tool: IP Lookup

Get more info about an IP address or domain name, such as organization, abuse contacts and geolocation.

One of a set of tools we are providing to everyone as a way of saying thank you for being a part of the community.

Question has a verified solution.

If you are experiencing a similar issue, please ask a related question

Suggested Solutions

Hello fellow BSD lovers, I've created a patch process for patching openjdk6 for BSD (FreeBSD specifically), although I tried to keep all BSD versions in mind when creating my patch. Welcome to OpenJDK6 on BSD First let me start with a little …
Every server (virtual or physical) needs a console: and the console can be provided through hardware directly connected, software for remote connections, local connections, through a KVM, etc. This document explains the different types of consol…
Learn how to get help with Linux/Unix bash shell commands. Use help to read help documents for built in bash shell commands.: Use man to interface with the online reference manuals for shell commands.: Use man to search man pages for unknown command…
Learn how to find files with the shell using the find and locate commands. Use locate to find a needle in a haystack.: With locate, check if the file still exists.: Use find to get the actual location of the file.:
Suggested Courses

710 members asked questions and received personalized solutions in the past 7 days.

Join the community of 500,000 technology professionals and ask your questions.

Join & Ask a Question