Solved

AIX oracle  memory issues

Posted on 2014-02-15
2
3,091 Views
Last Modified: 2014-03-13
This Thread is related to the last 2 Q's
we have run into a major problem after increasing Oracle SGA from 60GB to 160GB at our prod server.

7.1
11g R2 oarcle
no large pages.

the difference is as follows


Original prod lpar where we had the issue

root@1


:/root >vmo -aF
             ame_cpus_per_pool = n/a
               ame_maxfree_mem = n/a
           ame_min_ucpool_size = n/a
               ame_minfree_mem = n/a
               ams_loan_policy = n/a
  enhanced_affinity_affin_time = 1
enhanced_affinity_vmpool_limit = 10
           force_relalias_lite = 0
             kernel_heap_psize = 65536
                  lgpg_regions = 128
                     lgpg_size = 16777216
               low_ps_handling = 1
                       maxfree = 3200
                       maxperm = 67382967
                        maxpin = 70624564
                       maxpin% = 90
                 memory_frames = 78643184
                 memplace_data = 0
          memplace_mapped_file = 0
        memplace_shm_anonymous = 0
            memplace_shm_named = 0
                memplace_stack = 0
                 memplace_text = 0
        memplace_unmapped_file = 0
                       minfree = 1500
                       minperm = 3743495
                      minperm% = 5
                     nokilluid = 0
                       npskill = 125696
                       npswarn = 502784
                     numpsblks = 16089088
               pinnable_frames = 70700261
           relalias_percentage = 0
                         scrub = 0
                      v_pinshm = 1
              vmm_default_pspa = 0
            wlm_memlimit_nonpg = 1
##Restricted tunables
               ame_sys_memview = n/a
                cpu_scale_memp = 8
         data_stagger_interval = 161
                         defps = 1
enhanced_affinity_attach_limit = 100
     enhanced_affinity_balance = 100
     enhanced_affinity_private = 40
      enhanced_memory_affinity = 1
                esid_allocator = 1
                     framesets = 2
                     htabscale = n/a
                  kernel_psize = 65536
          large_page_heap_size = 0
             lru_poll_interval = 10
                     lrubucket = 131072
                    maxclient% = 90
                      maxperm% = 90
               mbuf_heap_psize = 65536
               memory_affinity = 1
                 munmap_npages = 65536
                     npsrpgmax = 1005568
                     npsrpgmin = 754176
                   npsscrubmax = 1005568
                   npsscrubmin = 754176
            num_sem_undo_lists = 0
              num_spec_dataseg = 0
             page_steal_method = 1
          psm_timeout_interval = 20000
             relalias_lockmode = 0
                      rpgclean = 0
                    rpgcontrol = 2
                    scrubclean = 0
                shm_1tb_shared = 12
           shm_1tb_unsh_enable = 1
              shm_1tb_unshared = 256
         soft_min_lgpgs_vmpool = 0
              spec_dataseg_int = 512
              strict_maxclient = 1
                strict_maxperm = 0
          vm_modlist_threshold = -1
               vmm_fork_policy = 1
                vmm_klock_mode = 2
            vmm_mpsize_support = 2
               vmm_vmap_policy = 0
                  vtiol_avg_ms = 200
                  vtiol_minreq = 25
            vtiol_minth_active = 1
                    vtiol_mode = 0
               vtiol_pgin_mode = 2
              vtiol_pgout_mode = 2
               vtiol_q_cpu_pct = 2500
          vtiol_thread_cpu_pct = 5000

 ========================================================================================================================        
      Another test lpar where we were able to reproduce the issue    
root@2

:/root >vmo -aF
             ame_cpus_per_pool = n/a
               ame_maxfree_mem = n/a
           ame_min_ucpool_size = n/a
               ame_minfree_mem = n/a
               ams_loan_policy = n/a
  enhanced_affinity_affin_time = 1
enhanced_affinity_vmpool_limit = 10
                esid_allocator = 1
           force_relalias_lite = 0
             kernel_heap_psize = 65536
                  lgpg_regions = 128
                     lgpg_size = 16777216
               low_ps_handling = 1
                       maxfree = 1088
                       maxperm = 67587160
                        maxpin = 70606996
                       maxpin% = 90
                 memory_frames = 78643200
                 memplace_data = 0
          memplace_mapped_file = 0
        memplace_shm_anonymous = 0
            memplace_shm_named = 0
                memplace_stack = 0
                 memplace_text = 0
        memplace_unmapped_file = 0
                       minfree = 960
                       minperm = 3754838
                      minperm% = 5
                     nokilluid = 0
                       npskill = 167936
                       npswarn = 671744
           num_locks_per_semid = 1
                     numpsblks = 21495808
               pinnable_frames = 69879077
           relalias_percentage = 0
                         scrub = 0
                      v_pinshm = 1
              vmm_default_pspa = 0
                vmm_klock_mode = 2
            wlm_memlimit_nonpg = 1
##Restricted tunables
               ame_sys_memview = n/a
                cpu_scale_memp = 8
         data_stagger_interval = 161
                         defps = 1
enhanced_affinity_attach_limit = 100
     enhanced_affinity_balance = 100
     enhanced_affinity_private = 40
      enhanced_memory_affinity = 1
                     framesets = 2
                     htabscale = n/a
                  kernel_psize = 65536
          large_page_heap_size = 0
             lru_poll_interval = 10
                     lrubucket = 131072
                    maxclient% = 90
                      maxperm% = 90
               mbuf_heap_psize = 65536
               memory_affinity = 1
          multiple_semid_lists = 0
                 munmap_npages = 16384
                     npsrpgmax = 1343488
                     npsrpgmin = 1007616
                   npsscrubmax = 1343488
                   npsscrubmin = 1007616
            num_sem_undo_lists = 0
             num_sems_per_lock = 1
              num_spec_dataseg = 0
             page_steal_method = 1
          psm_timeout_interval = 20000
             relalias_lockmode = 1
                      rpgclean = 0
                    rpgcontrol = 2
                    scrubclean = 0
                shm_1tb_shared = 12
           shm_1tb_unsh_enable = 1
              shm_1tb_unshared = 256
         soft_min_lgpgs_vmpool = 0
              spec_dataseg_int = 512
              strict_maxclient = 1
                strict_maxperm = 0
                   sync_npages = 0
               vm_mmap_areload = 0
          vm_modlist_threshold = -1
              vm_pvlist_dohard = 0
              vm_pvlist_szpcnt = 0
               vmm_fork_policy = 1
            vmm_mpsize_support = 2
               vmm_vmap_policy = 0
                  vtiol_avg_ms = 200
                  vtiol_minreq = 25
            vtiol_minth_active = 1
                    vtiol_mode = 0
               vtiol_pgin_mode = 2
              vtiol_pgout_mode = 2
               vtiol_q_cpu_pct = 2500
          vtiol_thread_cpu_pct = 5000






==================
root@3

a 3rd lpar with higher OSLEVEL where we could not reproduce the issue and it was fine.

:/root >vmo -aF
             ame_cpus_per_pool = n/a
               ame_maxfree_mem = n/a
           ame_min_ucpool_size = n/a
               ame_minfree_mem = n/a
               ams_loan_policy = n/a
  enhanced_affinity_affin_time = 1
enhanced_affinity_vmpool_limit = 10
                esid_allocator = 0
           force_relalias_lite = 0
             kernel_heap_psize = 65536
                  lgpg_regions = 0
                     lgpg_size = 0
               low_ps_handling = 1
                       maxfree = 1088
                       maxperm = 447945
                        maxpin = 423142
                       maxpin% = 80
                 memory_frames = 524288
                 memplace_data = 2
          memplace_mapped_file = 2
        memplace_shm_anonymous = 2
            memplace_shm_named = 2
                memplace_stack = 2
                 memplace_text = 2
        memplace_unmapped_file = 2
                       minfree = 960
                       minperm = 14931
                      minperm% = 3
                     nokilluid = 0
                       npskill = 1024
                       npswarn = 4096
           num_locks_per_semid = 1
                     numpsblks = 131072
               pinnable_frames = 364227
           relalias_percentage = 0
                         scrub = 0
                      v_pinshm = 0
              vmm_default_pspa = 0
                vmm_klock_mode = 0
            wlm_memlimit_nonpg = 1
##Restricted tunables
                  ame_hw_accel = n/a
               ame_sys_memview = n/a
                     batch_tlb = 1
                cpu_scale_memp = 8
         data_stagger_interval = 161
                         defps = 1
enhanced_affinity_attach_limit = 100
     enhanced_affinity_balance = 100
     enhanced_affinity_private = 40
      enhanced_memory_affinity = 0
                     framesets = 2
                     htabscale = n/a
                  kernel_psize = 65536
          large_page_heap_size = 0
               lru_file_repage = 0
             lru_poll_interval = 10
                     lrubucket = 131072
                    maxclient% = 90
                      maxperm% = 90
               mbuf_heap_psize = 65536
               memory_affinity = 1
          multiple_semid_lists = 0
                 munmap_npages = 16384
                     npsrpgmax = 8192
                     npsrpgmin = 6144
                   npsscrubmax = 8192
                   npsscrubmin = 6144
            num_sem_undo_lists = 0
             num_sems_per_lock = 1
              num_spec_dataseg = 0
                numperm_global = 1
             page_steal_method = 1
          psm_timeout_interval = 20000
             relalias_lockmode = 1
                      rpgclean = 0
                    rpgcontrol = 2
                    scrubclean = 0
                shm_1tb_shared = 44
           shm_1tb_unsh_enable = 0
              shm_1tb_unshared = 256
         soft_min_lgpgs_vmpool = 0
              spec_dataseg_int = 512
              strict_maxclient = 1
                strict_maxperm = 0
                   sync_npages = 4096
                 thrpgio_inval = 1024
                thrpgio_npages = 1024
                vm_16m_lruopts = 0
               vm_mmap_areload = 0
          vm_modlist_threshold = -1
              vm_pvlist_dohard = 0
             vm_pvlist_szpcnt = 0
               vmm_fork_policy = 1
            vmm_mpsize_support = 3
               vmm_vmap_policy = 0
                  vtiol_avg_ms = 200
                  vtiol_minreq = 25
            vtiol_minth_active = 1
                    vtiol_mode = 0
               vtiol_pgin_mode = 2
              vtiol_pgout_mode = 2
               vtiol_q_cpu_pct = 2500
          vtiol_thread_cpu_pct = 5000
               wlm_rmem_filter = 0
0
Comment
Question by:it-rex
2 Comments
 
LVL 68

Accepted Solution

by:
woolmilkporc earned 500 total points
ID: 39862652
Well,

I see some inconsistencies.

You said you don't use large pages.

I assume you didn't set Oracle's "lock_sga" to "true", correct? Please check this, and examine your Oracle user's capabilities for CAP_BYPASS_RAC_VMM and CAP_PROPAGATE:

lsuser -a capabilities oracle_user

Why then did you set aside 128 16MB pages on LPARs 1 and 2?
("lgpg_size = 16777216" and "lgpg_regions = 128")
LPAR 3 doesn't reserve any memory for large pages which is the default behaviour.

Next, you have enabled pinning of shared memory on LPARs 1 and 2 ("v_pinshm = 1").
LPAR 3 has "v_pinshm = 0" (the default).
Further, on LPARs 1 and 2  you raised " maxpin%" (maximum percentage of real memory that can be pinned) from the default "80%" to "90%". LPAR 3 still has the default of "80%", yet this is meaningless there.

As far as I know Oracle doesn't pin SGA memory without "lock_sga = true", but with "v_pinshm = 1" other processes might pin memory pages, namely "aioserver".
How many aioservers do you run ("aioo -a")?

v_pinshm together with the wasted large page regions could actually lead to storage issues.

Some checks you could run at the moment you're reproducing the issue:

Start "nmon" and type "m".
What's the value for "Pinned"? Do the other values look OK?

What does "vmstat -l" report on large pages (last two columns, "alp" and "flp")?

Run as Oracle user:
ps -elf | grep [o]ra_smon_${ORACLE_SID} | awk '{print $4}') | egrep "Vsid|shmat"

What do you see under "Pin"?

What is the "pin" value you get with "svmon -G"? Which is the page size distribution?

Additionally, run

ps -elf | grep [a]ioserver | awk '{print $4}') | egrep "Vsid|shmat"

Pinned memory? How much?

Anyway, I'd suggest changing "v_pinshm" to "0" on the affected LPARs, as well as "lgpg_regions" and "lgpg_size". Done that, restart Oracle and look what happens.

vmo -o v_pinshm=1
vmo -o lgpg_regions=0
vmo -o lgpg_size=0

All parameters are "dynamic", i. e. can be changed on the fly, but to make them last across reboots you must run:

vmo -p -o v_pinshm=1
vmo -p -o lgpg_regions=0
vmo -p -o lgpg_size=0

and run "bosboot -a" against your ipldevice (and possibly mirrored boot disks):

bosboot -a -d /dev/ipldevice

wmp
0
 
LVL 11

Author Closing Comment

by:it-rex
ID: 39927639
thanks
0

Featured Post

What Security Threats Are You Missing?

Enhance your security with threat intelligence from the web. Get trending threat insights on hackers, exploits, and suspicious IP addresses delivered to your inbox with our free Cyber Daily.

Join & Write a Comment

Hello fellow BSD lovers, I've created a patch process for patching openjdk6 for BSD (FreeBSD specifically), although I tried to keep all BSD versions in mind when creating my patch. Welcome to OpenJDK6 on BSD First let me start with a little …
Attention: This article will no longer be maintained. If you have any questions, please feel free to mail me. jgh@FreeBSD.org Please see http://www.freebsd.org/doc/en_US.ISO8859-1/articles/freebsd-update-server/ for the updated article. It is avail…
Learn several ways to interact with files and get file information from the bash shell. ls lists the contents of a directory: Using the -a flag displays hidden files: Using the -l flag formats the output in a long list: The file command gives us mor…
In a previous video, we went over how to export a DynamoDB table into Amazon S3.  In this video, we show how to load the export from S3 into a DynamoDB table.

747 members asked questions and received personalized solutions in the past 7 days.

Join the community of 500,000 technology professionals and ask your questions.

Join & Ask a Question

Need Help in Real-Time?

Connect with top rated Experts

10 Experts available now in Live!

Get 1:1 Help Now