vCenter server 6.7 - 503 Service Unavailable

this service seems to keep on restarting and I see the below errors

# systemctl status vmware-pod

Mar 26 08:30:42 xxxxxxxxxxxxxx pod[109474]: Exception: Could not add service vmware.impl due to Traceback (most recent call last):
Mar 26 08:30:42 xxxxxxxxxxxxxx pod[109474]: File "/usr/lib/vmware-pod/lib/vapi/vapi_runtime-2.100.0-py2.py3-none-any.whl/vmware/vapi/provider/l...roperties
Mar 26 08:30:42 xxxxxxxxxxxxxx pod[109474]: module = __import__(service, globals(), locals(), [fn])
Mar 26 08:30:42 xxxxxxxxxxxxxx pod[109474]: File "/usr/lib/vmware-pod/py/vmware/impl.py", line 5, in <module>
Mar 26 08:30:42 xxxxxxxxxxxxxx pod[109474]: from vmware.pod_impl import CredentialsImpl, ServiceImpl
Mar 26 08:30:42 xxxxxxxxxxxxxx pod[109474]: File "/usr/lib/vmware-pod/py/vmware/pod_impl.py", line 17, in <module>
Mar 26 08:30:42 xxxxxxxxxxxxxx pod[109474]: from driver.vcenter.vc_lib import VcService
Mar 26 08:30:42 xxxxxxxxxxxxxx pod[109474]: File "/usr/lib/vmware-pod/py/driver/vcenter/vc_lib.py", line 16, in <module>
Mar 26 08:30:42 xxxxxxxxxxxxxx pod[109474]: from pyVim.invt import getContainingDatacenter
Mar 26 08:30:42 xxxxxxxxxxxxxx pod[109474]: ImportError: No module named 'pyVim.invt'

below partition is 100% full but many posts had suggested that its normal

/dev/mapper/archive_vg-archive             50G   47G   12M 100% /storage/archive

error messages

root@xxxxxxxxxxxxxx [ /storage/log/vmware ]# cat messages  | grep -i error
2020-03-26T07:25:33.444136-04:00 xxxxxxxxxxxxxx cli: vmware.vherd.linux_cli.cli Error getting authentication cookie from applmgmt service. Verify that the applmgmt service is running and that the method requested to obtain the cookie requires authentication.
2020-03-26T07:25:41.994142-04:00 xxxxxxxxxxxxxx cli: vmware.vherd.linux_cli.cli Error getting authentication cookie from applmgmt service. Verify that the applmgmt service is running and that the method requested to obtain the cookie requires authentication.
2020-03-26T08:45:42.991391-04:00 xxxxxxxxxxxxxx systemd-udevd[759]: error opening ATTR{/sys/devices/system/cpu/cpu0/online} for writing: Permission denied
2020-03-26T08:45:43.356250-04:00 xxxxxxxxxxxxxx augenrules[1081]: Error sending add rule data request (Rule exists)
2020-03-26T08:45:43.356383-04:00 xxxxxxxxxxxxxx augenrules[1081]: There was an error in line 91 of /etc/audit/audit.rules
2020-03-26T08:45:43.615930-04:00 xxxxxxxxxxxxxx vami-lighttp[1110]: Error: Failed to open the store.
2020-03-26T08:45:43.616594-04:00 xxxxxxxxxxxxxx vami-lighttp[1110]: vecs-cli failed. Error 1021: Cannot connect to vmafd service.
2020-03-26T08:46:05.932692-04:00 xxxxxxxxxxxxxx ntpd[1268]: kernel reports TIME_ERROR: 0x41: Clock Unsynchronized
2020-03-26T08:46:05.933281-04:00 xxxxxxxxxxxxxx ntpd[1268]: kernel reports TIME_ERROR: 0x41: Clock Unsynchronized
2020-03-26T08:46:29.231702-04:00 xxxxxxxxxxxxxx cli: vmware.vherd.linux_cli.cli Error getting authentication cookie from applmgmt service. Verify that the applmgmt service is running and that the method requested to obtain the cookie requires authentication.
2020-03-26T08:46:31.870687-04:00 xxxxxxxxxxxxxx lsassd[1543]: 0x7fc1b38d0700:C_GetFunctionList() failed, error: 0x00000006
2020-03-26T08:46:35.666314-04:00 xxxxxxxxxxxxxx lsassd[1543]: 0x7fc1b38d0700:Ignoring failure enumerating trusts for forest xxxxxxxxxxxx.com. Error was <null> (40121)

check /var/log/vmware/cm/cm.log or /var/log/vmware/vpxd-svcs/vpxd-svcs.log and attach it here please

cm.log is empty

root@xxxxxxxxxxxxxx [ /var/log/vmware/cm ]# ls -ltrh
total 60K
drwxr-xr-x 2 root root 4.0K Jul 19  2019 firstboot
-rw-r--r-- 1 root root    0 Jul 19  2019 cm.log
lrwxrwxrwx 1 root root   18 Jul 22  2019 cm -> /var/log/vmware/cm
-rw------- 1 root root   42 Mar 18 07:39 vmon-4.stdout
-rw------- 1 root root   88 Mar 18 07:39 vmon-4.stderr
-rw------- 1 root root   41 Mar 20 08:37 vmon-3.stdout
-rw------- 1 root root   87 Mar 20 08:37 vmon-3.stderr
-rw------- 1 root root   41 Mar 20 09:22 vmon-2.stdout
-rw------- 1 root root   87 Mar 20 09:22 vmon-2.stderr
-rw------- 1 root root   41 Mar 25 05:55 vmon-1.stdout
-rw------- 1 root root   87 Mar 25 05:55 vmon-1.stderr
-rw------- 1 root root   42 Mar 26 07:32 vmon-0.stdout
-rw------- 1 root root   88 Mar 26 07:32 vmon-0.stderr
-rw------- 1 root root   41 Mar 26 08:49 vmon.stdout
-rw------- 1 root root   87 Mar 26 08:49 vmon.stderr
-rw-r--r-- 1 root root 4.3K Mar 26 08:56 vmware-cm-gc.log.0.current
root@xxxxxxxxxxxxxx [ /var/log/vmware/cm ]# cat vmon.stderr
Starting service process with pid: 4759.
Picked up JAVA_TOOL_OPTIONS: -Xms32M -Xmx128M
root@xxxxxxxxxxxxxx [ /var/log/vmware/cm ]# cat vmon.stdout
Starting service process with pid: 4759.
root@xxxxxxxxxxxxxx [ /var/log/vmware/cm ]# cat vmon-0.stderr
Starting service process with pid: 58059.
Picked up JAVA_TOOL_OPTIONS: -Xms32M -Xmx128M
root@xxxxxxxxxxxxxx [ /var/log/vmware/cm ]# cat vmware-cm-gc.log.0.current
Java HotSpot(TM) 64-Bit Server VM (25.212-b31) for linux-amd64 JRE (1.8.0_212-b31), built on Apr  4 2019 21:39:53 by "java_re" with gcc 7.3.0
Memory: 4k page, physical 20543260k(18367152k free), swap 27254776k(27254776k free)
CommandLine flags: -XX:CompressedClassSpaceSize=134217728 -XX:ErrorFile=/var/log/vmware/cm/java_error%p.log -XX:+ForceTimeHighResolution -XX:GCLogFileSize=1048576 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/var/log/vmware/cm -XX:InitialHeapSize=33554432 -XX:MaxHeapSize=198180864 -XX:NumberOfGCLogFiles=10 -XX:ParallelGCThreads=1 -XX:+PrintGC -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintReferenceGC -XX:ThreadStackSize=256 -XX:+UseCompressedClassPointers -XX:+UseCompressedOops -XX:+UseGCLogFileRotation -XX:+UseParallelGC
2020-03-26T08:50:11.157-0400: 12.396: [GC (Allocation Failure) 2020-03-26T08:50:11.162-0400: 12.401: [SoftReference, 0 refs, 0.0000716 secs]2020-03-26T08:50:11.162-0400: 12.401: [WeakReference, 16 refs, 0.0000301 secs]2020-03-26T08:50:11.162-0400: 12.401: [FinalReference, 554 refs, 0.0019062 secs]2020-03-26T08:50:11.164-0400: 12.403: [PhantomReference, 0 refs, 0 refs, 0.0000214 secs]2020-03-26T08:50:11.164-0400: 12.403: [JNI Weak Reference, 0.0000257 secs][PSYoungGen: 8704K->1008K(9728K)] 8704K->1409K(31744K), 0.0083881 secs] [Times: user=0.00 sys=0.01, real=0.00 secs]
2020-03-26T08:50:12.487-0400: 13.726: [GC (Allocation Failure) 2020-03-26T08:50:12.493-0400: 13.731: [SoftReference, 0 refs, 0.0000766 secs]2020-03-26T08:50:12.493-0400: 13.732: [WeakReference, 10 refs, 0.0000184 secs]2020-03-26T08:50:12.493-0400: 13.732: [FinalReference, 482 refs, 0.0036522 secs]2020-03-26T08:50:12.497-0400: 13.735: [PhantomReference, 0 refs, 0 refs, 0.0000213 secs]2020-03-26T08:50:12.497-0400: 13.735: [JNI Weak Reference, 0.0000142 secs][PSYoungGen: 9712K->1000K(9728K)] 10113K->2030K(31744K), 0.0101492 secs] [Times: user=0.00 sys=0.00, real=0.01 secs]
2020-03-26T08:50:12.837-0400: 14.076: [GC (Allocation Failure) 2020-03-26T08:50:12.841-0400: 14.079: [SoftReference, 0 refs, 0.0000324 secs]2020-03-26T08:50:12.841-0400: 14.080: [WeakReference, 10 refs, 0.0000168 secs]2020-03-26T08:50:12.841-0400: 14.080: [FinalReference, 460 refs, 0.0005478 secs]2020-03-26T08:50:12.841-0400: 14.080: [PhantomReference, 0 refs, 0 refs, 0.0000276 secs]2020-03-26T08:50:12.841-0400: 14.080: [JNI Weak Reference, 0.0000171 secs][PSYoungGen: 9704K->1016K(9728K)] 10734K->2586K(31744K), 0.0049020 secs] [Times: user=0.00 sys=0.00, real=0.00 secs]
2020-03-26T08:50:13.365-0400: 14.604: [GC (Allocation Failure) 2020-03-26T08:50:13.373-0400: 14.611: [SoftReference, 0 refs, 0.0000536 secs]2020-03-26T08:50:13.373-0400: 14.611: [WeakReference, 15 refs, 0.0000189 secs]2020-03-26T08:50:13.373-0400: 14.611: [FinalReference, 417 refs, 0.0011503 secs]2020-03-26T08:50:13.374-0400: 14.613: [PhantomReference, 0 refs, 0 refs, 0.0000452 secs]2020-03-26T08:50:13.374-0400: 14.613: [JNI Weak Reference, 0.0000138 secs][PSYoungGen: 9720K->1016K(9728K)] 11290K->3430K(31744K), 0.0093017 secs] [Times: user=0.01 sys=0.00, real=0.02 secs]
2020-03-26T08:50:13.861-0400: 15.100: [GC (Allocation Failure) 2020-03-26T08:50:13.886-0400: 15.124: [SoftReference, 0 refs, 0.0001085 secs]2020-03-26T08:50:13.886-0400: 15.125: [WeakReference, 6 refs, 0.0000211 secs]2020-03-26T08:50:13.886-0400: 15.125: [FinalReference, 0 refs, 0.0000141 secs]2020-03-26T08:50:13.886-0400: 15.125: [PhantomReference, 0 refs, 0 refs, 0.0000139 secs]2020-03-26T08:50:13.886-0400: 15.125: [JNI Weak Reference, 0.0000133 secs][PSYoungGen: 9720K->1016K(9728K)] 12134K->4552K(31744K), 0.0259272 secs] [Times: user=0.01 sys=0.00, real=0.03 secs]
Heap
 PSYoungGen      total 9728K, used 2615K [0x00000000fc100000, 0x00000000fd700000, 0x0000000100000000)
  eden space 8704K, 18% used [0x00000000fc100000,0x00000000fc28fd40,0x00000000fc980000)
  from space 1024K, 99% used [0x00000000fc980000,0x00000000fca7e060,0x00000000fca80000)
  to   space 2560K, 0% used [0x00000000fd480000,0x00000000fd480000,0x00000000fd700000)
 ParOldGen       total 22016K, used 3536K [0x00000000f4200000, 0x00000000f5780000, 0x00000000fc100000)
  object space 22016K, 16% used [0x00000000f4200000,0x00000000f4574118,0x00000000f5780000)
 Metaspace       used 9810K, capacity 9936K, committed 9984K, reserved 141312K
  class space    used 1204K, capacity 1239K, committed 1280K, reserved 131072K
root@xxxxxxxxxxxxxx [ /var/log/vmware/cm ]# cat cm.log
root@xxxxxxxxxxxxxx [ /var/log/vmware/cm ]#

vpxd-svcs.log attached.  cm.log was empty as seen in my previous comment.
vpxd-svcs.log

can you start vmafdd service again and wanted to check as the vpxd-svcs is down and log being not updated

both started

root@xxxxxxxxxx [ ~ ]# service-control --start --verbose vmafdd
Operation not cancellable. Please wait for it to finish...
Perform start operation. vmon_profile=None, svc_names=['vmafdd'], include_coreossvcs=False, include_leafossvcs=False
Performing start operation on service vmafdd...
2020-03-27T13:26:48.290Z  Running command: ['/usr/bin/systemctl', 'is-enabled', 'vmafdd']
2020-03-27T13:26:48.385Z  Done running command
2020-03-27T13:26:48.386Z  Running command: ['/sbin/service', 'vmafdd', 'status']
2020-03-27T13:26:48.798Z  Done running command
Successfully started service vmafdd

root@xxxxxxxxxx [ ~ ]# service-control --start --verbose vpxd-svcs
Operation not cancellable. Please wait for it to finish...
Perform start operation. vmon_profile=None, svc_names=['vpxd-svcs'], include_coreossvcs=False, include_leafossvcs=False
Performing start operation on service vpxd-svcs...
2020-03-27T13:27:24.900Z  Running command: ['/sbin/service', 'vmware-vmon', 'status']
2020-03-27T13:27:25.610Z  Done running command
2020-03-27T13:27:25.660Z  Service vpxd-svcs state STOPPED
2020-03-27T13:28:12.031Z  Successfully started vpxd-svcs service
Successfully started service vpxd-svcs

please start all using service-control --start --all

root@xxxxxxx [ ~ ]# service-control --start --verbose vmafdd
Operation not cancellable. Please wait for it to finish...
Perform start operation. vmon_profile=None, svc_names=['vmafdd'], include_coreossvcs=False, include_leafossvcs=False
Performing start operation on service vmafdd...
2020-03-30T04:48:34.605Z  Running command: ['/usr/bin/systemctl', 'is-enabled', 'vmafdd']
2020-03-30T04:48:34.620Z  Done running command
2020-03-30T04:48:34.621Z  Running command: ['/sbin/service', 'vmafdd', 'status']
2020-03-30T04:48:34.681Z  Done running command
Successfully started service vmafdd

root@xxxxxxx [ ~ ]#  service-control --start --verbose vpxd-svcs
Operation not cancellable. Please wait for it to finish...
Perform start operation. vmon_profile=None, svc_names=['vpxd-svcs'], include_coreossvcs=False, include_leafossvcs=False
Performing start operation on service vpxd-svcs...
2020-03-30T04:48:43.096Z  Running command: ['/sbin/service', 'vmware-vmon', 'status']
2020-03-30T04:48:43.169Z  Done running command
2020-03-30T04:48:43.187Z  Service vpxd-svcs state STARTED
Successfully started service vpxd-svcs

root@xxxxxxx [ ~ ]# service-control --start --verbose --all
Operation not cancellable. Please wait for it to finish...
Perform start operation. vmon_profile=ALL, svc_names=None, include_coreossvcs=True, include_leafossvcs=True
Performing start operation on service lwsmd...
2020-03-30T04:48:52.360Z  Running command: ['/usr/bin/systemctl', 'is-enabled', 'lwsmd']
2020-03-30T04:48:52.400Z  Done running command
2020-03-30T04:48:52.400Z  Running command: ['/sbin/service', 'lwsmd', 'status']
2020-03-30T04:48:52.473Z  Done running command
Successfully started service lwsmd
Performing start operation on service vmafdd...
2020-03-30T04:48:52.475Z  Running command: ['/usr/bin/systemctl', 'is-enabled', 'vmafdd']
2020-03-30T04:48:52.525Z  Done running command
2020-03-30T04:48:52.526Z  Running command: ['/sbin/service', 'vmafdd', 'status']
2020-03-30T04:48:52.627Z  Done running command
Successfully started service vmafdd
Performing start operation on service vmdird...
2020-03-30T04:48:52.678Z  Running command: ['/usr/bin/systemctl', 'is-enabled', 'vmdird']
2020-03-30T04:48:52.762Z  Done running command
2020-03-30T04:48:52.763Z  Running command: ['/sbin/service', 'vmdird', 'status']
2020-03-30T04:48:52.841Z  Done running command
Successfully started service vmdird
Performing start operation on service vmcad...
2020-03-30T04:48:52.843Z  Running command: ['/usr/bin/systemctl', 'is-enabled', 'vmcad']
2020-03-30T04:48:52.869Z  Done running command
2020-03-30T04:48:52.869Z  Running command: ['/sbin/service', 'vmcad', 'status']
2020-03-30T04:48:52.960Z  Done running command
Successfully started service vmcad
Performing start operation on service vmware-sts-idmd...
2020-03-30T04:48:52.961Z  Running command: ['/usr/bin/systemctl', 'is-enabled', 'vmware-sts-idmd']
2020-03-30T04:48:52.982Z  Done running command
2020-03-30T04:48:52.983Z  Running command: ['/sbin/service', 'vmware-sts-idmd', 'status']
2020-03-30T04:48:53.035Z  Done running command
Successfully started service vmware-sts-idmd
Performing start operation on service vmware-stsd...
2020-03-30T04:48:53.037Z  Running command: ['/usr/bin/systemctl', 'is-enabled', 'vmware-stsd']
2020-03-30T04:48:53.051Z  Done running command
2020-03-30T04:48:53.052Z  Running command: ['/sbin/service', 'vmware-stsd', 'status']
2020-03-30T04:48:53.116Z  Done running command
Successfully started service vmware-stsd
Performing start operation on service vmdnsd...
2020-03-30T04:48:53.117Z  Running command: ['/usr/bin/systemctl', 'is-enabled', 'vmdnsd']
2020-03-30T04:48:53.132Z  Done running command
2020-03-30T04:48:53.132Z  Running command: ['/sbin/service', 'vmdnsd', 'status']
2020-03-30T04:48:53.189Z  Done running command
Successfully started service vmdnsd
Performing start operation on profile: ALL...
2020-03-30T04:48:53.192Z  Running command: ['/sbin/service', 'vmware-vmon', 'status']
2020-03-30T04:48:53.277Z  Done running command
2020-03-30T04:48:53.278Z  Running command: ['/usr/lib/vmware-vmon/vmon-cli', '--batchstart', 'ALL']
2020-03-30T05:00:55.628Z  Done running command
Service-control failed. Error: Failed to start services in profile ALL. RC=1, stderr=Failed to start topologysvc, vmonapi, applmgmt, cm services. Error: Operation timed out

root@xxxxxxx [ ~ ]#

please help... still its not working

Is the time and timezone on your vCenter correct?
Can you verify that? Is it syncing time from the AD?

Sudeep