Solved

Apache suddenly won't start with SSLEngine On - "Server should be SSL-aware but has no certificate configured"

Posted on 2010-08-23
12
2,419 Views
Last Modified: 2012-06-27
SLES 10 SP 2
Apache 2.2.3

I thought I was darn near an Apache expert, but this problem has really humbled me.   My server was running fine for a year since it's install and then my self-signed certificate was to expire in 4 days.   Well before those 4 days were up, I had to reboot my server for other reasons and Apache would not start after that.   Aside from the near-expiration of my cert, the ONLY other thing I can think of changing is that I recently upgraded Groupwise on this box, and there are two config files included for Groupwise's web access.   However, I have commented those references out and the problem persists.

Apache simply won't start, and the only error I can get no matter what I do is the following:

 Server should be SSL-aware but has no certificate configured [HINT: SSLCertificatefile]

Clearly the error message isn't very helpful since a) I do have an SSLCertificateFile directive and b) I get the exact same message even if the key or cert files configured are missing.  

Steps I've taken:

- Renewed my cert.   Same error.
- Generated a new key and cert.  Same error.
- Verified my cert with openssl verify.   Returns ok.
- Commented ALL unnecessary conf files included in httpd.conf.   Same error as long as SSLEngine is on.
- Did the standard Google searches; most people say this occurs if you have SSL configured inside a virtual host.   I have checked and double-checked - I have NO virtual hosts defined anywhere.  
- Ran apachectl configtest which returns "syntax ok".
- Changed the SSLRandomSeed directive from "builtin" to use /dev/random.  Same error.

I have renewed my certificate and installed a new one but that didn't help.   I even generated a new key and a new cert and I get the same error.    The error will occur as long as SSLEngine is turned on.  

I ran an strace -fF on apachectl and I still can't figure out what the heck the issue is.   It doesn't even appear to try to open the cert and key files.    It seems to bind, read mime.types, then open /dev/urandom and finally write the error to the error_log.

7116  socket(PF_NETLINK, SOCK_RAW, 0)   = 3
7116  bind(3, {sa_family=AF_NETLINK, pid=0, groups=00000000}, 12) = 0
7116  getsockname(3, {sa_family=AF_NETLINK, pid=7116, groups=00000000}, [12]) = 0
7116  sendto(3, "\24\0\0\0\26\0\1\3\6lrL\0\0\0\0\0\0\0\0", 20, 0, {sa_family=AF_NETLINK, pid=0, groups=00000000}, 12) = 20
7116  recvmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"D\0\0\0\24\0\2\0\6lrL\314\33\0\0\2\10\200\376\1\0\0\0\10"..., 4096}], msg_controllen=0, msg_flags=0}, 0) = 136
7116  recvmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"@\0\0\0\24\0\2\0\6lrL\314\33\0\0\n\200\200\376\1\0\0\0"..., 4096}], msg_controllen=0, msg_flags=0}, 0) = 128
7116  recvmsg(3, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"\24\0\0\0\3\0\2\0\6lrL\314\33\0\0\0\0\0\0\1\0\0\0\24\0"..., 4096}], msg_controllen=0, msg_flags=0}, 0) = 20
7116  close(3)                          = 0
7116  socket(PF_INET, SOCK_STREAM, IPPROTO_IP) = 3
7116  stat("/srv/www/htdocs", {st_mode=S_IFDIR|0755, st_size=4096, ...}) = 0
7116  brk(0x5555557e6000)               = 0x5555557e6000
7116  uname({sys="Linux", node="slesmail", ...}) = 0
7116  socket(PF_NETLINK, SOCK_RAW, 0)   = 4
7116  bind(4, {sa_family=AF_NETLINK, pid=0, groups=00000000}, 12) = 0
7116  getsockname(4, {sa_family=AF_NETLINK, pid=7116, groups=00000000}, [2211908157452]) = 0
7116  sendto(4, "\24\0\0\0\26\0\1\3\6lrL\0\0\0\0\0\0\0\0", 20, 0, {sa_family=AF_NETLINK, pid=0, groups=00000000}, 12) = 20
7116  recvmsg(4, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"D\0\0\0\24\0\2\0\6lrL\314\33\0\0\2\10\200\376\1\0\0\0\10"..., 4096}], msg_controllen=0, msg_flags=0}, 0) = 136
7116  recvmsg(4, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"@\0\0\0\24\0\2\0\6lrL\314\33\0\0\n\200\200\376\1\0\0\0"..., 4096}], msg_controllen=0, msg_flags=0}, 0) = 128
7116  recvmsg(4, {msg_name(12)={sa_family=AF_NETLINK, pid=0, groups=00000000}, msg_iov(1)=[{"\24\0\0\0\3\0\2\0\6lrL\314\33\0\0\0\0\0\0\1\0\0\0\24\0"..., 4096}], msg_controllen=0, msg_flags=0}, 0) = 20
7116  close(4)                          = 0
7116  socket(PF_FILE, SOCK_STREAM, 0)   = 4
7116  fcntl(4, F_GETFL)                 = 0x2 (flags O_RDWR)
7116  fcntl(4, F_SETFL, O_RDWR|O_NONBLOCK) = 0
7116  connect(4, {sa_family=AF_FILE, path="/var/run/nscd/socket"}, 110) = 0
7116  poll([{fd=4, events=POLLOUT|POLLERR|POLLHUP, revents=POLLOUT}], 1, 5000) = 1
7116  sendto(4, "\2\0\0\0\r\0\0\0\6\0\0\0hosts\0Cy", 20, MSG_NOSIGNAL, NULL, 0) = 20
7116  poll([{fd=4, events=POLLIN|POLLERR|POLLHUP, revents=POLLIN|POLLERR|POLLHUP}], 1, 5000) = 1
7116  recvmsg(4, {msg_name(0)=NULL, msg_iov(1)=[{"hosts\0", 6}], msg_controllen=24, {cmsg_len=20, cmsg_level=SOL_SOCKET, cmsg_type=SCM_RIGHTS, {5}}, msg_flags=0}, 0) = 6
7116  fstat(5, {st_mode=S_IFREG|0600, st_size=217016, ...}) = 0
7116  pread(5, "\1\0\0\0h\0\0\0\216\0\0\0\1\0\0\0\215$qL\0\0\0\0\323\0"..., 104, 0) = 104
7116  mmap(NULL, 217016, PROT_READ, MAP_SHARED, 5, 0) = 0x2b4834835000
7116  close(5)                          = 0
7116  close(4)                          = 0
7116  brk(0x555555808000)               = 0x555555808000
7116  brk(0x55555582a000)               = 0x55555582a000
7116  open("/var/run/httpd2.pid", O_RDONLY) = -1 ENOENT (No such file or directory)
7116  setsockopt(3, SOL_SOCKET, SO_REUSEADDR, [1], 4) = 0
7116  setsockopt(3, SOL_SOCKET, SO_KEEPALIVE, [1], 4) = 0
7116  setsockopt(3, SOL_TCP, TCP_NODELAY, [1], 4) = 0
7116  bind(3, {sa_family=AF_INET, sin_port=htons(80), sin_addr=inet_addr("10.40.0.93")}, 16) = 0
7116  listen(3, 511)                    = 0
7116  setsockopt(3, SOL_TCP, TCP_DEFER_ACCEPT, [1], 4) = 0
7116  pipe([4, 5])                      = 0
7116  fcntl(4, F_GETFL)                 = 0 (flags O_RDONLY)
7116  fcntl(4, F_SETFL, O_RDONLY|O_NONBLOCK) = 0
7116  open("/var/log/apache2/error_log", O_WRONLY|O_APPEND|O_CREAT, 0666) = 6
7116  dup2(6, 2)                        = 2
7116  open("/var/log/apache2/access_log", O_WRONLY|O_APPEND|O_CREAT, 0666) = 7
7116  open("/etc/apache2/mime.types", O_RDONLY) = 8
7116  fstat(8, {st_mode=S_IFREG|0644, st_size=12890, ...}) = 0
7116  read(8, "# This file controls what Intern"..., 4096) = 4096
7116  read(8, "m dc\napplication/x-deb deb\nappli"..., 4096) = 4096
7116  read(8, "n/x-stuffit bin sit\napplication/"..., 4096) = 4096
7116  brk(0x55555584c000)               = 0x55555584c000
7116  read(8, "m ts\ntext/x-troff-me me\ntext/x-t"..., 4096) = 602
7116  read(8, "", 4096)                 = 0
7116  close(8)                          = 0
7116  open("/dev/urandom", O_RDONLY|O_NONBLOCK|O_NOCTTY) = 8
7116  fstat(8, {st_mode=S_IFCHR|0644, st_rdev=makedev(1, 9), ...}) = 0
7116  select(9, [8], NULL, NULL, {0, 10000}) = 1 (in [8], left {0, 10000})
7116  read(8, "\262\357l\254\277\336\33L\265N\275q#,\270\305\26\372\345"..., 32) = 32
7116  close(8)                          = 0
7116  getuid()                          = 0
7116  open("/etc/localtime", O_RDONLY)  = 8
7116  fstat(8, {st_mode=S_IFREG|0644, st_size=1267, ...}) = 0
7116  fstat(8, {st_mode=S_IFREG|0644, st_size=1267, ...}) = 0
7116  mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x2b483486a000
7116  read(8, "TZif\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\4\0\0\0\4\0"..., 4096) = 1267
7116  close(8)                          = 0
7116  munmap(0x2b483486a000, 4096)      = 0
7116  write(6, "[Mon Aug 23 08:39:34 2010] [erro"..., 123) = 123
7116  exit_group(1)                     = ?
7113  <... wait4 resumed> [{WIFEXITED(s) && WEXITSTATUS(s) == 1}], 0, NULL) = 7116
7113  rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
7113  --- SIGCHLD (Child exited) @ 0 (0) ---
7113  wait4(-1, 0x7fffcf1351c4, WNOHANG, NULL) = -1 ECHILD (No child processes)
7113  rt_sigreturn(0xffffffffffffffff)  = 0
7113  rt_sigaction(SIGINT, {SIG_DFL}, {0x42e931, [], SA_RESTORER, 0x2adddbf6ac30}, 8) = 0
7113  rt_sigprocmask(SIG_BLOCK, NULL, [], 8) = 0
7113  read(255, "\nexit $ERROR\n\n", 3811) = 14
7113  rt_sigprocmask(SIG_BLOCK, NULL, [], 8) = 0
7113  rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
7113  exit_group(1)                     = ?

I've clearly run out of ideas, so any of yours will be greatly appreciated.   This is our production webmail server, but luckily our users have other methods to access their mail.  

 ssl-global.conf.txt
0
Comment
Question by:izgoblin
  • 7
  • 5
12 Comments
 
LVL 6

Expert Comment

by:JRoyse
ID: 33501686
I would set the Apache HTTPD daemon to LogLevel Debug - in the httpd.conf or equivalent config files.  Also check the file.group permissions on the cert key txt files. - maybe Apache can't read them.
0
 

Author Comment

by:izgoblin
ID: 33503721
Thanks.   I changed the permissions so that everyone can read the key and cert files and that didn't make a difference.   But as I mentioned, it never even attempts to open those files as evidenced by the strace output.  

I also set the LogLevel to debug and I get absolutely no additional output or error messages.  
0
 
LVL 6

Expert Comment

by:JRoyse
ID: 33504424
1. Looks like SSLES has 10.3 updates, which bumps Apache up a notch.  It could be a fixed problem??

2. GroupWise got updated.  You using httpd.d directories to include apache config files? are there extra not-used files inside those directories left over from an upgrade?  Can you compare these to a backup?




0
 

Author Comment

by:izgoblin
ID: 33504612
1.   Hee hee, that's about where I'm at now myself,  looking to update Apache since it seems that logging got better in later versions.   Of course as things usually go I've got to get over a hurdle before I can get that done, but I'm working on it.

2.  The Groupwise files are indeed in conf.d, but I renamed them and commented out all extra references that I could find and still had the issue.   No, there are no additional files that I can see, short of a few distribution templates that shouldn't be read.  

Yeah, this is driving me batty.   If the log won't tell me what's up and an strace won't tell me, I'm lost.
0
 

Author Comment

by:izgoblin
ID: 33510254
I can't win.   I just built Apache 2.2.16, modified the config files accordingly and got the exact same error, and the strace output looks just like the above.   Opens the logs, opens mime.types, /etc/localtime and /dev/urandom and then writes to the error log before quitting.  

Points increased because I really need to get past this soon.
0
 
LVL 6

Expert Comment

by:JRoyse
ID: 33510489
You built it from source?  Have you considered updating the OS with the package manager, that will update Apache (not to 2.2.16 though) along with SSL misc parts.
0
Why You Should Analyze Threat Actor TTPs

After years of analyzing threat actor behavior, it’s become clear that at any given time there are specific tactics, techniques, and procedures (TTPs) that are particularly prevalent. By analyzing and understanding these TTPs, you can dramatically enhance your security program.

 

Author Comment

by:izgoblin
ID: 33510579
I had considered that, but it's not an option for me right now since this is a production Groupwise server (I'm paranoid to break that by updating RPMs).   I can't get the bloody thing to virtualize yet (that WAS what I was working on when this broke!) so I don't have an easy method of trial-and-error without actually screwing up the production server.

I'm lucky that the webmail piece isn't used so much, but the rest of the server certainly is.  
0
 
LVL 6

Expert Comment

by:JRoyse
ID: 33511006
You have any support from Novell on Groupwise?  They may have a compatibility matrix with linux?
0
 

Author Comment

by:izgoblin
ID: 33511038
Thanks, that's where I'm going now.    According to YaST2, it tells me SSL is disabled on my server and does not give me the option to enable the module.   I am beginning to think that my last Groupwise update messed around with something here, some SSL libraries or something along those lines.    I wasn't sure if they handled Apache support, but perhaps they'd be able to help since this could be tied into Groupwise.

I will report back when I find something.
0
 

Accepted Solution

by:
izgoblin earned 0 total points
ID: 33512522
Well, what I couldn't do in 2 days the Novell tech support rep did in 10 minutes.

Can't say I know how it got wiped out, but the problem was in /etc/sysconfig/apache2, a file I didn't even realize existed.   In it, APACHE_SERVER_FLAGS was empty.   We set it as follows:

APACHE_SERVER_FLAGS = "SSL"

Apache then started successfully with SSL support.   Thanks for at least giving this the old college try, JRoyse.   I'm just thrilled to be able to move on to the problem I was trying to resolve when I came across this one.  :)  
0
 

Author Comment

by:izgoblin
ID: 33512766
Would like to offer JRoyse 100 points for making the effort to assist if that's acceptable.   Couldn't find a way to do so here.
0
 
LVL 6

Expert Comment

by:JRoyse
ID: 33514694
No worries!
0

Featured Post

How your wiki can always stay up-to-date

Quip doubles as a “living” wiki and a project management tool that evolves with your organization. As you finish projects in Quip, the work remains, easily accessible to all team members, new and old.
- Increase transparency
- Onboard new hires faster
- Access from mobile/offline

Join & Write a Comment

Hi, in this article I'm going to teach you how to run your own site, and how to let people in (without IP). I'll talk about and explain each step... :) By the way, everything in this Tutorial is completely free and legal. This article is for …
In Solr 4.0 it is possible to atomically (or partially) update individual fields in a document. This article will show the operations possible for atomic updating as well as setting up your Solr instance to be able to perform the actions. One major …
Illustrator's Shape Builder tool will let you combine shapes visually and interactively. This video shows the Mac version, but the tool works the same way in Windows. To follow along with this video, you can draw your own shapes or download the file…
When you create an app prototype with Adobe XD, you can insert system screens -- sharing or Control Center, for example -- with just a few clicks. This video shows you how. You can take the full course on Experts Exchange at http://bit.ly/XDcourse.

762 members asked questions and received personalized solutions in the past 7 days.

Join the community of 500,000 technology professionals and ask your questions.

Join & Ask a Question

Need Help in Real-Time?

Connect with top rated Experts

24 Experts available now in Live!

Get 1:1 Help Now