Best way to communicate between a Windows Service application and a UI Application

I do not use Delphi but for .NET I prefer to expose a restful web service which I can call from the UI app

I have used NamedPipes via IPC.  I never actually experienced any problems with the implementation that I used which is based on Ben Day's implementation - https://www.benday.com/2008/06/22/playing-with-nets-named-pipe-streams-with-net-to-net-and-win32-to-net-samples/:

public delegate void CommandReceivedEventHandler(object sender, CommandReceivedEventArgs e);
public delegate void CommandQueuedEventHandler(object sender, CommandQueuedEventArgs e);

public class CommandReceivedEventArgs : EventArgs
{
	public Command Command { get; private set; }

	private CommandReceivedEventArgs()
	{
		Command = default(Command);
	}

	public CommandReceivedEventArgs(Command command)
	{
		Command = command;
	}
}

public class CommandQueuedEventArgs : EventArgs
{
	public Command Command { get; private set; }

	private CommandQueuedEventArgs()
	{
		Command = default(Command);
	}

	public CommandQueuedEventArgs(Command command)
	{
		Command = command;
	}
}

[Serializable]
public sealed class Command
{
	public CommandType CommandType { get; private set; }
	public ICommandShell CommandShell { get; private set; }

	public Command(CommandType commandType, ICommandShell commandShell)
	{
		CommandType = commandType;
		CommandShell = commandShell;
	}
}

[Serializable]
public sealed class CommandShell<T> : ICommandShell where T : struct, IConvertible
{
	public object[] Arguments { get; private set; }
	public T Verb { get; private set; }

	public CommandShell(T verb, params object[] arguments)
	{
		if (!typeof(T).IsEnum)
			throw new ArgumentException("T must be an enumerated type");
		Verb = verb;
		Arguments = arguments;
	}
}

public enum CommandType
{
	[Description("Client Pipe Command")] ClientPipeCommand,
	[Description("Service Pipe Command")] ServicePipeCommand
}

public enum ServicePipeCommands
{
	[Description("Background heartbeat")] BackgroundHeartbeat,
	[Description("Heartbeat")] Heartbeat,
	[Description("Inter-process communications pipe communicating optimally")] PipeCommunicatingOptimally,
	[Description("Inter-process communications pipe started")] PipeStarted,
	[Description("Inter-process communications pipe Stopped")] PipeStopped,
	[Description("Service running")] ServiceStarted,
	[Description("Service stopping")] ServiceStopping
}

public enum ClientPipeCommands
{
	[Description("Background heartbeat")] BackgroundHeartbeat,
	[Description("Client started")] ClientStarted,
	[Description("Get service status")] GetServiceStatus,
	[Description("Heartbeat")] Heartbeat,
	[Description("Inter-process communications pipe is communicating optimally")] PipeCommunicatingOptimallyReceived,
	[Description("Inter-process communications pipe started received")] PipeStartedReceived,
	[Description("Inter-process communications pipe stopped received")] PipeStoppedReceived,
	[Description("Service stopping received")] ServiceStoppingReceived
}

public interface ICommandShell
{
	object[] Arguments { get; }
}

public abstract class PipeStreamWrapperBase<T> where T : PipeStream
{
	protected const int BUFFER_SIZE = 4096;
	protected bool m_stopRequested = false;

	protected T Pipe { get; set; }
	public string PipeName { get; private set; }
	protected BinaryWriter PipeWriter { get; set; }

	[NonSerialized]
	private CommandReceivedEventHandler _commandReceived;
	public event CommandReceivedEventHandler CommandReceived
	{
		add { _commandReceived += value; }
		remove { _commandReceived -= value; }
	}

	protected virtual void OnCommandReceived(object sender, CommandReceivedEventArgs e)
	{
		CommandReceivedEventHandler handler = _commandReceived;
		if (handler != null)
			handler(sender, e);
	}

	public PipeStreamWrapperBase(string pipeName)
	{
		if (pipeName == null)
			throw new ArgumentNullException("pipeName", "Argument cannot be null.");

		PipeName = pipeName;
	}

	protected abstract bool AutoFlushPipeWriter { get; }
	protected abstract T CreateStream();

	protected abstract void ReadFromPipe(object state);

	public bool IsConnected()
	{
		return Pipe.IsConnected;
	}

	public void Start()
	{
		try
		{
			Pipe = CreateStream();
			ThreadPool.QueueUserWorkItem(ReadFromPipe);
		}
		catch (Exception ex)
		{
			Debug.WriteLine(ex.ToString());
		}
	}

	public void Stop()
	{
		try
		{
			m_stopRequested = true;
			Pipe.Close();
			Pipe.Dispose();
		}
		catch (Exception ex)
		{
			Debug.WriteLine(ex.ToString());
		}
	}

	public void Write(Command command)
	{
		if (Pipe.IsConnected == true && Pipe.CanWrite == true)
		{
			try
			{
				if (PipeWriter == null)
					PipeWriter = new BinaryWriter(Pipe);

				WriteToStream(command.ToBinary());
			}
			catch (PipeStreamWrapperException ex)
			{
				throw (new PipeStreamWrapperException(ex.Message, ex));
			}
			catch (Exception ex)
			{
				Debug.WriteLine(ex.ToString());
			}
		}
	}

	protected static byte[] ReadMessage(PipeStream stream)
	{
		var memoryStream = new MemoryStream();

		try
		{

			byte[] buffer = new byte[BUFFER_SIZE];

			do
			{
				memoryStream.Write(buffer, 0, stream.Read(buffer, 0, buffer.Length));

			} while (stream.IsMessageComplete == false);

		}
		catch (Exception ex)
		{
			Debug.WriteLine(ex.ToString());
		}

		return memoryStream.ToArray();
	}

	protected virtual void WriteToStream(byte[] message)
	{
		try
		{
			PipeWriter.Write(message);
			PipeWriter.Flush();
		}
		catch (IOException ex)
		{
			throw (new PipeStreamWrapperException(ex.Message, ex));
		}
		catch (Exception ex)
		{
			Debug.WriteLine(ex.ToString());
		}
	}

	protected void ThrowOnReceivedMessage(byte[] message)
	{
		OnCommandReceived(this, new CommandReceivedEventArgs(message.FromBinary()));
	}
}

public class NamedPipeServer : PipeStreamWrapperBase<NamedPipeServerStream>
{
	public bool AllowRestart { get; set; } = false;

	protected override bool AutoFlushPipeWriter
	{
		get { return true; }
	}


	public NamedPipeServer(string pipeName) : base(pipeName) { }

	protected override NamedPipeServerStream CreateStream()
	{
		var security = new PipeSecurity();
		var accessRule = new PipeAccessRule("Everyone", PipeAccessRights.ReadWrite, System.Security.AccessControl.AccessControlType.Allow);
		security.AddAccessRule(accessRule);
		var stream = new NamedPipeServerStream(PipeName, PipeDirection.InOut, 1, PipeTransmissionMode.Message, PipeOptions.Asynchronous, BUFFER_SIZE, BUFFER_SIZE, security);
		return stream;
	}

	protected override void ReadFromPipe(object state)
	{
		try
		{
			while (Pipe != null && m_stopRequested == false)
			{
				if (Pipe.IsConnected == false) Pipe.WaitForConnection();

				byte[] msg = ReadMessage(Pipe);

				ThrowOnReceivedMessage(msg);
			}
		}
		catch (Exception ex)
		{
			Debug.WriteLine(ex.ToString());
		}
		finally
		{
			Debug.WriteLine($"{nameof(ReadFromPipe)} is exiting.");
		}
	}

	~NamedPipeServer()
	{
		if (Pipe != null) Pipe.Dispose();
	}
}

public class NamedPipeClient : PipeStreamWrapperBase<NamedPipeClientStream>
{
	protected override bool AutoFlushPipeWriter
	{
		get { return true; }
	}

	public NamedPipeClient(string pipeName) : base(pipeName) { }

	protected override NamedPipeClientStream CreateStream()
	{
		var stream = new NamedPipeClientStream(".", PipeName, PipeDirection.InOut, PipeOptions.Asynchronous);
		stream.Connect();
		stream.ReadMode = PipeTransmissionMode.Message;
		return stream;
	}

	protected override void ReadFromPipe(object state)
	{
		try
		{
			while (Pipe != null && m_stopRequested == false)
			{
				if (Pipe.IsConnected == true)
				{
					byte[] msg = ReadMessage(Pipe);

					ThrowOnReceivedMessage(msg);
				}
			}
		}
		catch (Exception ex)
		{
			Debug.WriteLine(ex.ToString());
		}
		finally
		{
			Debug.WriteLine($"{nameof(ReadFromPipe)} is exiting.");
		}
	}
}

public class PipeStreamWrapperException : Exception
{
	public PipeStreamWrapperException() { }
	public PipeStreamWrapperException(string message) : base(message) { }
	public PipeStreamWrapperException(string message, Exception innerException) : base(message, innerException) { }
	protected PipeStreamWrapperException(SerializationInfo info, StreamingContext context) : base(info, context) { }
}

public static class Extensions
{
	public static Command FromBinary(this byte[] data)
	{
		BinaryFormatter format = new BinaryFormatter();
		using (MemoryStream stream = new MemoryStream(data) { Position = 0 })
		{
			return format.Deserialize(stream) as Command;
		}
	}

	public static byte[] ToBinary(this Command command)
	{
		BinaryFormatter format = new BinaryFormatter();
		using (MemoryStream stream = new MemoryStream())
		{
			format.Serialize(stream, command);
			return stream.ToArray();
		}
	}
}

Select allOpen in new window

ServiceExample -

class ServiceExample : ServiceBase
{
	private const string NAME_OF_PIPE = "ExampleServicePipe";
	private readonly Queue<Command> fromClientQueue = new Queue<Command>();
	public NamedPipeServer ServicePipe { get; set; }

	[NonSerialized]
	private CommandQueuedEventHandler commandQueued;
	public event CommandQueuedEventHandler CommandQueued
	{
		add { commandQueued += value; }
		remove { commandQueued -= value; }
	}

	protected virtual void OnCommandQueued(object sender, CommandQueuedEventArgs e)
	{
		CommandQueuedEventHandler handler = commandQueued;
		if (handler != null)
			handler(sender, e);
	}

	protected override void OnStart(string[] args)
	{
		try
		{
			ServicePipe = new NamedPipeServer(NAME_OF_PIPE);
			ServicePipe.CommandReceived += OnCommandReceived;
			CommandQueued += OnQueuedCommand;
			ServicePipe.Start();
			ServicePipe.AllowRestart = true;
		}
		catch (Exception ex)
		{
			Debug.WriteLine(ex.ToString());
		}
	}

	protected override void OnStop()
	{
		try
		{
			ServicePipe.AllowRestart = false;

			WriteMessageToClient(new Command(CommandType.ServicePipeCommand, new CommandShell<ServicePipeCommands>(ServicePipeCommands.ServiceStopping)));

			if (ServicePipe != null)
				ServicePipe.Stop();
		}
		catch (Exception ex)
		{
			Debug.WriteLine(ex.ToString());
		}
	}

	private void OnQueuedCommand(object sender, CommandQueuedEventArgs e)
	{
		try
		{
			if (fromClientQueue.Count > 0)
			{
				var value = fromClientQueue.Dequeue();
				switch (((CommandShell<ClientPipeCommands>)value.CommandShell).Verb)
				{
					case ClientPipeCommands.ClientStarted:
						WriteMessageToClient(new Command(CommandType.ServicePipeCommand, new CommandShell<ServicePipeCommands>(ServicePipeCommands.PipeCommunicatingOptimally)));
						break;
					case ClientPipeCommands.GetServiceStatus:
						WriteMessageToClient(new Command(CommandType.ServicePipeCommand, new CommandShell<ServicePipeCommands>(ServicePipeCommands.ServiceStarted)));
						break;
					case ClientPipeCommands.Heartbeat:
						WriteMessageToClient(new Command(CommandType.ServicePipeCommand, new CommandShell<ServicePipeCommands>(ServicePipeCommands.Heartbeat)));
						break;
					case ClientPipeCommands.PipeStartedReceived:
					case ClientPipeCommands.PipeStoppedReceived:
					case ClientPipeCommands.PipeCommunicatingOptimallyReceived:
					case ClientPipeCommands.ServiceStoppingReceived:
					case ClientPipeCommands.BackgroundHeartbeat:
					default:
						break;
				}
			}
		}
		catch (Exception ex)
		{
			Debug.WriteLine(ex.ToString());
		}
	}

	private void OnCommandReceived(object sender, CommandReceivedEventArgs e)
	{
		try
		{
			fromClientQueue.Enqueue(e.Command);
			OnCommandQueued(this, new CommandQueuedEventArgs(e.Command));
		}
		catch (Exception ex)
		{
			Debug.WriteLine(ex.ToString());
		}
	}

	private void WriteMessageToClient(Command command)
	{
		try
		{
			ServicePipe.Write(command);
		}
		catch (PipeStreamWrapperException ex)
		{
			Debug.WriteLine(ex.ToString());
			ServicePipe.CommandReceived -= OnCommandReceived;
			CommandQueued -= OnQueuedCommand;
			ServicePipe.Stop();
			ServicePipe = new NamedPipeServer(NAME_OF_PIPE);
			ServicePipe.CommandReceived += OnCommandReceived;
			CommandQueued += OnQueuedCommand;
			ServicePipe.Start();
			ServicePipe.AllowRestart = true;
			ServicePipe.Write(command);
		}
		catch (Exception ex)
		{
			Debug.WriteLine(ex.ToString());
		}
	}
}

Select allOpen in new window

ClientExample -

class ClientExample
{
	private const string NAME_OF_PIPE = "ExampleServicePipe";
	private readonly Queue<Command> fromServiceQueue = new Queue<Command>();
	public NamedPipeClient ClientPipe { get; set; }

	[NonSerialized]
	private CommandQueuedEventHandler commandQueued;
	public event CommandQueuedEventHandler CommandQueued
	{
		add { commandQueued += value; }
		remove { commandQueued -= value; }
	}

	protected virtual void OnCommandQueued(object sender, CommandQueuedEventArgs e)
	{
		CommandQueuedEventHandler handler = commandQueued;
		if (handler != null)
			handler(sender, e);
	}

	public ClientExample()
	{
		try
		{
			ClientPipe = new NamedPipeClient(NAME_OF_PIPE);
			ClientPipe.CommandReceived += OnCommandReceived;
			CommandQueued += OnQueuedCommand;
			ClientPipe.Start();
			while (!ClientPipe.IsConnected()) { ;}

			WriteMessageToServer(new Command(CommandType.ClientPipeCommand, new CommandShell<ClientPipeCommands>(ClientPipeCommands.ClientStarted)));
		}
		catch (Exception ex)
		{
			Debug.WriteLine(ex.ToString());
		}
	}

	private void OnQueuedCommand(object sender, CommandQueuedEventArgs e)
	{
		try
		{
			if (fromServiceQueue.Count > 0)
			{
				var value = fromServiceQueue.Dequeue();
				switch (((CommandShell<ServicePipeCommands>)value.CommandShell).Verb)
				{
					case ServicePipeCommands.PipeStarted:
						WriteMessageToServer(new Command(CommandType.ClientPipeCommand, new CommandShell<ClientPipeCommands>(ClientPipeCommands.PipeStartedReceived)));
						break;
					case ServicePipeCommands.PipeStopped:
						WriteMessageToServer(new Command(CommandType.ClientPipeCommand, new CommandShell<ClientPipeCommands>(ClientPipeCommands.PipeStoppedReceived)));
						break;
					case ServicePipeCommands.PipeCommunicatingOptimally:
						WriteMessageToServer(new Command(CommandType.ClientPipeCommand, new CommandShell<ClientPipeCommands>(ClientPipeCommands.PipeCommunicatingOptimallyReceived)));
						break;
					case ServicePipeCommands.ServiceStopping:
						WriteMessageToServer(new Command(CommandType.ClientPipeCommand, new CommandShell<ClientPipeCommands>(ClientPipeCommands.ServiceStoppingReceived)));
						break;
					case ServicePipeCommands.ServiceStarted:
					case ServicePipeCommands.Heartbeat:
					case ServicePipeCommands.BackgroundHeartbeat:
					default:
						break;
				}
			}
		}
		catch (Exception ex)
		{
			Debug.WriteLine(ex.ToString());
		}
	}

	void OnCommandReceived(object sender, CommandReceivedEventArgs e)
	{
		try
		{
			fromServiceQueue.Enqueue(e.Command);
			OnCommandQueued(this, new CommandQueuedEventArgs(e.Command));
		}
		catch (Exception ex)
		{
			Debug.WriteLine(ex.ToString());
		}
	}

	private void WriteMessageToServer(Command command)
	{
		try
		{
			ClientPipe.Write(command);
		}
		catch (Exception ex)
		{
			Debug.WriteLine(ex.ToString());
		}
	}
}

Select allOpen in new window

-saige-

Thanks it_saige for the detailed reply. But my issue with Named Pipes is that just like Sockets the Firewalls will seek permission.
My customers will give the permission during installation but my application gets blocked when they change their Antivirus or when my exe is updated.

The both applications (webservice and UI) are running in the same machine, so use WM_COPYDATA without passing by firewall or sockets

here a tutorial
https://www.thoughtco.com/send-information-between-applications-1058476

In my Question I mentioned

 IPC at times gives problems when the users are different (Local System and Standard user)

Select allOpen in new window

the reference is to WM_COPYDATA.
It does not work across all users

with delphi, use datasnap

https://www.embarcadero.com/unleash-the-power-of-delphi-with-delphi-labs-datasnap

it's even got an example of building a service with a frontend app :)

My requirement is just to show few display items and Progress messages (Service to UI) and passing some User selected details (UI to Service). Using Datasnap for that would be heavy.
I just stumbled upon part of the solution. WM_CopyData  is blocked when lower rights application sends messages to higher level application. We need to use ChangeWindowMessageFilterEx  in the Higher level application.
Is my finding right ?
I can't find any simple example of this implementation. Any help will be appreciated.