Win32: ModelessDialogHook class

Update: I've officially deprecated this class, in the face of growing popularity of non-x86 platforms and Windows XP SP2's "data execution prevention" technology.  I strongly recommend against using this code in new projects.

However, I'm keeping the code available here, just because it's the coolest thing I've ever written.  ;-)

So what if you've used this code in the past, and SP2's DEP is getting in the way?  Or you need to support IA64?  Best bet is to forego the x86 assembly code thunk, in favor of a simple, straightforward static member function, and accompanying static vector of HWNDs for the various dialogs in your app.  Not glamorous, but hey, it works.

---

Ever try to encapsulate a dialog behind an in-proc COM object?  If it's a modal dialog, easy.  If it's a modeless dialog... not so easy!  Your COM server code probably doesn't own the message pump -- in all likelihood, the client does.  And the rules of Win32 dictate that message pumps call IsDialogMessage whenever modeless dialogs are visible/active, else your keyboard accelerators won't work.  Bummer!

Even if you have access to your client's source code, if it's written in MFC or VB the message pump code will be buried under 6 feet of boilerplate macros, header files, or runtime library code.

But never fear -- with a little ATL-style thunking code, we can hook into the message pump, and call IsDialogMessage ourselves!  (Note: This code is intended for use in ATL projects, but there's no reason it shouldn't play nicely with any Win32 C++ code.)

Mad props to Andrew Nosenko's AtlAux library for the inspiration to use ATL-style thunks to implement a closure around SetWindowsHookEx.

void ShowModelessDialogsWhilePumpingMessagesImproperly()
{
   // Create a few modeless dialogs
   CTestDialog dlg1;
   CTestDialog dlg2;
   CTestDialog dlg3;

   dlg1.Create(GetDesktopWindow());
   dlg2.Create(GetDesktopWindow());
   dlg3.Create(GetDesktopWindow());

   // Hook onto them
   CModelessDialogHook hook1(dlg1);
   CModelessDialogHook hook2(dlg2);
   CModelessDialogHook hook3(dlg3);

   // Pump messages, while waiting for the dlgs to go away...
   // Note that this message pump doesn't call IsDialogMessage(), 
   // to properly dispatch keyboard/accelerator messages intended 
   // for our modeless dialogs!
   MSG msg;
   while (GetMessage(&msg,0,0,0))
      DispatchMessage(&msg);
}

#pragma once

#ifndef _M_IX86
 #error "This code is implemented for x86 only."
#endif

#pragma pack(push, 1)

template <class TDerived>
class CThiscallThunk
{
    BYTE    m_mov;     // opcode: mov ecx, [dword]
    DWORD   m_this;    // dword value
    BYTE    m_jmp;     // opcode: jmp
    DWORD   m_relproc; // relative jmp distance

public:

    typedef void (TDerived::*TMFP)();

    void InitThunk(TMFP method, const TDerived* pThis)
    {
        union { DWORD func; TMFP method; } addr;
        addr.method = (TMFP)method;
        m_mov = 0xB9;
        m_this = (DWORD)pThis;
        m_jmp = 0xE9;
        m_relproc = addr.func - (DWORD)(this+1);
        FlushInstructionCache(GetCurrentProcess(), this, sizeof(*this));
    }

    FARPROC GetThunk() const 
    {
        _ASSERTE(m_mov == 0xB9);
        return (FARPROC)this; 
    }
};

#pragma pack(pop)


class CModelessDialogHook : public CThiscallThunk<CModelessDialogHook>
{
private:

    HHOOK m_hHook;
    HWND m_hWnd;

public:

    CModelessDialogHook() : m_hHook(0), m_hWnd(0)
    { }

    CModelessDialogHook(HWND h)
    { Hook(h); }

    ~CModelessDialogHook()
    { Unhook(); }

    void Hook(HWND h)
    {
        // Initialize the thunk
        InitThunk((TMFP)HookProc,this);

        // Install the hook
        m_hHook = ::SetWindowsHookEx(WH_GETMESSAGE,(HOOKPROC)GetThunk(),0,GetCurrentThreadId());

        // Remember the window handle
        m_hWnd = h;
    }

    void Unhook()
    {
        if (m_hHook)
            ::UnhookWindowsHookEx(m_hHook);

        m_hHook = 0;
        m_hWnd = 0;
    }

    LRESULT HookProc(int iHookCode, WPARAM wParam, LPARAM lParam) // note: not static! sweet!
    {
        MSG* pMsg = reinterpret_cast<MSG*>(lParam);

        // Intercept keystroke messages, and process them
        if ((iHookCode >= 0) &&
            (wParam == PM_REMOVE) &&
            (pMsg->message >= WM_KEYFIRST) &&
            (pMsg->message <= WM_KEYLAST) &&
            (::IsDialogMessage(m_hWnd,pMsg)))
        {
            pMsg->message = WM_NULL;
            pMsg->lParam = 0;
            pMsg->wParam = 0;
            return 0; // note: we are not calling the next hook
        }

        // Pass on everything else
        return ::CallNextHookEx(m_hHook,iHookCode,wParam,lParam);
    }

};