// The basic of 16-bit Galois Field arithmetic is based on Galois.c by James S. Plank.
// Modified by Yutaka Sawada to support MMX, SSE2, and SSSE3.

/* Galois.c
 * James S. Plank

Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure Coding Techniques

Revision 1.2A
May 24, 2011

James S. Plank
Department of Electrical Engineering and Computer Science
University of Tennessee
Knoxville, TN 37996
plank@cs.utk.edu

Copyright (c) 2011, James S. Plank
All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:

 - Redistributions of source code must retain the above copyright
   notice, this list of conditions and the following disclaimer.

 - Redistributions in binary form must reproduce the above copyright
   notice, this list of conditions and the following disclaimer in
   the documentation and/or other materials provided with the
   distribution.

 - Neither the name of the University of Tennessee nor the names of its
   contributors may be used to endorse or promote products derived
   from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
POSSIBILITY OF SUCH DAMAGE.

 */

#define _WIN32_WINNT 0x0600	// Windows Vista or later

#include <stdlib.h>
#include <stdio.h>

#include <windows.h>
#include <intrin.h>	// 組み込み関数(intrinsic)を使用する場合インクルード

#include "gf16.h"
#include "gf_jit.h"	// ParPar の JIT コード用

extern unsigned int cpu_flag;	// declared in common2.h

#ifndef _WIN64	// 32-bit 版なら
#pragma warning(disable:4731)		// inhibit VC's "ebp modified" warning
#pragma warning(disable:4799)		// inhibit VC's "missing emms" warning
#endif

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
// CPU によって使う関数を変更する際の仮宣言

//#define NO_SIMD	// SIMD を使わない場合
//#define NO_ALTMAP	// SSSE3 や JIT(SSE2) の並び替えを使わない場合 (CLMULや32バイト単位は有効)

int sse_unit;

void galois_align16_multiply(unsigned char *r1, unsigned char *r2, unsigned int len, int factor);
void galois_align32_multiply(unsigned char *r1, unsigned char *r2, unsigned int len, int factor);
void galois_align32avx_multiply(unsigned char *r1, unsigned char *r2, unsigned int len, int factor);
void galois_align256_multiply(unsigned char *r1, unsigned char *r2, unsigned int len, int factor);

void galois_altmap_none(unsigned char *data, unsigned int bsize);

// AVX2 と SSSE3 の ALTMAP は 32バイト単位で行う
void galois_altmap32_change(unsigned char *data, unsigned int bsize);
void galois_altmap32_return(unsigned char *data, unsigned int bsize);
void checksum16_altmap32(unsigned char *data, unsigned char *hash, int byte_size);
void checksum16_return32(unsigned char *data, unsigned char *hash, int byte_size);

// JIT(SSE2) は 256バイト単位で計算する
void galois_altmap256_change(unsigned char *data, unsigned int bsize);
void galois_altmap256_return(unsigned char *data, unsigned int bsize);
void checksum16_altmap256(unsigned char *data, unsigned char *hash, int byte_size);
void checksum16_return256(unsigned char *data, unsigned char *hash, int byte_size);

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

#define NW   65536
#define NWM1 65535
#define PRIM_POLY 0x1100B

// なぜかテーブルは 2バイト整数を使った方が速い
static unsigned short *galois_log_table = NULL;
static unsigned short *galois_exp_table;

int galois_create_table(void)
{
	unsigned int j, b;

	if (galois_log_table != NULL) return 0;
	galois_log_table = _aligned_malloc(sizeof(unsigned short) * NW * 2, 64);
	if (galois_log_table == NULL) return -1;
	galois_exp_table = galois_log_table + NW;	// 要素数は 65536個

	b = 1;
	for (j = 0; j < NWM1; j++){
		galois_log_table[b] = (unsigned short)j;
		galois_exp_table[j] = (unsigned short)b;
		b = b << 1;
		if (b & NW) b ^= PRIM_POLY;
	}
	galois_exp_table[NWM1] = galois_exp_table[0];	// copy for reduction (? mod NWM1)

	// CPU によって使う関数を変更する
	sse_unit = 16;	// 16, 32, 64, 128 のどれでもいい (32のSSSE3は少し速い、GPUが識別するのに注意)
	galois_align_multiply = galois_align16_multiply;
	galois_altmap_change = galois_altmap_none;
	galois_altmap_return = galois_altmap_none;
	checksum16_altmap = checksum16;
	checksum16_return = checksum16;
#ifndef NO_SIMD
#ifndef NO_ALTMAP
	if (cpu_flag & 16){	// AVX2 対応なら
		//printf("\nUse AVX2 & ALTMAP\n");
		sse_unit = 32;	// 32, 64, 128 のどれでもいい
		galois_align_multiply = galois_align32avx_multiply;
		galois_altmap_change = galois_altmap32_change;
		galois_altmap_return = galois_altmap32_return;
		checksum16_altmap = checksum16_altmap32;
		checksum16_return = checksum16_return32;
	} else if (cpu_flag & 1){	// SSSE3 対応なら
		if ((cpu_flag & 256) == 0){	// SSSE3 & ALTMAP を使う
			//printf("\nUse SSSE3 & ALTMAP\n");
			sse_unit = 32;	// 32, 64, 128 のどれでもいい
			galois_align_multiply = galois_align32_multiply;
			galois_altmap_change = galois_altmap32_change;
			galois_altmap_return = galois_altmap32_return;
			checksum16_altmap = checksum16_altmap32;
			checksum16_return = checksum16_return32;
		}
	} else {	// SSSE3 が利用できない場合
		if ((cpu_flag & 128) && (jit_alloc() == 0)){	// JIT(SSE2) を使う
			//printf("\nUse JIT(SSE2) & ALTMAP\n");
			sse_unit = 256;
			galois_align_multiply = galois_align256_multiply;
			galois_altmap_change = galois_altmap256_change;
			galois_altmap_return = galois_altmap256_return;
			checksum16_altmap = checksum16_altmap256;
			checksum16_return = checksum16_return256;
		}
	}
#endif
#endif

	return 0;
}

unsigned short galois_multiply(int x, int y)
{
	int sum;

	if ((x == 0) || (y == 0)) return 0;

	sum = galois_log_table[x] + galois_log_table[y];	// result is from 2 to NWM1 * 2
	//if (sum >= NWM1) sum -= NWM1;
	sum = (sum >> 16) + (sum & NWM1);	// result is from 0 to NWM1
	return galois_exp_table[sum];
}

// multiply when "y" is a fixed value
unsigned short galois_multiply_fix(int x, int log_y)
{
	int sum;

	if (x == 0) return 0;

	sum = galois_log_table[x] + log_y;	// result is from 2 to NWM1 * 2
	sum = (sum >> 16) + (sum & NWM1);	// result is from 0 to NWM1
	return galois_exp_table[sum];
}

unsigned short galois_divide(int x, int y)
{
	int sum;

	if (y == 0) return NWM1;	// 除算エラー
	if (x == 0) return 0;

	sum = galois_log_table[x] - galois_log_table[y];
	if (sum < 0) sum += NWM1;
	return galois_exp_table[sum];
}

// ガロア体上での乗数計算、x の y 乗
unsigned short galois_power(int x, int y)
{
	unsigned int sum;

	if (x == 0) return 0;	// 0**y = 0
	if (y == 0) return 1;	// x**0 = 1
	if (y == 1) return (unsigned short)x;	// x**1 = x

	sum = (unsigned int)(galois_log_table[x]) * (unsigned int)y;	// result is from 1 to NWM1 * NWM1
	//sum = sum % NWM1;
	sum = (sum >> 16) + (sum & NWM1);	// result is from 1 to NWM1 * 2
	sum = (sum >> 16) + (sum & NWM1);	// result is from 0 to NWM1
	return galois_exp_table[sum];
}

// ガロア体上での逆数、1 / x
unsigned short galois_reciprocal(int x)
{
	if (x == 0) return NWM1;	// 除算エラー
	return galois_exp_table[NWM1 - (int)(galois_log_table[x])];
}

void galois_free_table(void) // テーブルを解放するために追加
{
	if (galois_log_table != NULL){
		_aligned_free(galois_log_table);
		galois_log_table = NULL;
#ifndef _WIN64	// 32-bit 版ならインライン・アセンブラを使う
		if (((cpu_flag & 1) == 0) && ((cpu_flag & 128) == 0))	// SSSE3 を使わない場合、MMX の終了処理
			_mm_empty();
#endif
		// SSSE3 を使わない場合で、JIT(SSE2) を使った場合の終了処理
		if (((cpu_flag & 1) == 0) && ((cpu_flag & 128) != 0))
			jit_free();
	}
}

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
// MMX functions are based on code by Paul Houle (paulhoule.com) March 22, 2008

#ifndef _WIN64	// 32-bit 版ならインライン・アセンブラを使う

// Processes block of data a multiple of 8 bytes long using SIMD (mmx) opcodes.
// The amount of data to process (bsize) must be a non-zero multiple of 8.
// Paul's original code was modified to calculate each 8-bytes and removed last shift by Yutaka Sawada
static void DoBlock8(unsigned char *input, unsigned char *output, unsigned int bsize, unsigned int *pMtab)
{
	__asm {
	push	ebp
	mov		ebx,bsize					;bytes to process (multiple of 8)
	mov		esi,input					;source
	mov		edi,output					;destination
	mov		ebp,pMtab					;combined multiplication table

	mov		eax,[esi]					;load 1st 8 source bytes
	movd	mm4,[esi+4]

	sub		ebx,8						;reduce last 8-bytes from loop
	add		esi,ebx						;point to end of input/output
	add		edi,ebx
	neg		ebx							;convert byte size to count-up

lp8:
	movzx	edx,al
	movzx	ecx,ah
	shr		eax,16
	movd	mm0,[ebp+edx*4]				;order is [_][_][_][0]
	movd	mm1,[ebp+400h+ecx*4]
	movzx	edx,al
	movzx	ecx,ah
	movd	eax,mm4
	movq	mm4,[esi+ebx+8]				;read-ahead next 8 source bytes
	movd	mm2,[ebp+edx*4]				;order is [_][_][_][1]
	movzx	edx,al
	movq	mm5,[edi+ebx]
	movd	mm3,[ebp+400h+ecx*4]
	movzx	ecx,ah
	shr		eax,16
	punpcklwd mm0,[ebp+edx*4]			;order is [_][_][2][0]
	movzx	edx,al
	punpcklwd mm1,[ebp+400h+ecx*4]
	movzx	ecx,ah
	punpcklwd mm2,[ebp+edx*4]			;order is [_][_][3][1]
	pxor	mm1,mm0
	punpcklwd mm3,[ebp+400h+ecx*4]
	pxor	mm3,mm2
	movd	eax,mm4						;prepare src bytes 3-0 for next loop
	punpcklwd mm1,mm3					;order is [3][2][1][0]
	psrlq	mm4,32						;align src bytes 7-4 for next loop
	pxor	mm1,mm5
	movq	[edi+ebx],mm1
	add		ebx,8
	jnz		lp8

	;no need to pre-read in last 8-bytes
	movzx	edx,al
	movzx	ecx,ah
	shr		eax,16
	movd	mm0,[ebp+edx*4]				;order is [_][_][_][0]
	movd	mm1,[ebp+400h+ecx*4]
	movzx	edx,al
	movzx	ecx,ah
	movd	eax,mm4
	movd	mm2,[ebp+edx*4]				;order is [_][_][_][1]
	movzx	edx,al
	movq	mm5,[edi+ebx]
	movd	mm3,[ebp+400h+ecx*4]
	movzx	ecx,ah
	shr		eax,16
	punpcklwd mm0,[ebp+edx*4]			;order is [_][_][2][0]
	movzx	edx,al
	punpcklwd mm1,[ebp+400h+ecx*4]
	movzx	ecx,ah
	punpcklwd mm2,[ebp+edx*4]			;order is [_][_][3][1]
	pxor	mm1,mm0
	punpcklwd mm3,[ebp+400h+ecx*4]
	pxor	mm3,mm2
	punpcklwd mm1,mm3					;order is [3][2][1][0]
	pxor	mm1,mm5
	movq	[edi+ebx],mm1

	pop		ebp
	}
}

#endif

// calculate multiplication tables instantly
static void create_two_table(unsigned int *mtab, int factor){
	int shift_table[8], i, j, sum;

	// factor * 2の乗数を計算する
	shift_table[0] = factor;	// factor * 1
	for (i = 1; i < 8; i++){
//		if (factor & 0x8000){
//			factor <<= 1;
//			factor ^= 0x1100B;
//		} else {
//			factor <<= 1;
//		}
		factor = (factor << 1) ^ (((factor << 16) >> 31) & 0x1100B);
		shift_table[i] = factor;	// factor * (2**i)
	}

	for (j = 0; j < 2; j++){
		for (i = 0; i < 256; i += 32){
/*
			sum = 0;
			if (i & 32)
				sum = shift_table[5];
			if (i & 64)
				sum ^= shift_table[6];
			if (i & 128)
				sum ^= shift_table[7];
*/
			sum  = shift_table[5] & ((i << 26) >> 31);
			sum ^= shift_table[6] & ((i << 25) >> 31);
			sum ^= shift_table[7] & ((i << 24) >> 31);

			mtab[i     ] = sum;
			mtab[i +  1] = sum ^ shift_table[0];
			mtab[i +  2] = sum ^ shift_table[1];
			mtab[i +  3] = sum ^ shift_table[1] ^ shift_table[0];
			mtab[i +  4] = sum ^ shift_table[2];
			mtab[i +  5] = sum ^ shift_table[2] ^ shift_table[0];
			mtab[i +  6] = sum ^ shift_table[2] ^ shift_table[1];
			mtab[i +  7] = sum ^ shift_table[2] ^ shift_table[1] ^ shift_table[0];
			mtab[i +  8] = sum ^ shift_table[3];
			mtab[i +  9] = sum ^ shift_table[3] ^ shift_table[0];
			mtab[i + 10] = sum ^ shift_table[3] ^ shift_table[1];
			mtab[i + 11] = sum ^ shift_table[3] ^ shift_table[1] ^ shift_table[0];
			mtab[i + 12] = sum ^ shift_table[3] ^ shift_table[2];
			mtab[i + 13] = sum ^ shift_table[3] ^ shift_table[2] ^ shift_table[0];
			mtab[i + 14] = sum ^ shift_table[3] ^ shift_table[2] ^ shift_table[1];
			mtab[i + 15] = sum ^ shift_table[3] ^ shift_table[2] ^ shift_table[1] ^ shift_table[0];
			sum ^= shift_table[4];
			mtab[i + 16] = sum;
			mtab[i + 17] = sum ^ shift_table[0];
			mtab[i + 18] = sum ^ shift_table[1];
			mtab[i + 19] = sum ^ shift_table[1] ^ shift_table[0];
			mtab[i + 20] = sum ^ shift_table[2];
			mtab[i + 21] = sum ^ shift_table[2] ^ shift_table[0];
			mtab[i + 22] = sum ^ shift_table[2] ^ shift_table[1];
			mtab[i + 23] = sum ^ shift_table[2] ^ shift_table[1] ^ shift_table[0];
			mtab[i + 24] = sum ^ shift_table[3];
			mtab[i + 25] = sum ^ shift_table[3] ^ shift_table[0];
			mtab[i + 26] = sum ^ shift_table[3] ^ shift_table[1];
			mtab[i + 27] = sum ^ shift_table[3] ^ shift_table[1] ^ shift_table[0];
			mtab[i + 28] = sum ^ shift_table[3] ^ shift_table[2];
			mtab[i + 29] = sum ^ shift_table[3] ^ shift_table[2] ^ shift_table[0];
			mtab[i + 30] = sum ^ shift_table[3] ^ shift_table[2] ^ shift_table[1];
			mtab[i + 31] = sum ^ shift_table[3] ^ shift_table[2] ^ shift_table[1] ^ shift_table[0];
		}

		for (i = 0; i < 8; i++){
			factor = (factor << 1) ^ (((factor << 16) >> 31) & 0x1100B);
			shift_table[i] = factor;	// factor * (2**i)
		}
		mtab += 256;
	}
}

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
// The method of using SSSE3 is based on Plank's papar;
// "Screaming Fast Galois Field Arithmetic Using Intel SIMD Instructions".

/*
static void create_eight_table_c(unsigned char *mtab, int factor)
{
	int shift_table[16], i, sum;

	// factor * 2の乗数を計算する
	shift_table[0] = factor;	// factor * 1
	for (i = 1; i < 16; i++){
//		if (factor & 0x8000){
//			factor <<= 1;
//			factor ^= 0x1100B;
//		} else {
//			factor <<= 1;
//		}
		factor = (factor << 1) ^ (((factor << 16) >> 31) & 0x1100B);
		shift_table[i] = factor;	// factor * (2**i)
	}

	for (i = 0; i < 16; i += 4){	// 4-bit ごとに計算する
		mtab[i * 8     ] = 0;
		mtab[i * 8 + 16] = 0;
		sum = shift_table[i];
		mtab[i * 8 +  1] = (unsigned char)sum;			// lower 8-bit
		mtab[i * 8 + 17] = (unsigned char)(sum >> 8);	// higher 8-bit
		sum = shift_table[i + 1];
		mtab[i * 8 +  2] = (unsigned char)sum;
		mtab[i * 8 + 18] = (unsigned char)(sum >> 8);
		sum = sum ^ shift_table[i];
		mtab[i * 8 +  3] = (unsigned char)sum;
		mtab[i * 8 + 19] = (unsigned char)(sum >> 8);
		sum = shift_table[i + 2];
		mtab[i * 8 +  4] = (unsigned char)sum;
		mtab[i * 8 + 20] = (unsigned char)(sum >> 8);
		sum = sum ^ shift_table[i];
		mtab[i * 8 +  5] = (unsigned char)sum;
		mtab[i * 8 + 21] = (unsigned char)(sum >> 8);
		sum = shift_table[i + 2] ^ shift_table[i + 1];
		mtab[i * 8 +  6] = (unsigned char)sum;
		mtab[i * 8 + 22] = (unsigned char)(sum >> 8);
		sum = sum ^ shift_table[i];
		mtab[i * 8 +  7] = (unsigned char)sum;
		mtab[i * 8 + 23] = (unsigned char)(sum >> 8);
		sum = shift_table[i + 3];
		mtab[i * 8 +  8] = (unsigned char)sum;
		mtab[i * 8 + 24] = (unsigned char)(sum >> 8);
		sum = sum ^ shift_table[i];
		mtab[i * 8 +  9] = (unsigned char)sum;
		mtab[i * 8 + 25] = (unsigned char)(sum >> 8);
		sum = shift_table[i + 3] ^ shift_table[i + 1];
		mtab[i * 8 + 10] = (unsigned char)sum;
		mtab[i * 8 + 26] = (unsigned char)(sum >> 8);
		sum = sum ^ shift_table[i];
		mtab[i * 8 + 11] = (unsigned char)sum;
		mtab[i * 8 + 27] = (unsigned char)(sum >> 8);
		sum = shift_table[i + 3] ^ shift_table[i + 2];
		mtab[i * 8 + 12] = (unsigned char)sum;
		mtab[i * 8 + 28] = (unsigned char)(sum >> 8);
		sum = sum ^ shift_table[i];
		mtab[i * 8 + 13] = (unsigned char)sum;
		mtab[i * 8 + 29] = (unsigned char)(sum >> 8);
		sum = shift_table[i + 3] ^ shift_table[i + 2] ^ shift_table[i + 1];
		mtab[i * 8 + 14] = (unsigned char)sum;
		mtab[i * 8 + 30] = (unsigned char)(sum >> 8);
		sum = sum ^ shift_table[i];
		mtab[i * 8 + 15] = (unsigned char)sum;
		mtab[i * 8 + 31] = (unsigned char)(sum >> 8);
	}
}
*/

#ifndef _WIN64	// 32-bit 版ならインライン・アセンブラを使う

// tables for split four combined multiplication
static void create_eight_table(unsigned char *mtab, int factor)
{
	__asm {	// This implementation requires SSE2
	mov		eax, factor
	mov		edx, mtab
	mov		ecx, -128

	; create mask for 8-bit
	pcmpeqw	xmm7, xmm7		; 0xFFFF *8
	psrlw	xmm7, 8			; 0x00FF *8

lp32:
	; factor * 1, *2, *4, *8
	movd	xmm0, eax		; [_][_][_][_][_][_][_][1]
	pxor	xmm1, xmm1

	movsx	ebx, ax
	sar		ebx, 31
	shl		eax, 1
	and		ebx, 0x1100B
	xor		eax, ebx
	pinsrw	xmm1, eax, 1	; [_][_][_][_][_][_][2][_]
	pxor	xmm2, xmm2

	movsx	ebx, ax
	sar		ebx, 31
	shl		eax, 1
	and		ebx, 0x1100B
	xor		eax, ebx
	pinsrw	xmm2, eax, 4	; [_][_][_][4][_][_][_][_]
	punpcklwd	xmm1, xmm1	; [_][_][_][_][2][2][_][_]

	movsx	ebx, ax
	sar		ebx, 31
	shl		eax, 1
	and		ebx, 0x1100B
	xor		eax, ebx
	movd	xmm3, eax		; [_][_][_][_][_][_][_][8]

	pshuflw	xmm0, xmm0, 17	; [_][_][_][_][1][_][1][_]
	punpcklwd	xmm3, xmm3	; [_][_][_][_][_][_][8][8]
	pxor	xmm0, xmm1		; [_][_][_][_][3][2][1][_]
	pshufhw	xmm2, xmm2, 0	; [4][4][4][4][_][_][_][_]
	punpcklqdq	xmm0, xmm0	; [3][2][1][_][3][2][1][_]
	pshufd	xmm3, xmm3, 0	; [8][8][8][8][8][8][8][8]
	pxor	xmm2, xmm0		; [7][6][5][4][3][2][1][_]
	pxor	xmm3, xmm2		; [15][14][13][12][11][10][9][8]

	movdqa	xmm0, xmm2
	movdqa	xmm1, xmm3
	pand	xmm0, xmm7
	pand	xmm1, xmm7
	packuswb	xmm0, xmm1	; lower 8-bit * 16
	psrlw	xmm2, 8
	psrlw	xmm3, 8
	packuswb	xmm2, xmm3	; higher 8-bit * 16

	movdqa	[edx+128+ecx], xmm0
	movdqa	[edx+144+ecx], xmm2

	; for next loop
	movsx	ebx, ax		; move with sign of word
	sar		ebx, 31
	shl		eax, 1
	and		ebx, 0x1100B
	xor		eax, ebx

	add		ecx, 32
	jnz		lp32
	}
}

// VC2008 は SSSE3 をインライン・アセンブラで使える
// Address (input) does not need be 16-byte aligned
static void gf16_ssse3_block16u(unsigned char *input, unsigned char *output, unsigned int bsize, unsigned char *table)
{
	__asm {
	mov		ecx, input		; source
	mov		edx, output		; destination
	mov		eax, bsize		; bytes to process (multiple of 16)
	mov		ebx, table		; multiplication table

	; create mask for 8 entries
	pcmpeqw	xmm7, xmm7		; 0xFFFF *8
	psrlw	xmm7, 12		; 0x000F *8

	add		ecx, eax		; point to end of input/output
	add		edx, eax
	neg		eax				; convert byte size to count-up

lp16:
	movdqu	xmm0, [ecx+eax]	; read source 16-bytes
	movdqa	xmm2, [edx+eax]

	movdqa	xmm3, [ebx]		; low table
	movdqa	xmm4, [ebx+16]	; high table
	movdqa	xmm1, xmm0		; copy source
	psrlw	xmm0, 4			; prepare next 4-bit
	pand	xmm1, xmm7		; src & 0x000F
	pshufb	xmm3, xmm1		; table look-up
	psllw	xmm1, 8			; shift 8-bit for higher table
	pshufb	xmm4, xmm1
		movdqa	xmm5, [ebx+32]	; low table
		movdqa	xmm6, [ebx+48]	; high table
	pxor	xmm3, xmm4		; combine high and low
	pxor	xmm2, xmm3

		movdqa	xmm1, xmm0		; copy source
		psrlw	xmm0, 4			; prepare next 4-bit
		pand	xmm1, xmm7		; (src >> 4) & 0x000F
		pshufb	xmm5, xmm1		; table look-up
		psllw	xmm1, 8			; shift 8-bit for higher table
		pshufb	xmm6, xmm1
	movdqa	xmm3, [ebx+64]	; low table
	movdqa	xmm4, [ebx+80]	; high table
		pxor	xmm5, xmm6		; combine high and low
		pxor	xmm2, xmm5

	movdqa	xmm1, xmm0		; copy source
	psrlw	xmm0, 4			; prepare next 4-bit
	pand	xmm1, xmm7		; (src >> 8) & 0x000F
	pshufb	xmm3, xmm1		; table look-up
	psllw	xmm1, 8			; shift 8-bit for higher table
	pshufb	xmm4, xmm1
		movdqa	xmm5, [ebx+96]	; low table
		movdqa	xmm6, [ebx+112]	; high table
	pxor	xmm3, xmm4		; combine high and low
	pxor	xmm2, xmm3

		pshufb	xmm5, xmm0		; table look-up
		psllw	xmm0, 8			; shift 8-bit for higher table
		pshufb	xmm6, xmm0
		pxor	xmm5, xmm6		; combine high and low
		pxor	xmm2, xmm5

	movdqa	[edx+eax], xmm2

	add		eax, 16
	jnz		lp16
	}
}

// その場で 32バイトごとに並び替えて、計算後に戻す方法（50% faster than 16-byte version)
// Address (input) does not need be 16-byte aligned
static void gf16_ssse3_block32u(unsigned char *input, unsigned char *output, unsigned int bsize, unsigned char *table)
{
	__asm {
	mov		ecx, input		; source
	mov		edx, output		; destination
	mov		eax, bsize		; bytes to process (multiple of 32)
	mov		ebx, table		; multiplication table

	; create mask for 16 entries
	pcmpeqw	xmm7, xmm7		; 0xFFFF *8
	pcmpeqw	xmm6, xmm6		; 0xFFFF *8
	psrlw	xmm7, 12		; 0x000F *8
	psrlw	xmm6, 8			; 0x00FF *8
	packuswb	xmm7, xmm7	; 0x0F *16

	add		ecx, eax		; point to end of input/output
	add		edx, eax
	neg		eax				; convert byte size to count-up

lp32:
	movdqu	xmm0, [ecx+eax   ]	; read source 32-bytes
	movdqu	xmm2, [ecx+eax+16]
	movdqa	xmm1, xmm0		; copy source
	movdqa	xmm3, xmm2
	pand	xmm0, xmm6		; erase higher byte
	pand	xmm2, xmm6
	psrlw		xmm1, 8		; move higher byte to lower
	psrlw		xmm3, 8
	packuswb	xmm0, xmm2	; select lower byte of each word
	packuswb	xmm1, xmm3	; select higher byte of each word

	movdqa	xmm4, [ebx]		; low table
	movdqa	xmm5, [ebx+16]	; high table
	movdqa	xmm3, xmm0		; copy source
	psrlw	xmm0, 4			; prepare next 4-bit
	pand	xmm3, xmm7		; src & 0x0F
	pand	xmm0, xmm7		; (src >> 4) & 0x0F
	pshufb	xmm4, xmm3		; table look-up
	pshufb	xmm5, xmm3

	movdqa	xmm2, [ebx+32]	; low table
	movdqa	xmm3, [ebx+48]	; high table
	pshufb	xmm2, xmm0		; table look-up
	pshufb	xmm3, xmm0
	pxor	xmm4, xmm2		; combine result
	pxor	xmm5, xmm3

	movdqa	xmm2, [ebx+64]	; low table
	movdqa	xmm3, [ebx+80]	; high table
	movdqa	xmm0, xmm1		; copy source
	psrlw	xmm1, 4			; prepare next 4-bit
	pand	xmm0, xmm7		; src & 0x0F
	pand	xmm1, xmm7		; (src >> 4) & 0x0F
	pshufb	xmm2, xmm0		; table look-up
	pshufb	xmm3, xmm0
	pxor	xmm4, xmm2		; combine result
	pxor	xmm5, xmm3

	movdqa	xmm2, [ebx+96]	; low table
	movdqa	xmm3, [ebx+112]	; high table
	pshufb	xmm2, xmm1		; table look-up
	pshufb	xmm3, xmm1
	pxor	xmm4, xmm2		; combine result
	pxor	xmm5, xmm3

	movdqa	xmm0, [edx+eax]	; read dest 32-bytes
	movdqa	xmm1, [edx+eax+16]
	movdqa	xmm3, xmm4		; copy result
	punpcklbw	xmm3, xmm5	; interleave lower and higher bytes
	punpckhbw	xmm4, xmm5
	pxor	xmm0, xmm3
	pxor	xmm1, xmm4
	movdqa	[edx+eax], xmm0	; write dest 32-bytes
	movdqa	[edx+eax+16], xmm1

	add		eax, 32
	jnz		lp32
	}
}

// 先に 32バイトごとに並び替えてあるデータを扱う方法
static void gf16_ssse3_block32_altmap(unsigned char *input, unsigned char *output, unsigned int bsize, unsigned char *table)
{
	__asm {
	mov		ecx, input		; source
	mov		edx, output		; destination
	mov		eax, bsize		; bytes to process (multiple of 32)
	mov		ebx, table		; multiplication table

	; create mask for 16 entries
	pcmpeqw	xmm7, xmm7		; 0xFFFF *8
	psrlw	xmm7, 12		; 0x000F *8
	packuswb	xmm7, xmm7	; 0x0F *16

	add		ecx, eax		; point to end of input/output
	add		edx, eax
	neg		eax				; convert byte size to count-up

lp32:
	movdqa	xmm0, [ecx+eax]	; read source 32-bytes
	movdqa	xmm1, [ecx+eax+16]

	movdqa	xmm4, [ebx]		; low table
	movdqa	xmm5, [ebx+16]	; high table
	movdqa	xmm3, xmm0		; copy source
	psrlw	xmm0, 4			; prepare next 4-bit
	pand	xmm3, xmm7		; src & 0x0F
	pand	xmm0, xmm7		; (src >> 4) & 0x0F
	pshufb	xmm4, xmm3		; table look-up
	pshufb	xmm5, xmm3

	movdqa	xmm2, [ebx+32]	; low table
	movdqa	xmm3, [ebx+48]	; high table
	pshufb	xmm2, xmm0		; table look-up
	pshufb	xmm3, xmm0
	pxor	xmm2, xmm4		; combine result
	pxor	xmm3, xmm5

	movdqa	xmm4, [ebx+64]	; low table
	movdqa	xmm5, [ebx+80]	; high table
	movdqa	xmm0, xmm1		; copy source
	psrlw	xmm0, 4			; prepare next 4-bit
	pand	xmm1, xmm7		; src & 0x0F
	pand	xmm0, xmm7		; (src >> 4) & 0x0F
	pshufb	xmm4, xmm1		; table look-up
	pshufb	xmm5, xmm1
	pxor	xmm4, xmm2		; combine result
	pxor	xmm5, xmm3

	movdqa	xmm2, [ebx+96]	; low table
	movdqa	xmm3, [ebx+112]	; high table
	pshufb	xmm2, xmm0		; table look-up
	pshufb	xmm3, xmm0

	movdqa	xmm0, [edx+eax]	; read dest 32-bytes
	movdqa	xmm1, [edx+eax+16]
	pxor	xmm4, xmm2	; combine result
	pxor	xmm5, xmm3
	pxor	xmm4, xmm0
	pxor	xmm5, xmm1
	movdqa	[edx+eax], xmm4	; write dest 32-bytes
	movdqa	[edx+eax+16], xmm5

	add		eax, 32
	jnz		lp32
	}
}

#else	// 64-bit 版ではインライン・アセンブラを使えない
// (__m128i *) で逐次ポインターをキャスト変換するよりも、
// 先に __m128i* で定義しておいた方が、連続した領域へのアクセス最適化がうまくいく？

// tables for split four combined multiplication
static void create_eight_table(unsigned char *mtab, int factor){
	int count = 4;
	__m128i *tbl;
	__m128i xmm0, xmm1, xmm2, xmm3, xmm7;

	tbl = (__m128i *)mtab;

	// create mask for 8-bit
	xmm7 = _mm_setzero_si128();
	xmm7 = _mm_cmpeq_epi16(xmm7, xmm7);	// 0xFFFF *8
	xmm7 = _mm_srli_epi16(xmm7, 8);		// 0x00FF *8

	while (1){
		xmm0 = _mm_cvtsi32_si128(factor);			// [_][_][_][_][_][_][_][1]
		xmm1 = _mm_setzero_si128();

		factor = (factor << 1) ^ (((factor << 16) >> 31) & 0x1100B);
		xmm1 = _mm_insert_epi16(xmm1, factor, 1);	// [_][_][_][_][_][_][2][_]
		xmm2 = _mm_setzero_si128();

		factor = (factor << 1) ^ (((factor << 16) >> 31) & 0x1100B);
		xmm2 = _mm_insert_epi16(xmm2, factor, 4);	// [_][_][_][4][_][_][_][_]
		xmm1 = _mm_unpacklo_epi16(xmm1, xmm1);		// [_][_][_][_][2][2][_][_]

		factor = (factor << 1) ^ (((factor << 16) >> 31) & 0x1100B);
		xmm3 = _mm_cvtsi32_si128(factor);			// [_][_][_][_][_][_][_][8]

		xmm0 = _mm_shufflelo_epi16(xmm0, _MM_SHUFFLE(0, 1, 0, 1));	// [_][_][_][_][1][_][1][_]
		xmm3 = _mm_unpacklo_epi16(xmm3, xmm3);						// [_][_][_][_][_][_][8][8]
		xmm0 = _mm_xor_si128(xmm0, xmm1);							// [_][_][_][_][3][2][1][_]
		xmm2 = _mm_shufflehi_epi16(xmm2, _MM_SHUFFLE(0, 0, 0, 0));	// [4][4][4][4][_][_][_][_]
		xmm0 = _mm_unpacklo_epi64(xmm0, xmm0);						// [3][2][1][_][3][2][1][_]
		xmm3 = _mm_shuffle_epi32(xmm3, _MM_SHUFFLE(0, 0, 0, 0));	// [8][8][8][8][8][8][8][8]
		xmm2 = _mm_xor_si128(xmm2, xmm0);							// [7][6][5][4][3][2][1][_]
		xmm3 = _mm_xor_si128(xmm3, xmm2);							// [15][14][13][12][11][10][9][8]

		xmm0 = _mm_load_si128(&xmm2);
		xmm1 = _mm_load_si128(&xmm3);
		xmm0 = _mm_and_si128(xmm0, xmm7);
		xmm1 = _mm_and_si128(xmm1, xmm7);
		xmm0 = _mm_packus_epi16(xmm0, xmm1);	// lower 8-bit * 16
		xmm2 = _mm_srli_epi16(xmm2, 8);
		xmm3 = _mm_srli_epi16(xmm3, 8);
		xmm2 = _mm_packus_epi16(xmm2, xmm3);	// higher 8-bit * 16

		_mm_store_si128(tbl    , xmm0);
		_mm_store_si128(tbl + 1, xmm2);

		count--;
		if (count == 0)
			break;

		factor = (factor << 1) ^ (((factor << 16) >> 31) & 0x1100B);
		tbl += 2;
	}
}

// 16バイトごとに計算する方法、_mm_shuffle_epi8 の利用効率が悪い。
// Address (input) does not need be 16-byte aligned
static void gf16_ssse3_block16u(unsigned char *input, unsigned char *output, unsigned int bsize, unsigned char *table)
{
	__m128i *src, *dst, *tbl;
	__m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7;

	src = (__m128i *)input;
	dst = (__m128i *)output;
	tbl = (__m128i *)table;

	// create mask for 8 entries
	xmm7 = _mm_setzero_si128();
	xmm7 = _mm_cmpeq_epi16(xmm7, xmm7);	// 0xFFFF *8
	xmm7 = _mm_srli_epi16(xmm7, 12);	// 0x000F *8

	while (bsize != 0){
		xmm0 = _mm_loadu_si128(src);	// read source 16-bytes
		xmm2 = _mm_load_si128(dst);

		xmm3 = _mm_load_si128(tbl);		// low table
		xmm4 = _mm_load_si128(tbl + 1);	// high table
		xmm1 = _mm_load_si128(&xmm0);			// copy source
		xmm0 = _mm_srli_epi16(xmm0, 4);			// prepare next 4-bit
		xmm1 = _mm_and_si128(xmm1, xmm7);		// src & 0x000F
		xmm3 = _mm_shuffle_epi8(xmm3, xmm1);	// table look-up
		xmm1 = _mm_slli_epi16(xmm1, 8);			// shift 8-bit for higher table
		xmm4 = _mm_shuffle_epi8(xmm4, xmm1);
			xmm5 = _mm_load_si128(tbl + 2);	// low table
			xmm6 = _mm_load_si128(tbl + 3);	// high table
		xmm3 = _mm_xor_si128(xmm3, xmm4);	// combine high and low
		xmm2 = _mm_xor_si128(xmm2, xmm3);

			xmm1 = _mm_load_si128(&xmm0);			// copy source
			xmm0 = _mm_srli_epi16(xmm0, 4);			// prepare next 4-bit
			xmm1 = _mm_and_si128(xmm1, xmm7);		// src & 0x000F
			xmm5 = _mm_shuffle_epi8(xmm5, xmm1);	// table look-up
			xmm1 = _mm_slli_epi16(xmm1, 8);			// shift 8-bit for higher table
			xmm6 = _mm_shuffle_epi8(xmm6, xmm1);
		xmm3 = _mm_load_si128(tbl + 4);	// low table
		xmm4 = _mm_load_si128(tbl + 5);	// high table
			xmm5 = _mm_xor_si128(xmm5, xmm6);	// combine high and low
			xmm2 = _mm_xor_si128(xmm2, xmm5);

		xmm1 = _mm_load_si128(&xmm0);			// copy source
		xmm0 = _mm_srli_epi16(xmm0, 4);			// prepare next 4-bit
		xmm1 = _mm_and_si128(xmm1, xmm7);		// src & 0x000F
		xmm3 = _mm_shuffle_epi8(xmm3, xmm1);	// table look-up
		xmm1 = _mm_slli_epi16(xmm1, 8);			// shift 8-bit for higher table
		xmm4 = _mm_shuffle_epi8(xmm4, xmm1);
			xmm5 = _mm_load_si128(tbl + 6);	// low table
			xmm6 = _mm_load_si128(tbl + 7);	// high table
		xmm3 = _mm_xor_si128(xmm3, xmm4);	// combine high and low
		xmm2 = _mm_xor_si128(xmm2, xmm3);

			xmm5 = _mm_shuffle_epi8(xmm5, xmm0);	// table look-up
			xmm0 = _mm_slli_epi16(xmm0, 8);			// shift 8-bit for higher table
			xmm6 = _mm_shuffle_epi8(xmm6, xmm0);
			xmm5 = _mm_xor_si128(xmm5, xmm6);	// combine high and low
			xmm2 = _mm_xor_si128(xmm2, xmm5);

		_mm_store_si128(dst, xmm2);

		src += 1;
		dst += 1;
		bsize -= 16;
	}
}

// その場で 32バイトごとに並び替えて、計算後に戻す方法（50% faster than 16-byte version)
// なぜか asm を使わない方が速い!? 32-bit と 64-bit の両方で使える
// xmm レジスタを 8個までしか使わない方が 32-bit 版で速いし安定する
// Address (input) does not need be 16-byte aligned
static void gf16_ssse3_block32u(unsigned char *input, unsigned char *output, unsigned int bsize, unsigned char *table)
{
	__m128i *src, *dst;
	__m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7;
	__m128i tbl0, tbl1, tbl2, tbl3, tbl4, tbl5, tbl6, tbl7;

	src = (__m128i *)input;
	dst = (__m128i *)output;

	// copy tables to local
	tbl0 = _mm_load_si128((__m128i *)table);
	tbl1 = _mm_load_si128((__m128i *)table + 1);
	tbl2 = _mm_load_si128((__m128i *)table + 2);
	tbl3 = _mm_load_si128((__m128i *)table + 3);
	tbl4 = _mm_load_si128((__m128i *)table + 4);
	tbl5 = _mm_load_si128((__m128i *)table + 5);
	tbl6 = _mm_load_si128((__m128i *)table + 6);
	tbl7 = _mm_load_si128((__m128i *)table + 7);

	// create mask for 16 entries
	xmm7 = _mm_setzero_si128();
	xmm7 = _mm_cmpeq_epi16(xmm7, xmm7);	// 0xFFFF *8
	xmm6 = _mm_srli_epi16(xmm7, 8);		// 0x00FF *8
	xmm7 = _mm_srli_epi16(xmm7, 12);	// 0x000F *8
	xmm7 = _mm_packus_epi16(xmm7, xmm7);	// 0x0F *16

	while (bsize != 0){
		xmm1 = _mm_loadu_si128(src);	// read source 32-bytes
		xmm3 = _mm_loadu_si128(src + 1);
		xmm0 = _mm_and_si128(xmm1, xmm6);	// erase higher byte
		xmm2 = _mm_and_si128(xmm3, xmm6);
		xmm1 = _mm_srli_epi16(xmm1, 8);		// move higher byte to lower
		xmm3 = _mm_srli_epi16(xmm3, 8);
		xmm0 = _mm_packus_epi16(xmm0, xmm2);	//  select lower byte of each word
		xmm1 = _mm_packus_epi16(xmm1, xmm3);	//  select higher byte of each word

		xmm4 = _mm_load_si128(&tbl0);	// load tables
		xmm5 = _mm_load_si128(&tbl1);
		xmm3 = _mm_and_si128(xmm0, xmm7);	// src & 0x0F
		xmm0 = _mm_and_si128(_mm_srli_epi16(xmm0, 4), xmm7);	// (src >> 4) & 0x0F
		xmm4 = _mm_shuffle_epi8(xmm4, xmm3);	// table look-up
		xmm5 = _mm_shuffle_epi8(xmm5, xmm3);

		xmm2 = _mm_load_si128(&tbl2);	// load tables
		xmm3 = _mm_load_si128(&tbl3);
		xmm2 = _mm_shuffle_epi8(xmm2, xmm0);	// table look-up
		xmm3 = _mm_shuffle_epi8(xmm3, xmm0);
		xmm4 = _mm_xor_si128(xmm4, xmm2);	// combine result
		xmm5 = _mm_xor_si128(xmm5, xmm3);

		xmm2 = _mm_load_si128(&tbl4);	// load tables
		xmm3 = _mm_load_si128(&tbl5);
		xmm0 = _mm_and_si128(xmm1, xmm7);	// src & 0x0F
		xmm1 = _mm_and_si128(_mm_srli_epi16(xmm1, 4), xmm7);	// (src >> 4) & 0x0F
		xmm2 = _mm_shuffle_epi8(xmm2, xmm0);	// table look-up
		xmm3 = _mm_shuffle_epi8(xmm3, xmm0);
		xmm4 = _mm_xor_si128(xmm4, xmm2);	// combine result
		xmm5 = _mm_xor_si128(xmm5, xmm3);

		xmm2 = _mm_load_si128(&tbl6);	// load tables
		xmm3 = _mm_load_si128(&tbl7);
		xmm2 = _mm_shuffle_epi8(xmm2, xmm1);	// table look-up
		xmm3 = _mm_shuffle_epi8(xmm3, xmm1);
		xmm4 = _mm_xor_si128(xmm4, xmm2);	// combine result
		xmm5 = _mm_xor_si128(xmm5, xmm3);

		xmm0 = _mm_load_si128(dst);	// read dest 32-bytes
		xmm1 = _mm_load_si128(dst + 1);
		xmm3 = _mm_unpacklo_epi8(xmm4, xmm5);	// interleave lower and higher bytes
		xmm4 = _mm_unpackhi_epi8(xmm4, xmm5);
		xmm0 = _mm_xor_si128(xmm0, xmm3);
		xmm1 = _mm_xor_si128(xmm1, xmm4);
		_mm_store_si128(dst, xmm0);	// write dest 32-bytes
		_mm_store_si128(dst + 1, xmm1);

		src += 2;
		dst += 2;
		bsize -= 32;
	}
}

// xmm レジスタにテーブルを読み込む方が 64-bit 版で微妙に速い
static void gf16_ssse3_block32_altmap(unsigned char *input, unsigned char *output, unsigned int bsize, unsigned char *table)
{
	__m128i *src, *dst;
	__m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm7;
	__m128i tbl0, tbl1, tbl2, tbl3, tbl4, tbl5, tbl6, tbl7;

	src = (__m128i *)input;
	dst = (__m128i *)output;

	// copy tables to local
	tbl0 = _mm_load_si128((__m128i *)table);
	tbl1 = _mm_load_si128((__m128i *)table + 1);
	tbl2 = _mm_load_si128((__m128i *)table + 2);
	tbl3 = _mm_load_si128((__m128i *)table + 3);
	tbl4 = _mm_load_si128((__m128i *)table + 4);
	tbl5 = _mm_load_si128((__m128i *)table + 5);
	tbl6 = _mm_load_si128((__m128i *)table + 6);
	tbl7 = _mm_load_si128((__m128i *)table + 7);

	// create mask for 16 entries
	xmm7 = _mm_setzero_si128();
	xmm7 = _mm_cmpeq_epi16(xmm7, xmm7);	// 0xFFFF *8
	xmm7 = _mm_srli_epi16(xmm7, 12);	// 0x000F *8
	xmm7 = _mm_packus_epi16(xmm7, xmm7);	// 0x0F *16

	while (bsize != 0){
		xmm0 = _mm_load_si128(src);	// read source 32-bytes
		xmm1 = _mm_load_si128(src + 1);

		xmm3 = _mm_load_si128(&xmm0);	// copy source
		xmm0 = _mm_srli_epi16(xmm0, 4);	// prepare next 4-bit
		xmm3 = _mm_and_si128(xmm3, xmm7);	// src & 0x0F
		xmm0 = _mm_and_si128(xmm0, xmm7);	// (src >> 4) & 0x0F

		xmm4 = _mm_load_si128(&tbl0);	// load tables
		xmm5 = _mm_load_si128(&tbl1);
		xmm4 = _mm_shuffle_epi8(xmm4, xmm3);	// table look-up
		xmm5 = _mm_shuffle_epi8(xmm5, xmm3);

		xmm2 = _mm_load_si128(&tbl2);	// load tables
		xmm3 = _mm_load_si128(&tbl3);
		xmm2 = _mm_shuffle_epi8(xmm2, xmm0);	// table look-up
		xmm3 = _mm_shuffle_epi8(xmm3, xmm0);
		xmm4 = _mm_xor_si128(xmm4, xmm2);	// combine result
		xmm5 = _mm_xor_si128(xmm5, xmm3);

		xmm0 = _mm_load_si128(&xmm1);	// copy source
		xmm0 = _mm_srli_epi16(xmm0, 4);	// prepare next 4-bit
		xmm1 = _mm_and_si128(xmm1, xmm7);	// src & 0x0F
		xmm0 = _mm_and_si128(xmm0, xmm7);	// (src >> 4) & 0x0F

		xmm2 = _mm_load_si128(&tbl4);	// load tables
		xmm3 = _mm_load_si128(&tbl5);
		xmm2 = _mm_shuffle_epi8(xmm2, xmm1);	// table look-up
		xmm3 = _mm_shuffle_epi8(xmm3, xmm1);
		xmm4 = _mm_xor_si128(xmm4, xmm2);	// combine result
		xmm5 = _mm_xor_si128(xmm5, xmm3);

		xmm2 = _mm_load_si128(&tbl6);	// load tables
		xmm3 = _mm_load_si128(&tbl7);
		xmm2 = _mm_shuffle_epi8(xmm2, xmm0);	// table look-up
		xmm3 = _mm_shuffle_epi8(xmm3, xmm0);

		xmm0 = _mm_load_si128(dst);	// read dest 32-bytes
		xmm1 = _mm_load_si128(dst + 1);
		xmm4 = _mm_xor_si128(xmm4, xmm2);	// combine result
		xmm5 = _mm_xor_si128(xmm5, xmm3);
		xmm4 = _mm_xor_si128(xmm4, xmm0);
		xmm5 = _mm_xor_si128(xmm5, xmm1);
		_mm_store_si128(dst, xmm4);	// write dest 32-bytes
		_mm_store_si128(dst + 1, xmm5);

		src += 2;
		dst += 2;
		bsize -= 32;
	}
}

/*
static void gf16_ssse3_block32_altmap(unsigned char *input, unsigned char *output, unsigned int bsize, unsigned char *table)
{
	__m128i *src, *dst, *tbl;
	__m128i xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm7;

	src = (__m128i *)input;
	dst = (__m128i *)output;
	tbl = (__m128i *)table;

	// create mask for 16 entries
	xmm7 = _mm_setzero_si128();
	xmm7 = _mm_cmpeq_epi16(xmm7, xmm7);	// 0xFFFF *8
	xmm7 = _mm_srli_epi16(xmm7, 12);	// 0x000F *8
	xmm7 = _mm_packus_epi16(xmm7, xmm7);	// 0x0F *16

	while (bsize != 0){
		xmm0 = _mm_load_si128(src);	// read source 32-bytes
		xmm1 = _mm_load_si128(src + 1);

		xmm3 = _mm_load_si128(&xmm0);	// copy source
		xmm0 = _mm_srli_epi16(xmm0, 4);	// prepare next 4-bit
		xmm3 = _mm_and_si128(xmm3, xmm7);	// src & 0x0F
		xmm0 = _mm_and_si128(xmm0, xmm7);	// (src >> 4) & 0x0F

		xmm4 = _mm_load_si128(tbl);	// load tables
		xmm5 = _mm_load_si128(tbl + 1);
		xmm4 = _mm_shuffle_epi8(xmm4, xmm3);	// table look-up
		xmm5 = _mm_shuffle_epi8(xmm5, xmm3);

		xmm2 = _mm_load_si128(tbl + 2);	// load tables
		xmm3 = _mm_load_si128(tbl + 3);
		xmm2 = _mm_shuffle_epi8(xmm2, xmm0);	// table look-up
		xmm3 = _mm_shuffle_epi8(xmm3, xmm0);
		xmm4 = _mm_xor_si128(xmm4, xmm2);	// combine result
		xmm5 = _mm_xor_si128(xmm5, xmm3);

		xmm0 = _mm_load_si128(&xmm1);	// copy source
		xmm0 = _mm_srli_epi16(xmm0, 4);	// prepare next 4-bit
		xmm1 = _mm_and_si128(xmm1, xmm7);	// src & 0x0F
		xmm0 = _mm_and_si128(xmm0, xmm7);	// (src >> 4) & 0x0F

		xmm2 = _mm_load_si128(tbl + 4);	// load tables
		xmm3 = _mm_load_si128(tbl + 5);
		xmm2 = _mm_shuffle_epi8(xmm2, xmm1);	// table look-up
		xmm3 = _mm_shuffle_epi8(xmm3, xmm1);
		xmm4 = _mm_xor_si128(xmm4, xmm2);	// combine result
		xmm5 = _mm_xor_si128(xmm5, xmm3);

		xmm2 = _mm_load_si128(tbl + 6);	// load tables
		xmm3 = _mm_load_si128(tbl + 7);
		xmm2 = _mm_shuffle_epi8(xmm2, xmm0);	// table look-up
		xmm3 = _mm_shuffle_epi8(xmm3, xmm0);

		xmm0 = _mm_load_si128(dst);	// read dest 32-bytes
		xmm1 = _mm_load_si128(dst + 1);
		xmm4 = _mm_xor_si128(xmm4, xmm2);	// combine result
		xmm5 = _mm_xor_si128(xmm5, xmm3);
		xmm4 = _mm_xor_si128(xmm4, xmm0);
		xmm5 = _mm_xor_si128(xmm5, xmm1);
		_mm_store_si128(dst, xmm4);	// write dest 32-bytes
		_mm_store_si128(dst + 1, xmm5);

		src += 2;
		dst += 2;
		bsize -= 32;
	}
}
*/

#endif

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
// AVX2 命令を使うには Windows 7 以降じゃないといけない

// _mm256_permute2x128_si256 の control の意味は以下を参照
// http://www.felixcloutier.com/x86/VPERM2I128.html

// テーブルを並び替えて使えば、ループ内の並び替え回数を一回に減らせる
static void gf16_avx2_block32(unsigned char *input, unsigned char *output, unsigned int bsize, unsigned char *table)
{
	__m256i tbl0, tbl1, tbl2, tbl3, mask, src0, src1, tmp0, tmp1, tmp2, tmp3;

	// copy tables to local
	tmp0 = _mm256_load_si256((__m256i *)table);		// tbl0[low0][high0] <- 0x0f[lo][lo]
	tmp1 = _mm256_load_si256((__m256i *)table + 1);	// tbl1[low1][high1] <- 0xf0[lo][lo]
	tmp2 = _mm256_load_si256((__m256i *)table + 2);	// tbl2[low2][high2] <- 0x0f[hi][hi]
	tmp3 = _mm256_load_si256((__m256i *)table + 3);	// tbl3[low3][high3] <- 0xf0[hi][hi]

	// re-arrange table order
	tbl0 = _mm256_permute2x128_si256(tmp0, tmp2, 0x30);	// tblA[low0][high2] <- 0x0f[lo][hi]
	tbl1 = _mm256_permute2x128_si256(tmp1, tmp3, 0x30);	// tblB[low1][high3] <- 0xf0[lo][hi]
	tbl2 = _mm256_permute2x128_si256(tmp2, tmp0, 0x03);	// tblC[high0][low2] <- 0x0f[lo][hi]
	tbl3 = _mm256_permute2x128_si256(tmp3, tmp1, 0x03);	// tblD[high1][low3] <- 0xf0[lo][hi]

	// create mask for 32 entries
	mask = _mm256_cmpeq_epi16(tmp0, tmp0);	// 0xFFFF *16
	mask = _mm256_srli_epi16(mask, 12);		// 0x000F *16
	mask = _mm256_packus_epi16(mask, mask);	// 0x0F *32

	while (bsize != 0){
		src0 = _mm256_load_si256((__m256i *)input);	// read source 32-bytes
		src1 = _mm256_srli_epi16(src0, 4);	// prepare next 4-bit
		src0 = _mm256_and_si256(src0, mask);	// src & 0x0F
		src1 = _mm256_and_si256(src1, mask);	// (src >> 4) & 0x0F

		tmp0 = _mm256_shuffle_epi8(tbl0, src0);	// table look-up
		tmp1 = _mm256_shuffle_epi8(tbl1, src1);
		tmp2 = _mm256_shuffle_epi8(tbl2, src0);
		tmp3 = _mm256_shuffle_epi8(tbl3, src1);

		tmp0 = _mm256_xor_si256(tmp0, tmp1);	// combine result
		tmp2 = _mm256_xor_si256(tmp2, tmp3);
		tmp2 = _mm256_permute2x128_si256(tmp2, tmp2, 0x01);	// exchange low & high 128-bit

		src1 = _mm256_load_si256((__m256i *)output);	// read dest 32-bytes
		src1 = _mm256_xor_si256(src1, tmp0);
		src1 = _mm256_xor_si256(src1, tmp2);
		_mm256_store_si256((__m256i *)output, src1);	// write dest 32-bytes

		input += 32;
		output += 32;
		bsize -= 32;
	}

	// AVX-SSE 切り替えの回避
	_mm256_zeroupper();
}

/*
// テーブルを並び替えて使えば、ループ内の並び替え回数を減らせる
static void gf16_avx2_block32(unsigned char *input, unsigned char *output, unsigned int bsize, unsigned char *table)
{
	__m256i tbl0, tbl1, tbl2, tbl3, mask, dest, src0, src1, src2, src3;

	// copy tables to local
	src0 = _mm256_load_si256((__m256i *)table);		// tbl0[low0][high0] <- 0x0f[lo][lo]
	src1 = _mm256_load_si256((__m256i *)table + 1);	// tbl1[low1][high1] <- 0xf0[lo][lo]
	src2 = _mm256_load_si256((__m256i *)table + 2);	// tbl2[low2][high2] <- 0x0f[hi][hi]
	src3 = _mm256_load_si256((__m256i *)table + 3);	// tbl3[low3][high3] <- 0xf0[hi][hi]

	// re-arrange table order
	tbl0 = _mm256_permute2x128_si256(src0, src2, 0x30);	// tblA[low0][high2] <- 0x0f[lo][hi]
	tbl1 = _mm256_permute2x128_si256(src1, src3, 0x30);	// tblB[low1][high3] <- 0xf0[lo][hi]
	tbl2 = _mm256_permute2x128_si256(src2, src0, 0x30);	// tblC[low2][high0] <- 0x0f[hi][lo]
	tbl3 = _mm256_permute2x128_si256(src3, src1, 0x30);	// tblD[low3][high1] <- 0xf0[hi][lo]

	// create mask for 32 entries
	mask = _mm256_set1_epi8(0x0F);	// 0x0F *32

	while (bsize != 0){
		src0 = _mm256_load_si256((__m256i *)input);		// read source 32-bytes
		dest = _mm256_load_si256((__m256i *)output);	// read dest 32-bytes

		src1 = _mm256_srli_epi16(src0, 4);	// prepare next 4-bit
		src0 = _mm256_and_si256(src0, mask);	// src & 0x0F
		src1 = _mm256_and_si256(src1, mask);	// (src >> 4) & 0x0F

		src2 = _mm256_permute2x128_si256(src0, src0, 0x01);	// exchange low & high 128-bit of "src & 0x0F"
		src3 = _mm256_permute2x128_si256(src1, src1, 0x01);	// exchange low & high 128-bit of "(src >> 4) & 0x0F"

		src0 = _mm256_shuffle_epi8(tbl0, src0);	// table look-up
		src1 = _mm256_shuffle_epi8(tbl1, src1);
		src2 = _mm256_shuffle_epi8(tbl2, src2);
		src3 = _mm256_shuffle_epi8(tbl3, src3);

		src0 = _mm256_xor_si256(src0, src1);	// combine result
		src2 = _mm256_xor_si256(src2, src3);
		dest = _mm256_xor_si256(dest, src0);
		dest = _mm256_xor_si256(dest, src2);
		_mm256_store_si256((__m256i *)output, dest);	// write dest 32-bytes

		input += 32;
		output += 32;
		bsize -= 32;
	}

	// AVX-SSE 切り替えの回避
	_mm256_zeroupper();
}
*/

/*
// レジスタを大量に使って依存関係をなくせば並列処理できるかも？
static void gf16_avx2_block32(unsigned char *input, unsigned char *output, unsigned int bsize, unsigned char *table)
{
	__m256i tbl0, tbl1, tbl2, tbl3, mask, dest, src0, src1, tmp0, tmp1, tmp2, tmp3;

	// copy tables to local
	tbl0 = _mm256_load_si256((__m256i *)table);
	tbl1 = _mm256_load_si256((__m256i *)table + 1);
	tbl2 = _mm256_load_si256((__m256i *)table + 2);
	tbl3 = _mm256_load_si256((__m256i *)table + 3);

	// create mask for 32 entries
	mask = _mm256_set1_epi8(0x0F);	// 0x0F *32

	while (bsize != 0){
		src0 = _mm256_load_si256((__m256i *)input);		// read source 32-bytes
		dest = _mm256_load_si256((__m256i *)output);	// read dest 32-bytes

		src1 = _mm256_srli_epi16(src0, 4);	// prepare next 4-bit
		src0 = _mm256_and_si256(src0, mask);	// src & 0x0F
		src1 = _mm256_and_si256(src1, mask);	// (src >> 4) & 0x0F

		tmp0 = _mm256_permute2x128_si256(src0, src0, 0x00);	// copy low 128-bit to high from "src & 0x0F"
		tmp1 = _mm256_permute2x128_si256(src1, src1, 0x00);	// copy low 128-bit to high from "(src >> 4) & 0x0F"
		tmp2 = _mm256_permute2x128_si256(src0, src0, 0x11);	// copy high 128-bit to low from "src & 0x0F"
		tmp3 = _mm256_permute2x128_si256(src1, src1, 0x11);	// copy high 128-bit to low from "(src >> 4) & 0x0F"

		tmp0 = _mm256_shuffle_epi8(tbl0, tmp0);	// table look-up
		tmp1 = _mm256_shuffle_epi8(tbl1, tmp1);
		tmp2 = _mm256_shuffle_epi8(tbl2, tmp2);
		tmp3 = _mm256_shuffle_epi8(tbl3, tmp3);

		tmp0 = _mm256_xor_si256(tmp0, tmp1);	// combine result
		tmp2 = _mm256_xor_si256(tmp2, tmp3);
		dest = _mm256_xor_si256(dest, tmp0);
		dest = _mm256_xor_si256(dest, tmp2);
		_mm256_store_si256((__m256i *)output, dest);	// write dest 32-bytes

		input += 32;
		output += 32;
		bsize -= 32;
	}

	// AVX-SSE 切り替えの回避
	_mm256_zeroupper();
}
*/

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

// データを並び替えることで、メモリーアクセスを高速化する
void galois_altmap32_change(unsigned char *data, unsigned int bsize)
{
	__m128i xmm0, xmm1, xmm2, xmm3, mask;

	mask = _mm_setzero_si128();
	mask = _mm_cmpeq_epi16(mask, mask);	// 0xFFFF *8
	mask = _mm_srli_epi16(mask, 8);		// 0x00FF *8

	while (bsize != 0){
		xmm1 = _mm_load_si128((__m128i *)data);	// read 32-bytes
		xmm3 = _mm_load_si128((__m128i *)data + 1);

		xmm0 = _mm_and_si128(xmm1, mask);	// erase higher byte
		xmm2 = _mm_and_si128(xmm3, mask);
		xmm1 = _mm_srli_epi16(xmm1, 8);		// move higher byte to lower
		xmm3 = _mm_srli_epi16(xmm3, 8);
		xmm0 = _mm_packus_epi16(xmm0, xmm2);	//  select lower byte of each word
		xmm1 = _mm_packus_epi16(xmm1, xmm3);	//  select higher byte of each word

		_mm_store_si128((__m128i *)data, xmm0);	// write 32-bytes
		_mm_store_si128((__m128i *)data + 1, xmm1);

		data += 32;
		bsize -= 32;
	}
}

// データの並びを元に戻す
void galois_altmap32_return(unsigned char *data, unsigned int bsize)
{
	__m128i xmm0, xmm1, xmm2;

	while (bsize != 0){
		xmm1 = _mm_load_si128((__m128i *)data);	// read 32-bytes
		xmm2 = _mm_load_si128((__m128i *)data + 1);

		xmm0 = _mm_unpacklo_epi8(xmm1, xmm2);	// interleave lower and higher bytes
		xmm1 = _mm_unpackhi_epi8(xmm1, xmm2);

		_mm_store_si128((__m128i *)data, xmm0);	// write 32-bytes
		_mm_store_si128((__m128i *)data + 1, xmm1);

		data += 32;
		bsize -= 32;
	}
}

// 並び替えない場合
void galois_altmap_none(unsigned char *data, unsigned int bsize)
{
}

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
from ParPar; "gf_w16_additions.c"
gf_w16_xor_lazy_sse_altmap_multiply_region
gf_w16_xor_lazy_sse_jit_altmap_multiply_region
*/

// 256バイトごとにビット単位の XOR で計算する方法
// input と output の領域は重ならないようにすること
static void gf16_sse2_block256(unsigned char *input, unsigned char *output, unsigned int bsize, int factor)
{
	unsigned int i, bit;
	unsigned int counts[16];
	uintptr_t deptable[16][16];
	__m128i depmask1, depmask2, polymask1, polymask2, addvals1, addvals2;
	unsigned short tmp_depmask[16];

	// calculate dependent bits
	addvals1 = _mm_set_epi16(1<< 7, 1<< 6, 1<< 5, 1<< 4, 1<< 3, 1<< 2, 1<<1, 1<<0);
	addvals2 = _mm_set_epi16(1<<15, 1<<14, 1<<13, 1<<12, 1<<11, 1<<10, 1<<9, 1<<8);

	// duplicate each bit in the polynomial 16 times
	polymask2 = _mm_set1_epi16(PRIM_POLY & 0xFFFF); // chop off top bit, although not really necessary
	polymask1 = _mm_and_si128(polymask2, _mm_set_epi16(1<< 8, 1<< 9, 1<<10, 1<<11, 1<<12, 1<<13, 1<<14, 1<<15));
	polymask2 = _mm_and_si128(polymask2, _mm_set_epi16(1<< 0, 1<< 1, 1<< 2, 1<< 3, 1<< 4, 1<< 5, 1<< 6, 1<< 7));
	polymask1 = _mm_cmpeq_epi16(_mm_setzero_si128(), polymask1);
	polymask2 = _mm_cmpeq_epi16(_mm_setzero_si128(), polymask2);

	if (factor & (1<<15)){
		// XOR
		depmask1 = addvals1;
		depmask2 = addvals2;
	} else {
		depmask1 = _mm_setzero_si128();
		depmask2 = _mm_setzero_si128();
	}
	for (i = (1<<14); i; i >>= 1){
		// rotate
		__m128i last = _mm_shuffle_epi32(_mm_shufflelo_epi16(depmask1, 0), 0);
		depmask1 = _mm_insert_epi16(
			_mm_srli_si128(depmask1, 2),
			_mm_extract_epi16(depmask2, 0),
			7
		);
		depmask2 = _mm_srli_si128(depmask2, 2);

		// XOR poly
		depmask1 = _mm_xor_si128(depmask1, _mm_andnot_si128(polymask1, last));
		depmask2 = _mm_xor_si128(depmask2, _mm_andnot_si128(polymask2, last));

		if (factor & i){
			// XOR
			depmask1 = _mm_xor_si128(depmask1, addvals1);
			depmask2 = _mm_xor_si128(depmask2, addvals2);
		}
	}

	// generate needed tables
	_mm_storeu_si128((__m128i*)(tmp_depmask), depmask1);
	_mm_storeu_si128((__m128i*)(tmp_depmask + 8), depmask2);
	for (bit = 0; bit < 16; bit++){
		unsigned int cnt = 0;
		for (i = 0; i < 16; i++){
			if (tmp_depmask[bit] & (1<<i)){
				deptable[bit][cnt++] = i<<4; // pre-multiply because x86 addressing can't do a x16; this saves a shift operation later
			}
		}
		counts[bit] = cnt;
	}

	while (bsize != 0){
		#define STEP(bit) { \
			uintptr_t* deps = deptable[bit]; \
			__m128i tmp = _mm_load_si128((__m128i *)output + bit); \
			switch (counts[bit]){ \
				case 16: tmp = _mm_xor_si128(tmp, *(__m128i *)(input + deps[15])); \
				case 15: tmp = _mm_xor_si128(tmp, *(__m128i *)(input + deps[14])); \
				case 14: tmp = _mm_xor_si128(tmp, *(__m128i *)(input + deps[13])); \
				case 13: tmp = _mm_xor_si128(tmp, *(__m128i *)(input + deps[12])); \
				case 12: tmp = _mm_xor_si128(tmp, *(__m128i *)(input + deps[11])); \
				case 11: tmp = _mm_xor_si128(tmp, *(__m128i *)(input + deps[10])); \
				case 10: tmp = _mm_xor_si128(tmp, *(__m128i *)(input + deps[ 9])); \
				case  9: tmp = _mm_xor_si128(tmp, *(__m128i *)(input + deps[ 8])); \
				case  8: tmp = _mm_xor_si128(tmp, *(__m128i *)(input + deps[ 7])); \
				case  7: tmp = _mm_xor_si128(tmp, *(__m128i *)(input + deps[ 6])); \
				case  6: tmp = _mm_xor_si128(tmp, *(__m128i *)(input + deps[ 5])); \
				case  5: tmp = _mm_xor_si128(tmp, *(__m128i *)(input + deps[ 4])); \
				case  4: tmp = _mm_xor_si128(tmp, *(__m128i *)(input + deps[ 3])); \
				case  3: tmp = _mm_xor_si128(tmp, *(__m128i *)(input + deps[ 2])); \
				case  2: tmp = _mm_xor_si128(tmp, *(__m128i *)(input + deps[ 1])); \
				case  1: tmp = _mm_xor_si128(tmp, *(__m128i *)(input + deps[ 0])); \
			} \
			_mm_store_si128((__m128i *)output + bit, tmp); \
		}
		STEP( 0)
		STEP( 1)
		STEP( 2)
		STEP( 3)
		STEP( 4)
		STEP( 5)
		STEP( 6)
		STEP( 7)
		STEP( 8)
		STEP( 9)
		STEP(10)
		STEP(11)
		STEP(12)
		STEP(13)
		STEP(14)
		STEP(15)
		#undef STEP
		input += 256;
		output += 256;
		bsize -= 256;
	}
}

// bsize が 0 にならないようにすること
static void gf16_sse2_block256_jit(unsigned char *input, unsigned char *output, unsigned int bsize, int factor)
{
	FAST_U32 i, bit;
	long inBit;
	__m128i depmask1, depmask2, polymask1, polymask2, addvals1, addvals2;
	__m128i common_mask;
	unsigned short tmp_depmask[16], common_depmask[8];
	unsigned char * pos_startloop;
	unsigned char *jit_exec, *jit_ptr;
	int j, thread_id;

	// calculate dependent bits
	addvals1 = _mm_set_epi16(1<< 7, 1<< 6, 1<< 5, 1<< 4, 1<< 3, 1<< 2, 1<<1, 1<<0);
	addvals2 = _mm_set_epi16(1<<15, 1<<14, 1<<13, 1<<12, 1<<11, 1<<10, 1<<9, 1<<8);

	// duplicate each bit in the polynomial 16 times
	polymask2 = _mm_set1_epi16(PRIM_POLY & 0xFFFF); // chop off top bit, although not really necessary
	polymask1 = _mm_and_si128(polymask2, _mm_set_epi16(1<< 8, 1<< 9, 1<<10, 1<<11, 1<<12, 1<<13, 1<<14, 1<<15));
	polymask2 = _mm_and_si128(polymask2, _mm_set_epi16(1<< 0, 1<< 1, 1<< 2, 1<< 3, 1<< 4, 1<< 5, 1<< 6, 1<< 7));
	polymask1 = _mm_cmpeq_epi16(_mm_setzero_si128(), polymask1);
	polymask2 = _mm_cmpeq_epi16(_mm_setzero_si128(), polymask2);

	if (factor & (1<<15)){
		// XOR
		depmask1 = addvals1;
		depmask2 = addvals2;
	} else {
		depmask1 = _mm_setzero_si128();
		depmask2 = _mm_setzero_si128();
	}
	for (i = (1<<14); i; i >>= 1){
		// rotate
		__m128i last = _mm_shuffle_epi32(_mm_shufflelo_epi16(depmask1, 0), 0);
		depmask1 = _mm_insert_epi16(
			_mm_srli_si128(depmask1, 2),
			_mm_extract_epi16(depmask2, 0),
			7
		);
		depmask2 = _mm_srli_si128(depmask2, 2);

		// XOR poly
		depmask1 = _mm_xor_si128(depmask1, _mm_andnot_si128(polymask1, last));
		depmask2 = _mm_xor_si128(depmask2, _mm_andnot_si128(polymask2, last));

		if (factor & i){
			// XOR
			depmask1 = _mm_xor_si128(depmask1, addvals1);
			depmask2 = _mm_xor_si128(depmask2, addvals2);
		}
	}

	// attempt to remove some redundant XOR ops with a simple heuristic
	// heuristic: we just find common XOR elements between bit pairs
	{
		__m128i tmp1, tmp2;
		// first, we need to re-arrange words so that we can perform bitwise AND on neighbouring pairs
		// unfortunately, PACKUSDW is SSE4.1 only, so emulate it with shuffles
		// 01234567 -> 02461357
		tmp1 = _mm_shuffle_epi32(
			_mm_shufflelo_epi16(
				_mm_shufflehi_epi16(depmask1, 0xD8), /* 0xD8 == 0b11011000 */
				0xD8
			),
			0xD8
		);
		tmp2 = _mm_shuffle_epi32(
			_mm_shufflelo_epi16(
				_mm_shufflehi_epi16(depmask2, 0xD8),
				0xD8
			),
			0xD8
		);
		common_mask = _mm_and_si128(
			// [02461357, 8ACE9BDF] -> [02468ACE, 13579BDF]
			_mm_unpacklo_epi64(tmp1, tmp2),
			_mm_unpackhi_epi64(tmp1, tmp2)
		);
		// we have the common elements between pairs, but it doesn't make sense to process a separate queue if there's only one common element (0 XORs), so eliminate those
		common_mask = _mm_andnot_si128(_mm_cmpeq_epi16(
		_mm_setzero_si128(),
		// "(v & (v-1)) == 0" is true if only zero/one bit is set in each word
		_mm_and_si128(common_mask, _mm_sub_epi16(common_mask, _mm_set1_epi16(1)))
		), common_mask);
		// we now have a common elements mask without 1-bit words, just simply merge stuff in
		depmask1 = _mm_xor_si128(depmask1, _mm_unpacklo_epi16(common_mask, common_mask));
		depmask2 = _mm_xor_si128(depmask2, _mm_unpackhi_epi16(common_mask, common_mask));
		_mm_storeu_si128((__m128i*)common_depmask, common_mask);
	}

	_mm_storeu_si128((__m128i*)(tmp_depmask), depmask1);
	_mm_storeu_si128((__m128i*)(tmp_depmask + 8), depmask2);

	// Multi-threading だとスレッドごとに実行領域を分離しないとアクセス違反エラーが発生する
	thread_id = GetCurrentThreadId();	// 自分のスレッド ID を取得する
	for (j = 0; j < MAX_CPU; j++){	// 対応するスレッド個数は MAX_CPU 個まで
		if (jit_id[j] == thread_id)
			break;
	}
	if (j == MAX_CPU){	// 初期状態では jit_code 内は全て 0 なので jit_id も 0 だけ
		for (j = 0; j < MAX_CPU; j++){
			if (InterlockedCompareExchange(jit_id + j, thread_id, 0) == 0)	// 0と置き換えたなら
				break;
		}
	}
	jit_exec = jit_code + 4096 * j;
	jit_ptr = jit_exec;

#ifdef _WIN64
	_jit_push(&jit_ptr, BP);
	_jit_mov_r(&jit_ptr, BP, SP);
	// align pointer (avoid SP because stuff is encoded differently with it)
	_jit_mov_r(&jit_ptr, AX, SP);
	_jit_and_i(&jit_ptr, AX, 0xF);
	_jit_sub_r(&jit_ptr, BP, AX);

	// make Windows happy and save XMM6-15 registers
	// ideally should be done by this function, not JIT code, but MSVC has a convenient policy of no inline ASM
	for (i = 6; i < 16; i++)
		_jit_movaps_store(&jit_ptr, BP, -((int32_t)i-5)*16, (uint8_t)i);
#endif

	// adding 128 to the destination pointer allows the register offset to be coded in 1 byte
	// eg: 'movdqa xmm0, [rdx+0x90]' is 8 bytes, whilst 'movdqa xmm0, [rdx-0x60]' is 5 bytes
	_jit_mov_i(&jit_ptr, AX, (intptr_t)input + 128);
	_jit_mov_i(&jit_ptr, DX, (intptr_t)output + 128);
	_jit_mov_i(&jit_ptr, CX, (intptr_t)output + bsize + 128);

	_jit_align32(&jit_ptr);
	pos_startloop = jit_ptr;

	//_jit_movaps_load(reg, xreg, offs)
	// (we just save a conditional by hardcoding this)
	#define _LD_APS(xreg, mreg, offs) \
		*(int32_t*)(jit_ptr) = 0x40280F + ((xreg) <<19) + ((mreg) <<16) + (((offs)&0xFF) <<24); \
		jit_ptr += 4
	#define _ST_APS(mreg, offs, xreg) \
		*(int32_t*)(jit_ptr) = 0x40290F + ((xreg) <<19) + ((mreg) <<16) + (((offs)&0xFF) <<24); \
		jit_ptr += 4
	#define _LD_APS64(xreg, mreg, offs) \
		*(int64_t*)(jit_ptr) = 0x40280F44 + ((xreg-8) <<27) + ((mreg) <<24) + ((int64_t)((offs)&0xFF) <<32); \
		jit_ptr += 5
	#define _ST_APS64(mreg, offs, xreg) \
		*(int64_t*)(jit_ptr) = 0x40290F44 + ((xreg-8) <<27) + ((mreg) <<24) + ((int64_t)((offs)&0xFF) <<32); \
		jit_ptr += 5

#ifdef _WIN64
	#define _LD_DQA(xreg, mreg, offs) \
		*(int64_t*)(jit_ptr) = 0x406F0F66 + ((xreg) <<27) + ((mreg) <<24) + ((int64_t)((offs)&0xFF) <<32); \
		jit_ptr += 5
	#define _ST_DQA(mreg, offs, xreg) \
		*(int64_t*)(jit_ptr) = 0x407F0F66 + ((xreg) <<27) + ((mreg) <<24) + ((int64_t)((offs)&0xFF) <<32); \
		jit_ptr += 5
#else
	#define _LD_DQA(xreg, mreg, offs) \
		*(int32_t*)(jit_ptr) = 0x406F0F66 + ((xreg) <<27) + ((mreg) <<24); \
		*(jit_ptr +4) = (uint8_t)((offs)&0xFF); \
		jit_ptr += 5
	#define _ST_DQA(mreg, offs, xreg) \
		*(int32_t*)(jit_ptr) = 0x407F0F66 + ((xreg) <<27) + ((mreg) <<24); \
		*(jit_ptr +4) = (uint8_t)((offs)&0xFF); \
		jit_ptr += 5
#endif
	#define _LD_DQA64(xreg, mreg, offs) \
		*(int64_t*)(jit_ptr) = 0x406F0F4466 + ((int64_t)(xreg-8) <<35) + ((int64_t)(mreg) <<32) + ((int64_t)((offs)&0xFF) <<40); \
		jit_ptr += 6
	#define _ST_DQA64(mreg, offs, xreg) \
		*(int64_t*)(jit_ptr) = 0x407F0F4466 + ((int64_t)(xreg-8) <<35) + ((int64_t)(mreg) <<32) + ((int64_t)((offs)&0xFF) <<40); \
		jit_ptr += 6

	//_jit_xorps_m(reg, AX, offs<<4);
	#define _XORPS_M_(reg, offs, tr) \
		*(int32_t*)(jit_ptr) = (0x40570F + ((reg) << 19) + (((offs)&0xFF) <<28)) ^ (tr)
	#define _C_XORPS_M(reg, offs, c) \
		_XORPS_M_(reg, offs, 0); \
		jit_ptr += (c)<<2
	#define _XORPS_M64_(reg, offs, tr) \
		*(int64_t*)(jit_ptr) = (0x40570F44 + (((reg)-8) << 27) + ((int64_t)((offs)&0xFF) <<36)) ^ ((tr)<<8)
	#define _C_XORPS_M64(reg, offs, c) \
		_XORPS_M64_(reg, offs, 0); \
		jit_ptr += ((c)<<2)+(c)

	//_jit_pxor_m(1, AX, offs<<4);
#ifdef _WIN64
	#define _PXOR_M_(reg, offs, tr) \
		*(int64_t*)(jit_ptr) = (0x40EF0F66 + ((reg) << 27) + ((int64_t)((offs)&0xFF) << 36)) ^ (tr)
#else
	#define _PXOR_M_(reg, offs, tr) \
		*(int32_t*)(jit_ptr) = (0x40EF0F66 + ((reg) << 27)) ^ (tr); \
		*(jit_ptr +4) = (uint8_t)(((offs)&0xFF) << 4)
#endif
	#define _PXOR_M(reg, offs) \
		_PXOR_M_(reg, offs, 0); \
		jit_ptr += 5
	#define _C_PXOR_M(reg, offs, c) \
		_PXOR_M_(reg, offs, 0); \
		jit_ptr += ((c)<<2)+(c)
	#define _PXOR_M64_(reg, offs, tr) \
		*(int64_t*)(jit_ptr) = (0x40EF0F4466 + ((int64_t)((reg)-8) << 35) + ((int64_t)((offs)&0xFF) << 44)) ^ ((tr)<<8)
	#define _C_PXOR_M64(reg, offs, c) \
		_PXOR_M64_(reg, offs, 0); \
		jit_ptr += ((c)<<2)+((c)<<1)

	//_jit_xorps_r(r2, r1)
	#define _XORPS_R_(r2, r1, tr) \
		*(int32_t*)(jit_ptr) = (0xC0570F + ((r2) <<19) + ((r1) <<16)) ^ (tr)
	#define _XORPS_R(r2, r1) \
		_XORPS_R_(r2, r1, 0); \
		jit_ptr += 3
	#define _C_XORPS_R(r2, r1, c) \
		_XORPS_R_(r2, r1, 0); \
		jit_ptr += ((c)<<1)+(c)
	// r2 is always < 8, r1 here is >= 8
	#define _XORPS_R64_(r2, r1, tr) \
		*(int32_t*)(jit_ptr) = (0xC0570F41 + ((r2) <<27) + ((r1) <<24)) ^ ((tr)<<8)
	#define _C_XORPS_R64(r2, r1, c) \
		_XORPS_R64_(r2, r1, 0); \
		jit_ptr += (c)<<2

	//_jit_pxor_r(r2, r1)
	#define _PXOR_R_(r2, r1, tr) \
		*(int32_t*)(jit_ptr) = (0xC0EF0F66 + ((r2) <<27) + ((r1) <<24)) ^ (tr)
	#define _PXOR_R(r2, r1) \
		_PXOR_R_(r2, r1, 0); \
		jit_ptr += 4
	#define _C_PXOR_R(r2, r1, c) \
		_PXOR_R_(r2, r1, 0); \
		jit_ptr += (c)<<2
	#define _PXOR_R64_(r2, r1, tr) \
		*(int64_t*)(jit_ptr) = (0xC0EF0F4166 + ((int64_t)(r2) <<35) + ((int64_t)(r1) <<32)) ^ (((int64_t)tr)<<8)
	#define _C_PXOR_R64(r2, r1, c) \
		_PXOR_R64_(r2, r1, 0); \
		jit_ptr += ((c)<<2)+(c)

	// optimised mix of xor/mov operations
	#define _MOV_OR_XOR_FP_M(reg, offs, flag, c) \
		_XORPS_M_(reg, offs, flag); \
		flag &= (c)-1; \
		jit_ptr += (c)<<2
	#define _MOV_OR_XOR_FP_M64(reg, offs, flag, c) \
		_XORPS_M64_(reg, offs, flag); \
		flag &= (c)-1; \
		jit_ptr += ((c)<<2)+(c)
	#define _MOV_OR_XOR_FP_INIT (0x570F ^ 0x280F)

	#define _MOV_OR_XOR_INT_M(reg, offs, flag, c) \
		_PXOR_M_(reg, offs, flag); \
		flag &= (c)-1; \
		jit_ptr += ((c)<<2)+(c)
	#define _MOV_OR_XOR_INT_M64(reg, offs, flag, c) \
		_PXOR_M64_(reg, offs, flag); \
		flag &= (c)-1; \
		jit_ptr += ((c)<<2)+((c)<<1)
	#define _MOV_OR_XOR_INT_INIT (0xEF0F00 ^ 0x6F0F00)

	#define _MOV_OR_XOR_R_FP(r2, r1, flag, c) \
		_XORPS_R_(r2, r1, flag); \
		flag &= (c)-1; \
		jit_ptr += ((c)<<1)+(c)
	#define _MOV_OR_XOR_R64_FP(r2, r1, flag, c) \
		_XORPS_R64_(r2, r1, flag); \
		flag &= (c)-1; \
		jit_ptr += (c)<<2

	#define _MOV_OR_XOR_R_INT(r2, r1, flag, c) \
		_PXOR_R_(r2, r1, flag); \
		flag &= (c)-1; \
		jit_ptr += (c)<<2
	#define _MOV_OR_XOR_R64_INT(r2, r1, flag, c) \
		_PXOR_R64_(r2, r1, flag); \
		flag &= (c)-1; \
		jit_ptr += ((c)<<2)+(c)

	// generate code
#ifdef _WIN64
	// preload upper 13 inputs into registers
	#define _XORS_FROM_MEMORY 3
	for (inBit = 3; inBit < 8; inBit++){
		_LD_APS(inBit, AX, (inBit-8)<<4);
	}
	for (; inBit<16; inBit++){
		_LD_APS64(inBit, AX, (inBit-8)<<4);
	}
#else
	// can only fit 5 in 32-bit mode :(
	#define _XORS_FROM_MEMORY 11
	for (inBit = 3; inBit < 8; inBit++){	// despite appearances, we're actually loading the top 5, not mid 5
		_LD_APS(inBit, AX, inBit<<4);
	}
#endif
	for (bit = 0; bit < 16; bit += 2){
		int destOffs = (int)((bit<<4)-128);
		FAST_U32 movC = _MOV_OR_XOR_INT_INIT;
		FAST_U16 mask1 = tmp_depmask[bit], mask2 = tmp_depmask[bit+1], maskC = common_depmask[bit>>1];
		_LD_APS(0, DX, destOffs);
		_LD_DQA(1, DX, destOffs+16);

		for (inBit = -8; inBit < (_XORS_FROM_MEMORY-8); inBit++){
			_MOV_OR_XOR_INT_M(2, inBit, movC, maskC & 1);
			_C_XORPS_M(0, inBit, mask1 & 1);
			_C_PXOR_M(1, inBit, mask2 & 1);
			mask1 >>= 1;
			mask2 >>= 1;
			maskC >>= 1;
		}
		// at least 5 can come from registers
		for (inBit = 3; inBit < 8; inBit++){
			_MOV_OR_XOR_R_INT(2, inBit, (int32_t)movC, maskC & 1);
			_C_XORPS_R(0, inBit, mask1 & 1);
			_C_PXOR_R(1, inBit, mask2 & 1);
			mask1 >>= 1;
			mask2 >>= 1;
			maskC >>= 1;
		}
#ifdef _WIN64
		// more XORs can come from 64-bit registers
		for (inBit = 0; inBit < 8; inBit++){
			_MOV_OR_XOR_R64_INT(2, inBit, movC, maskC & 1);
			_C_XORPS_R64(0, inBit, mask1 & 1);
			_C_PXOR_R64(1, inBit, mask2 & 1);
			mask1 >>= 1;
			mask2 >>= 1;
			maskC >>= 1;
		}
#endif
		if (!movC){
			_XORPS_R(0, 2);
			_PXOR_R(1, 2);	// penalty?
		}
		_ST_APS(DX, destOffs, 0);
		_ST_DQA(DX, destOffs+16, 1);
	}
	#undef _XORS_FROM_MEMORY

	_jit_add_i(&jit_ptr, AX, 256);
	_jit_add_i(&jit_ptr, DX, 256);

	_jit_cmp_r(&jit_ptr, DX, CX);
	_jit_jcc(&jit_ptr, JL, pos_startloop);

#ifdef _WIN64
	for (i = 6; i < 16; i++)
		_jit_movaps_load(&jit_ptr, (uint8_t)i, BP, -((int32_t)i-5)*16);
	_jit_pop(&jit_ptr, BP);
#endif

	_jit_ret(&jit_ptr);

	// exec
	(*(void(*)(void))jit_exec)();

}

/*
XOR method is based on ParPar.
https://github.com/animetosho/ParPar/
*/

// bit 単位で XOR するには、256バイトごとに並び替える
void galois_altmap256_change(unsigned char *data, unsigned int bsize)
{
	unsigned short dtmp[128];
	int i, j;
	__m128i ta, tb, lmask, th, tl;

	lmask = _mm_set1_epi16(0xff);

	while (bsize != 0){
		for (j = 0; j < 8; j++){
			ta = _mm_load_si128((__m128i *)data);	// read 32-bytes
			tb = _mm_load_si128((__m128i *)data + 1);

			// split to high/low parts
			th = _mm_packus_epi16(_mm_srli_epi16(tb, 8), _mm_srli_epi16(ta, 8));
			tl = _mm_packus_epi16(_mm_and_si128(tb, lmask), _mm_and_si128(ta, lmask));

			// save to dest by extracting 16-bit masks
			dtmp[0 + j] = _mm_movemask_epi8(th);
			for (i = 1; i < 8; i++){
				th = _mm_slli_epi16(th, 1); // byte shift would be nicer, but ultimately doesn't matter here
				dtmp[i*8 + j] = _mm_movemask_epi8(th);
			}
			dtmp[64 + j] = _mm_movemask_epi8(tl);
			for (i = 1; i < 8; i++){
				tl = _mm_slli_epi16(tl, 1);
				dtmp[64 + i*8 + j] = _mm_movemask_epi8(tl);
			}
			data += 32;
		}
		// we only really need to copy temp -> dest
		memcpy(data - 256, dtmp, 256);
		bsize -= 256;
	}
}

void galois_altmap256_return(unsigned char *data, unsigned int bsize)
{
	unsigned short dtmp[128];
	int i, j;
	__m128i ta, tb, lmask, th, tl;

	th = _mm_setzero_si128();	// shut up compiler warning
	tl = _mm_setzero_si128();
	lmask = _mm_set1_epi16(0xff);

	while (bsize != 0){
		for (j = 0; j < 8; j++){
			// load in pattern: [0011223344556677] [8899AABBCCDDEEFF]
			tl = _mm_insert_epi16(tl, *(int *)(data + 240), 0);
			th = _mm_insert_epi16(th, *(int *)(data + 112), 0);
			tl = _mm_insert_epi16(tl, *(int *)(data + 224), 1);
			th = _mm_insert_epi16(th, *(int *)(data + 96), 1);
			tl = _mm_insert_epi16(tl, *(int *)(data + 208), 2);
			th = _mm_insert_epi16(th, *(int *)(data + 80), 2);
			tl = _mm_insert_epi16(tl, *(int *)(data + 192), 3);
			th = _mm_insert_epi16(th, *(int *)(data + 64), 3);
			tl = _mm_insert_epi16(tl, *(int *)(data + 176), 4);
			th = _mm_insert_epi16(th, *(int *)(data + 48), 4);
			tl = _mm_insert_epi16(tl, *(int *)(data + 160), 5);
			th = _mm_insert_epi16(th, *(int *)(data + 32), 5);
			tl = _mm_insert_epi16(tl, *(int *)(data + 144), 6);
			th = _mm_insert_epi16(th, *(int *)(data + 16), 6);
			tl = _mm_insert_epi16(tl, *(int *)(data + 128), 7);
			th = _mm_insert_epi16(th, *(int *)data, 7);

			// swizzle to [0123456789ABCDEF] [0123456789ABCDEF]
			ta = _mm_packus_epi16(_mm_srli_epi16(tl, 8),_mm_srli_epi16(th, 8));
			tb = _mm_packus_epi16(_mm_and_si128(tl, lmask),_mm_and_si128(th, lmask));

			// extract top bits
			dtmp[j*16 + 7] = _mm_movemask_epi8(ta);
			dtmp[j*16 + 15] = _mm_movemask_epi8(tb);
			for (i = 1; i < 8; i++){
				ta = _mm_slli_epi16(ta, 1);
				tb = _mm_slli_epi16(tb, 1);
				dtmp[j*16 + 7-i] = _mm_movemask_epi8(ta);
				dtmp[j*16 + 15-i] = _mm_movemask_epi8(tb);
			}
			data += 2;
		}

		// we only really need to copy temp -> dest
		memcpy(data - 16, dtmp, 256);
		data += 240;
		bsize -= 256;
	}
}

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

// 複数のテーブルを使って掛け算する
void galois_region_multiply(
	unsigned short *r1,	// Region to multiply
	unsigned short *r2,	// Products go here
	unsigned int count,	// Count of number in short
	int factor)			// Number to multiply by
{
	if (factor <= 1){
		if (factor == 0)
			return;

		// アドレスが 4の倍数で無い場合は 4バイト単位で計算する効率が落ちる
		if ((ULONG_PTR)r2 & 2){
			// そこで最初の 1個(2バイト)だけ普通に計算する
			*r2 ^= *r1;
			r1++;
			r2++;
			count--;
		}

		while (count >= 2){	// 2個(4バイト)ずつ計算する
			((unsigned int *)r2)[0] ^= ((unsigned int *)r1)[0];
			r1 += 2;
			r2 += 2;
			count -= 2;
		}
		if (count == 1)	// 最後に余ったやつだけ普通に計算する
			*r2 ^= *r1;
		return;
	}

	if (count >= 32){	// 64バイト以上なら掛け算用のテーブルを使った方が速い
#ifndef NO_SIMD
		if (cpu_flag & 1){	// SSSE3 対応なら
			__declspec( align(16) ) unsigned char small_table[128];
			int s, d;

			create_eight_table(small_table, factor);

			// アドレスが 16の倍数で無い場合は 16バイト単位で計算する効率が落ちる
			while ((ULONG_PTR)r2 & 0xE){
				// そこで最初の 1～7個(2～14バイト)だけ普通に計算する
				s = r1[0];
				d = r2[0];
				d ^= small_table[s & 0xF] | ((int)(small_table[16 + (s & 0xF)]) << 8);
				s = s >> 4;
				d ^= small_table[32 + (s & 0xF)] | ((int)(small_table[48 + (s & 0xF)]) << 8);
				s = s >> 4;
				d ^= small_table[64 + (s & 0xF)] | ((int)(small_table[80 + (s & 0xF)]) << 8);
				s = s >> 4;
				d ^= small_table[96 + s] | ((int)(small_table[112 + s]) << 8);
				r2[0] = (unsigned short)d;
				r1++;
				r2++;
				count--;
			}

			if (sse_unit == 16){
				// 8個ずつ計算するので 8の倍数にする
				gf16_ssse3_block16u((unsigned char *)r1, (unsigned char *)r2,
						(count & 0xFFFFFFF8) << 1, small_table);
				r1 += count & 0xFFFFFFF8;
				r2 += count & 0xFFFFFFF8;
				count &= 7;
			} else {
				// 16個ずつ計算するので 16の倍数にする
				gf16_ssse3_block32u((unsigned char *)r1, (unsigned char *)r2,
						(count & 0xFFFFFFF0) << 1, small_table);
				r1 += count & 0xFFFFFFF0;
				r2 += count & 0xFFFFFFF0;
				count &= 15;
			}

			// 残りは 1個ずつ計算する
			while (count != 0){
				s = r1[0];
				d = r2[0];
				d ^= small_table[s & 0xF] | ((int)(small_table[16 + (s & 0xF)]) << 8);
				s = s >> 4;
				d ^= small_table[32 + (s & 0xF)] | ((int)(small_table[48 + (s & 0xF)]) << 8);
				s = s >> 4;
				d ^= small_table[64 + (s & 0xF)] | ((int)(small_table[80 + (s & 0xF)]) << 8);
				s = s >> 4;
				d ^= small_table[96 + s] | ((int)(small_table[112 + s]) << 8);
				r2[0] = (unsigned short)d;
				r1++;
				r2++;
				count--;
			}

		} else {	// Combined Multi Table support (2 tables of 256-entries)
#endif
			unsigned int mtab[256 * 2];

			create_two_table(mtab, factor);	// build combined multiplication tables

			// アドレスが 8の倍数で無い場合は 8バイト単位で計算する効率が落ちる
			while ((ULONG_PTR)r2 & 6){
				// そこで最初の 1～3個(2～6バイト)だけ普通に計算する
				r2[0] ^= mtab[((unsigned char *)r1)[0]] ^ mtab[256 + ((unsigned char *)r1)[1]];
				r1++;
				r2++;
				count--;
			}

#ifndef _WIN64	// 32-bit 版なら MMX を使う
#ifndef NO_SIMD
			// 4個(8バイト)ずつ計算するので 4の倍数にする
			DoBlock8((unsigned char *)r1, (unsigned char *)r2, (count & 0xFFFFFFFC) << 1, mtab);
			r1 += count & 0xFFFFFFFC;
			r2 += count & 0xFFFFFFFC;
			count &= 3;

#else	// MMX を使わないなら
			// バッファーを 32-bit整数として扱う
			while (count >= 2){	// 2個(4バイト)ずつ計算する
				// 先に計算しておいた 2個の参照テーブルを使う
				((unsigned int *)r2)[0] ^= mtab[((unsigned char *)r1)[0]] ^ mtab[256 + ((unsigned char *)r1)[1]] ^
										 ((mtab[((unsigned char *)r1)[2]] ^ mtab[256 + ((unsigned char *)r1)[3]]) << 16);
				r1 += 2;
				r2 += 2;
				count -= 2;
			}
#endif

#else	// 64-bit 版なら 64-bit 整数を使う
			// バッファーを 64-bit整数として扱う
			while (count >= 4){	// 4個(8バイト)ずつ計算する
				// 先に計算しておいた 2個の参照テーブルを使う
				((unsigned __int64 *)r2)[0] ^=
						  mtab[((unsigned char *)r1)[0]] ^ mtab[256 + ((unsigned char *)r1)[1]] ^
						((mtab[((unsigned char *)r1)[2]] ^ mtab[256 + ((unsigned char *)r1)[3]]) << 16) ^
						((unsigned __int64)(mtab[((unsigned char *)r1)[4]] ^ mtab[256 + ((unsigned char *)r1)[5]]) << 32) ^
						((unsigned __int64)(mtab[((unsigned char *)r1)[6]] ^ mtab[256 + ((unsigned char *)r1)[7]]) << 48);
				r1 += 4;
				r2 += 4;
				count -= 4;
			}
#endif

			// 残りは 1個ずつ計算する
			while (count != 0){
				r2[0] ^= mtab[((unsigned char *)r1)[0]] ^ mtab[256 + ((unsigned char *)r1)[1]];
				r1++;
				r2++;
				count--;
			}
#ifndef NO_SIMD
		}
#endif

	} else {	// 小さいデータは普通に計算する
		int log_y = galois_log_table[factor];

		while (count != 0){
			r2[0] ^= galois_multiply_fix(r1[0], log_y);
			r1++;
			r2++;
			count--;
		}
	}
}

// 行列の割り算用、微妙に速いけど大差無し
void galois_region_divide(
	unsigned short *r1,	// Region to divide. products go here
	unsigned int count,	// Count of number in short
	int factor)			// Number to divide by
{
	factor = galois_reciprocal(factor);	// factor = 1 / factor

	if (count >= 32){
		unsigned int mtab[256 * 2];

		create_two_table(mtab, factor);	// 掛け算用のテーブルをその場で構成する

		// アドレスが 4の倍数で無い場合は 4バイト単位で計算する効率が落ちる
		if (((ULONG_PTR)r1 & 2) != 0){
			// そこで最初の 1個(2バイト)だけ普通に計算する
			r1[0] = (unsigned short)(mtab[((unsigned char *)r1)[0]] ^ mtab[256 + ((unsigned char *)r1)[1]]);
			r1++;
			count--;
		}

		// バッファーを 32-bit整数として扱う
		while (count >= 2){	// 2個(4バイト)ずつ計算する
			// 先に計算しておいた 2個の参照テーブルを使う
			((unsigned int *)r1)[0] = mtab[((unsigned char *)r1)[0]] ^ mtab[256 + ((unsigned char *)r1)[1]] ^
									((mtab[((unsigned char *)r1)[2]] ^ mtab[256 + ((unsigned char *)r1)[3]]) << 16);
			r1 += 2;
			count -= 2;
		}
		// 奇数なら最後に 1個余る
		if (count == 1)
			r1[0] = (unsigned short)(mtab[((unsigned char *)r1)[0]] ^ mtab[256 + ((unsigned char *)r1)[1]]);

	} else {	// 小さいデータは普通に計算する
		int log_y = galois_log_table[factor];

		while (count != 0){
			r1[0] = galois_multiply_fix(r1[0], log_y);
			r1++;
			count--;
		}
	}
}

// 16バイト境界のバッファー専用のXOR
void galois_align_xor(
	unsigned char *r1,	// Region to multiply
	unsigned char *r2,	// Products go here
	unsigned int len)	// Byte length
{
#ifndef NO_SIMD
	__m128i xmm0, xmm1;	// 16バイトごとに XOR する

	while (len != 0){
		xmm0 = _mm_load_si128((__m128i *)r1);
		xmm1 = _mm_load_si128((__m128i *)r2);
		xmm1 = _mm_xor_si128(xmm1, xmm0);
		_mm_store_si128((__m128i *)r2, xmm1);
		r1 += 16;
		r2 += 16;
		len -= 16;
	}

#else	// SSE2 を使わないなら
	while (len != 0){	// 4バイトずつ計算する
		((unsigned int *)r2)[0] ^= ((unsigned int *)r1)[0];
		r1 += 4;
		r2 += 4;
		len -= 4;
	}
#endif
}

// 16バイト境界のバッファー専用の掛け算
void galois_align16_multiply(
	unsigned char *r1,	// Region to multiply (must be aligned by 16)
	unsigned char *r2,	// Products go here
	unsigned int len,	// Byte length (must be multiple of 32)
	int factor)			// Number to multiply by
{
	if (factor <= 1){
		if (factor == 0)
			return;

#ifndef NO_SIMD
		{	__m128i xmm0, xmm1;	// 16バイトごとに XOR する
			while (len != 0){
				xmm0 = _mm_load_si128((__m128i *)r1);
				xmm1 = _mm_load_si128((__m128i *)r2);
				xmm1 = _mm_xor_si128(xmm1, xmm0);
				_mm_store_si128((__m128i *)r2, xmm1);
				r1 += 16;
				r2 += 16;
				len -= 16;
			}
		}
#else	// SSE2 を使わないなら
		while (len != 0){	// 4バイトずつ計算する
			((unsigned int *)r2)[0] ^= ((unsigned int *)r1)[0];
			r1 += 4;
			r2 += 4;
			len -= 4;
		}
#endif

	// 掛け算用のテーブルを常に作成する (32バイトだと少し遅くなる)
#ifndef NO_SIMD
	} else if (cpu_flag & 1){	// SSSE3 対応なら
		__declspec( align(16) ) unsigned char small_table[128];

		create_eight_table(small_table, factor);

		gf16_ssse3_block16u(r1, r2, len, small_table);
		// sse_unit が 32の倍数ならこちらでもいい
		//gf16_ssse3_block32u(r1, r2, len, small_table);

#endif
	} else {	// Combined Multi Table support (2 tables of 256-entries)
		unsigned int mtab[256 * 2];

		create_two_table(mtab, factor);	// build combined multiplication tables

#ifndef _WIN64	// 32-bit 版なら MMX を使う
#ifndef NO_SIMD
		DoBlock8(r1, r2, len, mtab); // process large chunk 8-bytes a shot

#else	// MMX を使わないなら
		// バッファーを 32-bit整数として扱う
		while (len != 0){	// 4バイトずつ計算する
			((unsigned int *)r2)[0] ^= mtab[r1[0]] ^ mtab[256 + r1[1]] ^
									 ((mtab[r1[2]] ^ mtab[256 + r1[3]]) << 16);
			r1 += 4;
			r2 += 4;
			len -= 4;
		}
#endif

#else	// 64-bit 版なら 64-bit 整数を使う
		// バッファーを 64-bit整数として扱う
		while (len != 0){	// 8バイトずつ計算する
			((unsigned __int64 *)r2)[0] ^=
					  mtab[r1[0]] ^ mtab[256 + r1[1]] ^
					((mtab[r1[2]] ^ mtab[256 + r1[3]]) << 16) ^
					((unsigned __int64)(mtab[r1[4]] ^ mtab[256 + r1[5]]) << 32) ^
					((unsigned __int64)(mtab[r1[6]] ^ mtab[256 + r1[7]]) << 48);
			r1 += 8;
			r2 += 8;
			len -= 8;
		}
#endif

	}
}

// 32バイトごとに並び替えられたバッファー専用の掛け算 (SSSE3 & ALTMAP)
void galois_align32_multiply(
	unsigned char *r1,	// Region to multiply (must be aligned by 16)
	unsigned char *r2,	// Products go here
	unsigned int len,	// Byte length (must be multiple of 32)
	int factor)			// Number to multiply by
{
	if (factor <= 1){
		if (factor != 0){
			__m128i xmm0, xmm1;	// 16バイトごとに XOR する

			while (len != 0){
				xmm0 = _mm_load_si128((__m128i *)r1);
				xmm1 = _mm_load_si128((__m128i *)r2);
				xmm1 = _mm_xor_si128(xmm1, xmm0);
				_mm_store_si128((__m128i *)r2, xmm1);
				r1 += 16;
				r2 += 16;
				len -= 16;
			}
		}

	// 掛け算用のテーブルを常に作成する (32バイトだと少し遅くなる)
	} else {
		__declspec( align(16) ) unsigned char small_table[128];

		create_eight_table(small_table, factor);

		gf16_ssse3_block32_altmap(r1, r2, len, small_table);
	}
}

// 256バイトごとに並び替えられたバッファー専用の JIT(SSE2) を使った掛け算
void galois_align256_multiply(
	unsigned char *r1,	// Region to multiply (must be aligned by 16)
	unsigned char *r2,	// Products go here
	unsigned int len,	// Byte length (must be multiple of 32)
	int factor)			// Number to multiply by
{
	if (factor <= 1){
		if (factor != 0){
			__m128i xmm0, xmm1;	// 16バイトごとに XOR する

			while (len != 0){
				xmm0 = _mm_load_si128((__m128i *)r1);
				xmm1 = _mm_load_si128((__m128i *)r2);
				xmm1 = _mm_xor_si128(xmm1, xmm0);
				_mm_store_si128((__m128i *)r2, xmm1);
				r1 += 16;
				r2 += 16;
				len -= 16;
			}
		}

	// 常に JIT(SSE2) を使う
	} else {
		gf16_sse2_block256_jit(r1, r2, len, factor);
	}
}

// 32バイトごとに並び替えられたバッファー専用の掛け算 (AVX2 & ALTMAP)
void galois_align32avx_multiply(
	unsigned char *r1,	// Region to multiply (must be aligned by 32)
	unsigned char *r2,	// Products go here
	unsigned int len,	// Byte length (must be multiple of 32)
	int factor)			// Number to multiply by
{
	if (factor <= 1){
		if (factor != 0){
			__m256i ymm0, ymm1;	// 32バイトごとに XOR する

			while (len != 0){
				ymm0 = _mm256_load_si256((__m256i *)r1);
				ymm1 = _mm256_load_si256((__m256i *)r2);
				ymm1 = _mm256_xor_si256(ymm1, ymm0);
				_mm256_store_si256((__m256i *)r2, ymm1);
				r1 += 32;
				r2 += 32;
				len -= 32;
			}

			_mm256_zeroupper();	// AVX-SSE 切り替えの回避

/*
			__m128i xmm0, xmm1;	// 16バイトごとに XOR する

			while (len != 0){
				xmm0 = _mm_load_si128((__m128i *)r1);
				xmm1 = _mm_load_si128((__m128i *)r2);
				xmm1 = _mm_xor_si128(xmm1, xmm0);
				_mm_store_si128((__m128i *)r2, xmm1);
				r1 += 16;
				r2 += 16;
				len -= 16;
			}
*/
		}

	// 掛け算用のテーブルを常に作成する (32バイトだと少し遅くなる)
	} else {
		__declspec( align(32) ) unsigned char small_table[128];

		create_eight_table(small_table, factor);

		gf16_avx2_block32(r1, r2, len, small_table);
	}
}

/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
// チェックサムを計算する

// buffer alignment must be 16, length must be multiple of 16
void checksum16(unsigned char *data, unsigned char *hash, int byte_size)
{
	int i, count;

#ifndef NO_SIMD	// SSE2 を使うなら
	__m128i temp16, prev16, data16, mask16, zero16, poly16;

	count = byte_size / 16;
	prev16 = _mm_setzero_si128();
	zero16 = _mm_setzero_si128();
	poly16 = _mm_set1_epi32(0x100B100B);	// PRIM_POLY = 0x1100B

	while (count > 0){	// HASH_RANGE バイトごとに
		// 16バイトごとに XOR する
		temp16 = _mm_setzero_si128();

		if (count < HASH_RANGE / 16){
			i = count;
			count = 0;
		} else {
			i = HASH_RANGE / 16;
			count -= HASH_RANGE / 16;
		}
		while (i > 0){
			data16 = _mm_load_si128((__m128i *)data);	// load 16-bytes
			temp16 = _mm_xor_si128(temp16, data16);
			data += 16;
			i--;
		}

		// 前回の値を 2倍して、今回の値を追加する
		//temp16 = _mm_xor_si128(temp16, prev16);	// 3倍する場合は、元の値も XOR すればいい
		mask16 = _mm_cmpgt_epi16(zero16, prev16);	// (0 > prev) ? 0xFFFF : 0x0000
		prev16 = _mm_slli_epi16(prev16, 1);			// prev *= 2
		mask16 = _mm_and_si128(mask16, poly16);		// 0x100B or 0x0000
		prev16 = _mm_xor_si128(prev16, mask16);

		prev16 = _mm_xor_si128(prev16, temp16);
	}
	_mm_store_si128((__m128i *)hash, prev16);

#else	// 4バイト整数で計算するなら
	unsigned int *data4, temp[4], prev[4], x, mask;

	data4 = (unsigned int *)data;
	count = byte_size / 4;
	for (i = 0; i < 4; i++)
		prev[i] = 0;

	while (count > 0){	// HASH_RANGE バイトごとに
		// 4バイトごとに 16バイトに XOR する
		for (i = 0; i < 4; i++)
			temp[i] = 0;
		for (i = 0; i < HASH_RANGE / 4; i++){
			temp[i & 3] ^= data4[i];
			count--;
			if (count == 0)
				break;
		}
		data4 += HASH_RANGE / 4;

		// 前回の値を 2倍して、今回の値を追加する
		for (i = 0; i < 4; i++){
			x = prev[i];
			mask = (x & 0x80008000) >> 15;	// 0x00010001 or 0x00000000
			x = (x & 0x7FFF7FFF) << 1;	// 3倍する場合は「^=」にすればいい
			x ^= mask * 0x100B;	// 0x100B100B or 0x00000000

			prev[i] = x ^ temp[i];
		}
	}
	for (i = 0; i < 4; i++)
		((unsigned int *)hash)[i] = prev[i];
#endif
}

// チェックサムを計算すると同時にデータを並び替える
// buffer alignment must be 16, length must be (multiple of 32) - 16
void checksum16_altmap32(unsigned char *data, unsigned char *hash, int byte_size)
{
	int i, count;
	__m128i temp16, prev16, mask16, zero16, poly16;
	__m128i dataA, dataB, dataC, dataD, maskB;

	count = byte_size / 16;
	prev16 = _mm_setzero_si128();
	zero16 = _mm_setzero_si128();
	poly16 = _mm_set1_epi32(0x100B100B);	// PRIM_POLY = 0x1100B
	maskB = _mm_set1_epi16(0x00FF);	// 0x00FF *8

	while (count > 0){	// HASH_RANGE バイトごとに
		// 16バイトごとに XOR する
		temp16 = _mm_setzero_si128();

		if (count < HASH_RANGE / 16){
			i = count;
			count = 0;
		} else {
			i = HASH_RANGE / 16;
			count -= HASH_RANGE / 16;
		}
		while (i >= 2){
			dataA = _mm_load_si128((__m128i *)data);	// read 32-bytes
			dataB = _mm_load_si128((__m128i *)data + 1);
			temp16 = _mm_xor_si128(temp16, dataA);
			temp16 = _mm_xor_si128(temp16, dataB);

			dataC = _mm_and_si128(dataA, maskB);	// erase higher byte
			dataD = _mm_and_si128(dataB, maskB);
			dataA = _mm_srli_epi16(dataA, 8);		// move higher byte to lower
			dataB = _mm_srli_epi16(dataB, 8);
			dataC = _mm_packus_epi16(dataC, dataD);	//  select lower byte of each word
			dataA = _mm_packus_epi16(dataA, dataB);	//  select higher byte of each word

			_mm_store_si128((__m128i *)data, dataC);	// write 32-bytes
			_mm_store_si128((__m128i *)data + 1, dataA);

			data += 32;
			i -= 2;
		}
		if (i > 0){
			dataA = _mm_load_si128((__m128i *)data);	// load 16-bytes
			temp16 = _mm_xor_si128(temp16, dataA);
		}

		// 前回の値を 2倍して、今回の値を追加する
		//temp16 = _mm_xor_si128(temp16, prev16);	// 3倍する場合は、元の値も XOR すればいい
		mask16 = _mm_cmpgt_epi16(zero16, prev16);	// (0 > prev) ? 0xFFFF : 0x0000
		prev16 = _mm_slli_epi16(prev16, 1);			// prev *= 2
		mask16 = _mm_and_si128(mask16, poly16);		// 0x100B or 0x0000
		prev16 = _mm_xor_si128(prev16, mask16);

		prev16 = _mm_xor_si128(prev16, temp16);
	}
	if (hash != data + 16)	// ハッシュ値の保存先が別なら
		_mm_store_si128((__m128i *)hash, prev16);

	// 最後にハッシュ値も並び替える
	dataC = _mm_and_si128(dataA, maskB);	// erase higher byte
	dataD = _mm_and_si128(prev16, maskB);
	dataA = _mm_srli_epi16(dataA, 8);		// move higher byte to lower
	dataB = _mm_srli_epi16(prev16, 8);
	dataC = _mm_packus_epi16(dataC, dataD);	//  select lower byte of each word
	dataA = _mm_packus_epi16(dataA, dataB);	//  select higher byte of each word

	_mm_store_si128((__m128i *)data, dataC);	// write 32-bytes
	_mm_store_si128((__m128i *)data + 1, dataA);
}

// データの並びを元に戻すと同時にチェックサムを計算する
// buffer alignment must be 32, length must be (multiple of 32) - 16
void checksum16_return32(unsigned char *data, unsigned char *hash, int byte_size)
{
	int i, count;
	__m128i temp16, prev16, mask16, zero16, poly16;
	__m128i dataA, dataB, dataC;

	count = byte_size / 16;
	prev16 = _mm_setzero_si128();
	zero16 = _mm_setzero_si128();
	poly16 = _mm_set1_epi32(0x100B100B);	// PRIM_POLY = 0x1100B

	while (count > 0){	// HASH_RANGE バイトごとに
		// 16バイトごとに XOR する
		temp16 = _mm_setzero_si128();

		if (count < HASH_RANGE / 16){
			i = count;
			count = 0;
		} else {
			i = HASH_RANGE / 16;
			count -= HASH_RANGE / 16;
		}
		while (i >= 2){
			dataA = _mm_load_si128((__m128i *)data);	// read 32-bytes
			dataB = _mm_load_si128((__m128i *)data + 1);

			dataC = _mm_unpacklo_epi8(dataA, dataB);	// interleave lower and higher bytes
			dataA = _mm_unpackhi_epi8(dataA, dataB);

			_mm_store_si128((__m128i *)data, dataC);	// write 32-bytes
			_mm_store_si128((__m128i *)data + 1, dataA);
			temp16 = _mm_xor_si128(temp16, dataC);
			temp16 = _mm_xor_si128(temp16, dataA);

			data += 32;
			i -= 2;
		}
		if (i > 0){
			dataA = _mm_load_si128((__m128i *)data);	// read 32-bytes
			dataB = _mm_load_si128((__m128i *)data + 1);

			dataC = _mm_unpacklo_epi8(dataA, dataB);	// interleave lower and higher bytes
			dataA = _mm_unpackhi_epi8(dataA, dataB);

			_mm_store_si128((__m128i *)data, dataC);	// write 32-bytes
			_mm_store_si128((__m128i *)data + 1, dataA);
			temp16 = _mm_xor_si128(temp16, dataC);
		}

		// 前回の値を 2倍して、今回の値を追加する
		//temp16 = _mm_xor_si128(temp16, prev16);	// 3倍する場合は、元の値も XOR すればいい
		mask16 = _mm_cmpgt_epi16(zero16, prev16);	// (0 > prev) ? 0xFFFF : 0x0000
		prev16 = _mm_slli_epi16(prev16, 1);			// prev *= 2
		mask16 = _mm_and_si128(mask16, poly16);		// 0x100B or 0x0000
		prev16 = _mm_xor_si128(prev16, mask16);

		prev16 = _mm_xor_si128(prev16, temp16);
	}

	if (hash != data + 16)	// ハッシュ値の保存先が別なら
		_mm_store_si128((__m128i *)hash, prev16);
}

// チェックサムを計算すると同時にデータを並び替える
// buffer alignment must be 256, length must be (multiple of 256) - 16
void checksum16_altmap256(unsigned char *data, unsigned char *hash, int byte_size)
{
	// 順番に処理する場合
//	checksum16(data, hash, byte_size);
//	galois_altmap256_change(data, byte_size + HASH_SIZE);

	unsigned short dtmp[128];
	int i, j;
	__m128i ta, tb, lmask, th, tl, temp16, prev16, poly16;

	lmask = _mm_set1_epi16(0xff);
	prev16 = _mm_setzero_si128();
	poly16 = _mm_set1_epi32(0x100B100B);	// PRIM_POLY = 0x1100B

	byte_size += HASH_SIZE;
	while (byte_size != 0){
		for (j = 0; j < 8; j++){
			ta = _mm_load_si128((__m128i *)data);	// read 32-bytes
			tb = _mm_load_si128((__m128i *)data + 1);

			// 128バイトを XOR して 16バイトにする
			if ((j & 3) == 0)
				temp16 = _mm_setzero_si128();
			temp16 = _mm_xor_si128(temp16, ta);
			if ((j != 7) || (byte_size != 256))	// 最後の 16バイトだけ含めない
				temp16 = _mm_xor_si128(temp16, tb);

			if ((j & 3) == 3){	// j = 3 or 7
				// 前回の値を 2倍して、今回の値を追加する
				//temp16 = _mm_xor_si128(temp16, prev16);	// 3倍する場合は、元の値も XOR すればいい
				tl = _mm_setzero_si128();
				th = _mm_cmpgt_epi16(tl, prev16);	// (0 > prev) ? 0xFFFF : 0x0000
				prev16 = _mm_slli_epi16(prev16, 1);	// prev *= 2
				th = _mm_and_si128(th, poly16);		// 0x100B or 0x0000
				prev16 = _mm_xor_si128(prev16, th);

				prev16 = _mm_xor_si128(prev16, temp16);
				if ((j == 7) && (byte_size == 256)){
					if (hash != data + 16)	// ハッシュ値の保存先が別なら
						_mm_store_si128((__m128i *)hash, prev16);
					_mm_store_si128(&tb, prev16);	// 最後の 16バイトにハッシュ値を置く
				}
			}

			// split to high/low parts
			th = _mm_packus_epi16(_mm_srli_epi16(tb, 8), _mm_srli_epi16(ta, 8));
			tl = _mm_packus_epi16(_mm_and_si128(tb, lmask), _mm_and_si128(ta, lmask));

			// save to dest by extracting 16-bit masks
			dtmp[0 + j] = _mm_movemask_epi8(th);
			for (i = 1; i < 8; i++){
				th = _mm_slli_epi16(th, 1); // byte shift would be nicer, but ultimately doesn't matter here
				dtmp[i*8 + j] = _mm_movemask_epi8(th);
			}
			dtmp[64 + j] = _mm_movemask_epi8(tl);
			for (i = 1; i < 8; i++){
				tl = _mm_slli_epi16(tl, 1);
				dtmp[64 + i*8 + j] = _mm_movemask_epi8(tl);
			}
			data += 32;
		}
		// we only really need to copy temp -> dest
		memcpy(data - 256, dtmp, 256);
		byte_size -= 256;
	}
}

// データの並びを元に戻すと同時にチェックサムを計算する
// buffer alignment must be 256, length must be (multiple of 256) - 16
void checksum16_return256(unsigned char *data, unsigned char *hash, int byte_size)
{
	// 順番に処理する場合
//	galois_altmap256_return(data, byte_size + HASH_SIZE);
//	checksum16(data, hash, byte_size);

	__declspec( align(16) ) unsigned short dtmp[128];
	int i, j;
	__m128i ta, tb, lmask, th, tl, temp16, prev16, poly16;

	th = _mm_setzero_si128();	// shut up compiler warning
	tl = _mm_setzero_si128();
	lmask = _mm_set1_epi16(0xff);
	prev16 = _mm_setzero_si128();
	poly16 = _mm_set1_epi32(0x100B100B);	// PRIM_POLY = 0x1100B

	byte_size += HASH_SIZE;
	while (byte_size != 0){
		for (j = 0; j < 8; j++){
			// load in pattern: [0011223344556677] [8899AABBCCDDEEFF]
			tl = _mm_insert_epi16(tl, *(int *)(data + 240), 0);
			th = _mm_insert_epi16(th, *(int *)(data + 112), 0);
			tl = _mm_insert_epi16(tl, *(int *)(data + 224), 1);
			th = _mm_insert_epi16(th, *(int *)(data + 96), 1);
			tl = _mm_insert_epi16(tl, *(int *)(data + 208), 2);
			th = _mm_insert_epi16(th, *(int *)(data + 80), 2);
			tl = _mm_insert_epi16(tl, *(int *)(data + 192), 3);
			th = _mm_insert_epi16(th, *(int *)(data + 64), 3);
			tl = _mm_insert_epi16(tl, *(int *)(data + 176), 4);
			th = _mm_insert_epi16(th, *(int *)(data + 48), 4);
			tl = _mm_insert_epi16(tl, *(int *)(data + 160), 5);
			th = _mm_insert_epi16(th, *(int *)(data + 32), 5);
			tl = _mm_insert_epi16(tl, *(int *)(data + 144), 6);
			th = _mm_insert_epi16(th, *(int *)(data + 16), 6);
			tl = _mm_insert_epi16(tl, *(int *)(data + 128), 7);
			th = _mm_insert_epi16(th, *(int *)data, 7);

			// swizzle to [0123456789ABCDEF] [0123456789ABCDEF]
			ta = _mm_packus_epi16(_mm_srli_epi16(tl, 8),_mm_srli_epi16(th, 8));
			tb = _mm_packus_epi16(_mm_and_si128(tl, lmask),_mm_and_si128(th, lmask));

			// extract top bits
			dtmp[j*16 + 7] = _mm_movemask_epi8(ta);
			dtmp[j*16 + 15] = _mm_movemask_epi8(tb);
			for (i = 1; i < 8; i++){
				ta = _mm_slli_epi16(ta, 1);
				tb = _mm_slli_epi16(tb, 1);
				dtmp[j*16 + 7-i] = _mm_movemask_epi8(ta);
				dtmp[j*16 + 15-i] = _mm_movemask_epi8(tb);
			}
			data += 2;
		}

		// 128バイトを XOR して 16バイトにする
		temp16 = _mm_setzero_si128();
		for (j = 0; j < 8; j++){
			ta = _mm_load_si128((__m128i *)dtmp + j);
			temp16 = _mm_xor_si128(temp16, ta);
		}
		// 前回の値を 2倍して、今回の値を追加する
		//temp16 = _mm_xor_si128(temp16, prev16);	// 3倍する場合は、元の値も XOR すればいい
		tl = _mm_setzero_si128();
		th = _mm_cmpgt_epi16(tl, prev16);	// (0 > prev) ? 0xFFFF : 0x0000
		prev16 = _mm_slli_epi16(prev16, 1);	// prev *= 2
		th = _mm_and_si128(th, poly16);		// 0x100B or 0x0000
		prev16 = _mm_xor_si128(prev16, th);
		prev16 = _mm_xor_si128(prev16, temp16);

		if (byte_size != 256){
			// 次の 128バイトを XOR して 16バイトにする
			temp16 = _mm_setzero_si128();
			for (j = 8; j < 16; j++){
				ta = _mm_load_si128((__m128i *)dtmp + j);
				temp16 = _mm_xor_si128(temp16, ta);
			}
			// 前回の値を 2倍して、今回の値を追加する
			//temp16 = _mm_xor_si128(temp16, prev16);	// 3倍する場合は、元の値も XOR すればいい
			tl = _mm_setzero_si128();
			th = _mm_cmpgt_epi16(tl, prev16);	// (0 > prev) ? 0xFFFF : 0x0000
			prev16 = _mm_slli_epi16(prev16, 1);	// prev *= 2
			th = _mm_and_si128(th, poly16);		// 0x100B or 0x0000
			prev16 = _mm_xor_si128(prev16, th);
			prev16 = _mm_xor_si128(prev16, temp16);

		} else {	// 最後の 128バイトは特別
			// 次の 112バイトを XOR して 16バイトにする
			temp16 = _mm_setzero_si128();
			for (j = 8; j < 15; j++){
				ta = _mm_load_si128((__m128i *)dtmp + j);
				temp16 = _mm_xor_si128(temp16, ta);
			}
			// 前回の値を 2倍して、今回の値を追加する
			//temp16 = _mm_xor_si128(temp16, prev16);	// 3倍する場合は、元の値も XOR すればいい
			tl = _mm_setzero_si128();
			th = _mm_cmpgt_epi16(tl, prev16);	// (0 > prev) ? 0xFFFF : 0x0000
			prev16 = _mm_slli_epi16(prev16, 1);	// prev *= 2
			th = _mm_and_si128(th, poly16);		// 0x100B or 0x0000
			prev16 = _mm_xor_si128(prev16, th);
			prev16 = _mm_xor_si128(prev16, temp16);
		}

		// we only really need to copy temp -> dest
		memcpy(data - 16, dtmp, 256);
		data += 240;
		byte_size -= 256;
	}

	if (hash != data - 16)	// ハッシュ値の保存先が別なら
		_mm_store_si128((__m128i *)hash, prev16);
}

