import itertools
import warnings
from collections import defaultdict
import numpy as np
import pandas as pd
import scipy.stats
import statsmodels.stats.api as sms
import math 
from scipy import stats
from scipy.stats import ttest_ind
warnings.filterwarnings("ignore")

def split_freq(text):
    '''
    把文学作品中的所有文字（符号）拆分成两个连续字母（符号）为单元的 dataframe 数据集
    '''
    # split data
    data = list(text)
    A = []
    for i, j in zip(data, data[1:]):
        c = i + j
        # print(c)
        A.append(c)
    # A[:5]
    B = []
    for i, j in zip(data[1:], data[2:]):
        c = i + j
        # print(c)
        B.append(c)
    # B[:5]
    all = A + B
    # print(len(A),"+",len(B),"=",len(all))

    # get the freq count for all
    d = defaultdict(int)

    for word in all:
        d[word] += 1

    df = pd.DataFrame(d.items())
    df.set_index(0, inplace=True)

    return (df, len(all))

def get_result(target, df):
    '''
    获得不同打字机（键盘）相邻字母在文字数据中出现的频次统计
    '''
    seq = df.loc[list(target["ABC"]),]
    seq.dropna(axis=0, inplace=True)
    seq = seq.reset_index()
    seq.columns = ["word_ABC", "freq_ABC"]
    data = []
    data.insert(0, {"word_ABC": "sum", "freq_ABC": sum(seq.freq_ABC)})
    seq_all = pd.concat([pd.DataFrame(data), seq], ignore_index=True)
    seq_all = seq_all.sort_values("freq_ABC", ascending=False).reset_index()

    qwerty = df.loc[list(target["QWERTY"]),]
    qwerty.dropna(axis=0, inplace=True)
    qwerty = qwerty.reset_index()
    qwerty.columns = ["word_QWERTY", "freq_QWERTY"]
    data = []
    data.insert(0, {"word_QWERTY": "sum", "freq_QWERTY": sum(qwerty.freq_QWERTY)})
    qwerty_all = pd.concat([pd.DataFrame(data), qwerty], ignore_index=True)
    qwerty_all = qwerty_all.sort_values("freq_QWERTY", ascending=False).reset_index()

    qwerty_m = df.loc[list(target["QWERTY_m"]),]
    qwerty_m.dropna(axis=0, inplace=True)
    qwerty_m = qwerty_m.reset_index()
    qwerty_m.columns = ["word_QWERTY_m", "freq_QWERTY_m"]
    data = []
    data.insert(
        0, {"word_QWERTY_m": "sum", "freq_QWERTY_m": sum(qwerty_m.freq_QWERTY_m)}
    )
    qwerty_m_all = pd.concat([pd.DataFrame(data), qwerty_m], ignore_index=True)
    qwerty_m_all = qwerty_m_all.sort_values("freq_QWERTY_m", ascending=False).reset_index()

    result = pd.concat([seq_all, qwerty_all, qwerty_m_all], axis=1, ignore_index=False)
    result = result[
        [
            "word_ABC",
            "freq_ABC",
            "word_QWERTY",
            "freq_QWERTY",
            "word_QWERTY_m",
            "freq_QWERTY_m",
        ]
    ]
    return result

def get_result_lr(QWERTY_right_left,QWERTY_left_right,ABC_right_left,ABC_left_right,QWERTY_m_right_left,QWERTY_m_left_right, df):
    '''
    获得不同打字机（键盘）者左右交叉字母在文字数据中出现的频次统计
    '''
    seq = df.loc[list(ABC_right_left) + list(ABC_left_right),]
    seq.dropna(axis=0, inplace=True)
    seq = seq.reset_index()
    seq.columns = ["ABC_right_left", "freq_ABC_right_left"]
    data = []
    data.insert(0, {"ABC_right_left": "sum", "freq_ABC_right_left": sum(seq.freq_ABC_right_left)})
    seq_all = pd.concat([pd.DataFrame(data), seq], ignore_index=True)
    seq_all = seq_all.sort_values("freq_ABC_right_left", ascending=False).reset_index()
   

    qwerty = df.loc[list(QWERTY_right_left) + list(QWERTY_left_right),]
    qwerty.dropna(axis=0, inplace=True)
    qwerty = qwerty.reset_index()
    qwerty.columns = ["QWERTY_right_left", "freq_QWERTY_right_left"]
    data = []
    data.insert(0, {"QWERTY_right_left": "sum", "freq_QWERTY_right_left": sum(qwerty.freq_QWERTY_right_left)})
    qwerty_all = pd.concat([pd.DataFrame(data), qwerty], ignore_index=True)
    qwerty_all = qwerty_all.sort_values("freq_QWERTY_right_left", ascending=False).reset_index()
    
    qwerty_m = df.loc[list(QWERTY_m_right_left) + list(QWERTY_m_left_right),]
    qwerty_m.dropna(axis=0, inplace=True)
    qwerty_m = qwerty_m.reset_index()
    qwerty_m.columns = ["QWERTY_m_right_left", "freq_QWERTY_m_right_left"]
    data = []
    data.insert(0, {"QWERTY_m_right_left": "sum", "freq_QWERTY_m_right_left": sum(qwerty_m.freq_QWERTY_m_right_left)})
    qwerty_m_all = pd.concat([pd.DataFrame(data), qwerty_m], ignore_index=True)
    qwerty_m_all = qwerty_m_all.sort_values("freq_QWERTY_m_right_left", ascending=False).reset_index()
    
    result = pd.concat([seq_all, qwerty_all,qwerty_m_all], axis=1, ignore_index=False)
    result = result[
        [
            "ABC_right_left",
            "freq_ABC_right_left",
            "QWERTY_right_left",
            "freq_QWERTY_right_left",
            "QWERTY_m_right_left",
            "freq_QWERTY_m_right_left",
        ]
    ]
    return result

col = [
"book No.",
"num_combinations",
"ABC%",
"QWERTY%",
"QWERTY_m%",
"QWERTY vs ABC %",
"QWERTY_m vs QWERTY %",
"QWERTY_m vs ABC %",
]

def eva(result,i,ni):
    '''
    获得每个文字数据集的各种打字机（键盘）参数百分比
    '''
    abc_per = round(result.freq_ABC[0] / n * 100, 3)
    QWERTY_per = round(result.freq_QWERTY[0] / n * 100, 3)
    QWERTY_m_per = round(result.freq_QWERTY_m[0] / n * 100, 3)

    delta = round((QWERTY_per - abc_per) / abc_per * 100, 3)
    delta2 = round((QWERTY_m_per - QWERTY_per) / QWERTY_m_per * 100, 3)
    delta3 = round((QWERTY_m_per - abc_per) / abc_per * 100, 3)

    compare_n = pd.DataFrame(
        [[i,ni, abc_per, QWERTY_per, QWERTY_m_per, delta, delta2, delta3]], columns=col
    )
    return compare_n

col_lr = [
    "book No.",
    "num_combinations",
    "ABC_right_left%",
    "QWERTY_right_left%",
    "QWERTY_m_right_left%",
    "QWERTY vs ABC right left %",
    "QWERTY_m vs QWERTY %"
]

def eva_lr(result,i,ni):
    '''
    获得每个文字数据集的各种打字机（键盘）参数百分比
    '''
    ABC_right_left_per = round(result.freq_ABC_right_left[0] / n * 100, 3)
    QWERTY_right_left_per = round(result.freq_QWERTY_right_left[0] / n * 100, 3)
    QWERTY_m_right_left_per = round(result.freq_QWERTY_m_right_left[0] / n * 100, 3)

    delta = round((QWERTY_right_left_per - ABC_right_left_per) / ABC_right_left_per * 100, 3)
    delta2 = round((QWERTY_m_right_left_per - QWERTY_right_left_per) / QWERTY_right_left_per * 100, 3)
    
    compare_n = pd.DataFrame(
        [[i,ni, ABC_right_left_per, QWERTY_right_left_per, QWERTY_m_right_left_per, delta,delta2]], columns=col_lr
    )
    return compare_n

def weighted_avg_and_std(data, weights):
    """
    计算加权平均和加权标准差
    """
    average = np.average(data, weights=weights)
    # Fast and numerically precise:
    variance = np.average((data-average)**2, weights=weights)
    return (average, math.sqrt(variance))


def weighted_mean_confidence_interval(data, weights, confidence=0.95):
    """
    计算加权置信区间
    """
    a = 1.0 * np.array(data)
    n = len(a)
    m, se = weighted_avg_and_std(data, weights)
    h = se * scipy.stats.t.ppf((1 + confidence) / 2.0, n - 1)
    print("Weighted Mean: %.3f \nWeighted %0.4f Confidence Interval: [%.3f,%.3f]"% (m, confidence, m - h, m + h))

def product(a, b):
    '''
    输入所有左（右）手打的字母array a 
    和所有右（左）手打的字母array b，
    输出所有可能的左右连续打字的组合，
    顺序从a到b
    '''
    c = list(itertools.product(a, b))
    QWERTY_right_left = pd.DataFrame(c).dropna()
    QWERTY_right_left = QWERTY_right_left[0] + QWERTY_right_left[1]
    return QWERTY_right_left

# get the target combinations
target = pd.read_csv("target.csv")
target

#load bible
text1 = open("books/Bible_ASV.txt", "r").read().lower()
df, n = split_freq(text1)
result1 = get_result(target, df)
result1.to_csv("result/Bible_ASV.csv")
#result1

eva(result1,"bible",n)

t1 = open("books/bad_boy.txt", "r").read().lower()
t2 = open("books/Blue_Jackets.txt", "r").read().lower()
t3 = open("books/From_the_Earth_to_the_Moon.txt", "r").read().lower()
t4 = open("books/Joseph_and_His_Friend.txt", "r").read().lower()
t5 = open("books/Lothair.txt", "r").read().lower()
t6 = open("books/Man_and_Wife.txt", "r").read().lower()
t7 = open("books/Memoir_of_Jane_Austen.txt", "r").read().lower()
t8 = open("books/The_Adventures_of_Harry_Richmond.txt", "r").read().lower()
t9 = open("books/The_Caged_Lion.txt", "r").read().lower()
t10 = open("books/The_Earthly_Paradis.txt", "r").read().lower()
t11 = open("books/The_Mystery_of_Edwin_Drood.txt", "r").read().lower()
t12 = open("books/The_Visionary.txt", "r").read().lower()
t13 = open("books/The_Vicar_of_Bullhampton_by_Anthony_Trollope.txt", "r").read().lower()
t14 = open("books/The_Wild_Garden.txt", "r").read().lower()
t15 = open("books/Twenty_Thousand_Leagues_under_the_Sea_by_Jules_Verne.txt", "r").read().lower()
t16 = open("books/Venus_in_Furs_by_Ritter_von_Leopold_Sacher-Masoch.txt", "r").read().lower()
t17 = open("books/The_Adventures_of_Tom_Sawyer.txt", "r").read().lower()
t18 = open("books/Atthe_Back_of_the_North_Wind.txt", "r").read().lower()
t19 = open("books/Coles_Funny_Picture_Book.txt", "r").read().lower()
t20 = open("books/The_Cuckoo_Clock.txt", "r").read().lower()
t21 = open("books/The_Lost_Princess.txt", "r").read().lower()
t22 = open("books/Mildred_Keith.txt", "r").read().lower()
t23 = open("books/The_Princess_and_the_Goblin.txt", "r").read().lower()
t24 = open("books/What_Katy_Did.txt", "r").read().lower()
t25 = open("books/Under_the_Window.txt", "r").read().lower()
t26 = open("books/Carmilla.txt", "r").read().lower()
t27 = open("books/Erewhon.txt", "r").read().lower()
t28 = open("books/Daisy_Miller.txt", "r").read().lower()
t29 = open("books/Leavenworth.txt", "r").read().lower()
t30 = open("books/Rosein_Bloom.txt", "r").read().lower()

text2 = (
    t1 + t2 + t3 + t4 + t5 + t6 + t7 + t8 + t9 + t10 + t11
    + t12 + t13 + t14 + t15 + t16 + t17 + t18 + t19 + t20 
    + t21 + t22 + t23 + t24 + t25 + t26 + t27 + t28 + t29 + t30
)

df, n = split_freq(text2)
result2 = get_result(target, df)
result2.to_csv("result/books30.csv")
#result2

eva(result2,"2-30",n)

text3 = text1 + text2
df, n = split_freq(text3)
result3 = get_result(target, df)
result3.to_csv("result/books_and_bible.csv")
result3

eva(result3,"all",n)

compare = pd.DataFrame(columns=col)
text_iter = [text1,t1,t2,t3,t4,t5,t6,t7,t8,t9,t10,t11,t12,t13,t14,t15,
             t16,t17,t18,t19,t20,t21,t22,t23,t24,t25,t26, t27,t28,t29,t30,
]
c=0
for i in text_iter:
    c+=1
    dfi, ni = split_freq(i)
    result_i = get_result(target, dfi)
    compare_i = eva(result_i,c,ni)
    compare = compare.append(compare_i)
compare = compare.sort_values("num_combinations", ascending=False).reset_index().drop(['index'], axis=1)
compare

data = compare["QWERTY vs ABC %"]
weights = compare["num_combinations"]
confidence=0.9999
weighted_mean_confidence_interval(data,weights,confidence)

data = compare["QWERTY_m vs QWERTY %"]
weights = compare["num_combinations"]
confidence=0.9999
weighted_mean_confidence_interval(data,weights,confidence)

lis = []
for i in result3["word_QWERTY_m"]:
    if i not in list(result3["word_QWERTY"]):
        a = result3["freq_QWERTY_m"][result3["word_QWERTY_m"] == i].values[0]
        lis.append([i, a])
diff = pd.DataFrame(lis).sort_values(1, ascending=False)
diff.columns = ['word_QWERTY_m', 'freq_QWERTY_m']
diff

a = target.QWERTY_right
b = target.QWERTY_left

ABC_right_left = product(target.ABC_right, target.ABC_left)
ABC_left_right = product(target.ABC_left, target.ABC_right)
QWERTY_right_left = product(target.QWERTY_right, target.QWERTY_left)
QWERTY_left_right = product(target.QWERTY_left, target.QWERTY_right)
QWERTY_m_right_left = product(target.QWERTY_m_right, target.QWERTY_m_left)
QWERTY_m_left_right = product(target.QWERTY_m_left, target.QWERTY_m_right)

result_lr = get_result_lr(QWERTY_right_left,QWERTY_left_right,ABC_right_left,ABC_left_right,QWERTY_m_right_left,QWERTY_m_left_right, df)
result_lr.to_csv("result/left_right.csv")
result_lr.head(30)

eva_lr(result_lr,"all",n)

compare = pd.DataFrame(columns=col_lr)
c=0
for i in text_iter:
    c+=1
    dfi, ni = split_freq(i)
    result_i = get_result_lr(QWERTY_right_left,QWERTY_left_right,ABC_right_left,ABC_left_right,QWERTY_m_right_left,QWERTY_m_left_right, dfi)
    compare_i = eva_lr(result_i,c,ni)
    compare = compare.append(compare_i)
compare_lr = compare.sort_values("num_combinations", ascending=False).reset_index().drop(['index'], axis=1)
compare_lr

data = compare_lr["QWERTY vs ABC right left %"]
weights = compare_lr["num_combinations"]
confidence=0.9999
weighted_mean_confidence_interval(data,weights,confidence)

data = compare_lr["QWERTY_m vs QWERTY %"]
weights = compare_lr["num_combinations"]
confidence=0.9999
weighted_mean_confidence_interval(data,weights,confidence)

a = compare_lr["QWERTY_right_left%"]
b = compare_lr["QWERTY_m_right_left%"]
t, p = ttest_ind(a, b, equal_var=False)
print("t-value: {} \np-value: {}".format(t,p))