Pythonを使用した潜在的な意味解析結果の視覚化

#!/usr/bin/env python # -*- coding: utf-8 -*- import numpy from numpy import * import nltk import scipy from nltk.corpus import brown from nltk.stem import SnowballStemmer from scipy.spatial.distance import pdist from scipy.cluster import hierarchy import matplotlib.pyplot as plt stemmer = SnowballStemmer('russian') stopwords=nltk.corpus.stopwords.words('russian') #      docs =[ "      WikiLeaks",#  № 0 "      ,  ",#  №1 "      19 ",#  №2 "      Wikileaks  ",#  №3 "     ",#  №4 "      Wikileaks",#  №5 "         ",#  №6 "    WikiLeaks, ,  ",#  №7 "        "#  №8 ] word=nltk.word_tokenize((' ').join(docs))#      n=[stemmer.stem(w).lower() for w in word if len(w) >1 and w.isalpha()]#      stopword=[stemmer.stem(w).lower() for w in stopwords]# - fdist=nltk.FreqDist(n) t=fdist.hapaxes()#       #    d={};c=[] for i in range(0,len(docs)): word=nltk.word_tokenize(docs[i]) word_stem=[stemmer.stem(w).lower() for w in word if len(w)>1 and w.isalpha()] word_stop=[ w for w in word_stem if w not in stopword] words=[ w for w in word_stop if w not in t] for w in words: if w not in c: c.append(w) d[w]= [i] elif w in c: d[w]= d[w]+[i] a=len(c); b=len(docs) A = numpy.zeros([a,b]) c.sort() for i, k in enumerate(c): for j in d[k]: A[i,j] += 1 # TF-IDF    wpd = sum(A, axis=0) dpw= sum(asarray(A > 0,'i'), axis=1) rows, cols = A.shape for i in range(rows): for j in range(cols): m=float(A[i,j])/wpd[j] n=log(float(cols) /dpw[i]) A[i,j] =round(n*m,2) #     U, S,Vt = numpy.linalg.svd(A) rows, cols = U.shape for j in range(0,cols): for i in range(0,rows): U[i,j]=round(U[i,j],4) print(' 2    U ') for i, row in enumerate(U): print(c[i], row[0:2]) res1=-1*U[:,0:1]; res2=-1*U[:,1:2] data_word=[] for i in range(0,len(c)):#         data_word.append([res1[i][0],res2[i][0]]) plt.figure() plt.subplot(221) dist = pdist(data_word, 'euclidean')#    ( ) plt.hist(dist, 500, color='green', alpha=0.5)#    Z = hierarchy.linkage(dist, method='average')#   plt.subplot(222) hierarchy.dendrogram(Z, labels=c, color_threshold=.25, leaf_font_size=8, count_sort=True,orientation='right') print(' 2    Vt ') rows, cols = Vt.shape for j in range(0,cols): for i in range(0,rows): Vt[i,j]=round(Vt[i,j],4) print(-1*Vt[0:2, :]) res3=(-1*Vt[0:1, :]);res4=(-1*Vt[1:2, :]) data_docs=[];name_docs=[] for i in range(0,len(docs)): name_docs.append(str(i)) data_docs.append([res3[0][i],res4[0][i]]) plt.subplot(223) dist = pdist(data_docs, 'euclidean') plt.hist(dist, 500, color='green', alpha=0.5) Z = hierarchy.linkage(dist, method='average') plt.subplot(224) hierarchy.dendrogram(Z, labels=name_docs, color_threshold=.25, leaf_font_size=8, count_sort=True) #plt.show() print(' 3    U ') for i, row in enumerate(U): print(c[i], row[0:3]) res1=-1*U[:,0:1]; res2=-1*U[:,1:2];res3=-1*U[:,2:3] data_word_xyz=[] for i in range(0,len(c)): data_word_xyz.append([res1[i][0],res2[i][0],res3[i][0]]) plt.figure() plt.subplot(221) dist = pdist(data_word_xyz, 'euclidean')#    ( ) plt.hist(dist, 500, color='green', alpha=0.5)#   Z = hierarchy.linkage(dist, method='average')#   plt.subplot(222) hierarchy.dendrogram(Z, labels=c, color_threshold=.25, leaf_font_size=8, count_sort=True,orientation='right') print(' 3    Vt ') rows, cols = Vt.shape for j in range(0,cols): for i in range(0,rows): Vt[i,j]=round(Vt[i,j],4) print(-1*Vt[0:3, :]) res3=(-1*Vt[0:1, :]);res4=(-1*Vt[1:2, :]);res5=(-1*Vt[2:3, :]) data_docs_xyz=[];name_docs_xyz=[] for i in range(0,len(docs)): name_docs_xyz.append(str(i)) data_docs_xyz.append([res3[0][i],res4[0][i],res5[0][i]]) plt.subplot(223) dist = pdist(data_docs_xyz, 'euclidean') plt.hist(dist, 500, color='green', alpha=0.5) Z = hierarchy.linkage(dist, method='average') plt.subplot(224) hierarchy.dendrogram(Z, labels=name_docs_xyz, color_threshold=.25, leaf_font_size=8, count_sort=True) plt.show() 

プログラムの結果
直交行列Uワードの最初の2列
wikileaks [-0.0741 0.0991]
逮捕[-0.023 0.0592]
イギリス[-0.023 0.0592]
手渡し[-0.0582 -0.5008]
ノーベルの[-0.0582 -0.5008]
創設者[-0.0804 0.1143]
警察官[-0.0337 0.0846]
prem [-0.0582 -0.5008]
prot [-0.5954 0.0695]
国[-0.3261 -0.169]
裁判所[-0.3965 0.1488]
アメリカ合衆国[-0.5954 0.0695]
式典[-0.055 -0.3875]
Vtドキュメントの直交行列の最初の2行
[[0.0513 0.6952 0.1338 0.045 0.0596 0.2102 0.6644 0.0426 0.0566]
[-0.1038 -0.1495 0.5166 -0.0913 0.5742 -0.1371 0.0155 -0.0987 0.5773]]
Uワードの直交行列の最初の3列
wikileaks [-0.0741 0.0991 -0.4372]
逮捕[-0.023 0.0592 -0.3241]
イギリス[-0.023 0.0592 -0.3241]
手渡し[-0.0582 -0.5008 -0.1117]
ノーベルの[-0.0582 -0.5008 -0.1117]
創設者[-0.0804 0.1143 -0.5185]
警察官[-0.0337 0.0846 -0.4596]
prem [-0.0582 -0.5008 -0.1117]
prot [-0.5954 0.0695 0.1414]
国[-0.3261 -0.169 0.0815]
裁判所[-0.3965 0.1488 -0.1678]
アメリカ合衆国[-0.5954 0.0695 0.1414]
式典[-0.055 -0.3875 -0.0802]
Vtドキュメントの直交行列の最初の3行
[[0.0513 0.6952 0.1338 0.045 0.0596 0.2102 0.6644 0.0426 0.0566]
[-0.1038 -0.1495 0.5166 -0.0913 0.5742 -0.1371 0.0155 -0.0987 0.5773]
[0.5299 -0.0625 0.0797 0.4675 0.1314 0.3714 -0.1979 0.5271 0.1347]]