BigARTMツールを使用した主題モデリング

# In[1]: #!/usr/bin/env python # coding: utf-8 # In[2]: text=u'        \         \          \         ' # In[3]: import time start = time.time() import pymystem3 mystem = pymystem3 . Mystem ( ) z=text.split() lem="" for i in range(0,len(z)): lem =lem + " "+ mystem. lemmatize (z[i])[0] stop = time.time() print u",  lemmatize- %f   %i  "%(stop - start,len(z)) 

 #In [4]: start = time.time() import nltk from nltk.stem import SnowballStemmer stemmer = SnowballStemmer('russian') stem=[stemmer.stem(w) for w in text.split()] stem= ' '.join(stem) stop = time.time() print u",  stemmer NLTK- %f   %i  "%(stop - start,len(z)) 

 #In [5]: start = time.time() from Stemmer import Stemmer stemmer = Stemmer('russian') text = ' '.join( stemmer.stemWords( text.split() ) ) stop = time.time() print u",  Stemmer- %f   %i "%(stop - start,len(z)) 

 #In [6]: import nltk from nltk.corpus import brown stop_words= nltk.corpus.stopwords.words('russian') stop_word=" " for i in stop_words: stop_word= stop_word+" "+i print stop_word 

 #In [6]: #!/usr/bin/env python # -*- coding: utf-8 -* from __future__ import unicode_literals import nltk from nltk import word_tokenize from nltk.util import ngrams from collections import Counter text = "          \           \           " #In [7]: token = nltk.word_tokenize(text) bigrams = ngrams(token,2) trigrams = ngrams(token,3) fourgrams = ngrams(token,4) fivegrams = ngrams(token,5) #In [8]: for k1, k2 in Counter(bigrams): print (k1+"_"+k2) #In [9]: for k1, k2,k3 in Counter(trigrams): print (k1+"_"+k2+"_"+k3) #In [10]: for k1, k2,k3,k4 in Counter(fourgrams): print (k1+"_"+k2+"_"+k3+"_"+k4) #In [11]: for k1, k2,k3,k4,k5 in Counter(fivegrams): print (k1+"_"+k2+"_"+k3+"_"+k4+"_"+k5) 

 #In [12]: #!/usr/bin/env python # -*- coding: utf-8 -*- import matplotlib.pyplot as plt import codecs import os import nltk import numpy as np from nltk.corpus import brown stop_words= nltk.corpus.stopwords.words('russian') import pymystem3 mystem = pymystem3.Mystem() path='Texts_habrahabr' f=open('habrahabr.txt','a') x=[];y=[]; s=[] for i in range(1,len(os.listdir(path))+1): #      i filename=path+'/'+str(i)+".txt" text=" " with codecs.open(filename, encoding = 'UTF-8') as file_object:#     i-  for line in file_object: if len(line)!=0: text=text+" "+line word=nltk.word_tokenize(text)#   i-  word_ws=[w.lower() for w in word if w.isalpha() ]#    word_w=[w for w in word_ws if w not in stop_words ]#  lem = mystem . lemmatize ((" ").join(word_w))#  i -  lema=[w for w in lem if w.isalpha() and len(w)>1] freq=nltk.FreqDist(lema)#    i -    z=[]#      z=[(key+":"+str(val)) for key,val in freq.items() if val>1] #    :   f.write("|text" +" "+(" ").join(z).encode('utf-8')+'\n')#       |text c=[];d=[] for key,val in freq.items():#        i -  if val>1: c.append(val); d.append(key) a=[];b=[] for k in np.arange(0,len(c),1):#      i -  ind=c.index(max(c)); a.append(c[ind]) b.append(d[ind]); del c[ind]; del d[ind] x.append(i)#   y.append(len(a))#     a=a[0:10];b=b[0:10]# TOP-10   a   b  i -  y_pos = np.arange(1,len(a)+1,1)# TOP-10  performance =a plt.barh(y_pos, a) plt.yticks(y_pos, b) plt.xlabel(u' ') plt.title(u'    № %i'%i, size=12) plt.grid(True) plt.show() plt.title(u'   ', size=12) plt.xlabel(u' ', size=12) plt.ylabel(u' ', size=12) plt.bar(x,y, 1) plt.grid(True) plt.show() f.close() 

 #In [1]: #!/usr/bin/env python # -*- coding: utf-8 -* import artm #     batch batch_vectorizer = artm.BatchVectorizer(data_path='habrahabr.txt',#   " " data_format='vowpal_wabbit',#   target_folder='habrahabr', #     batch batch_size=10)#     batch 

 #In [2]: batch_vectorizer = artm.BatchVectorizer(data_path='habrahabr',data_format='batches') dictionary = artm.Dictionary(data_path='habrahabr')#     model = artm.ARTM(num_topics=10, num_document_passes=10,#10    dictionary=dictionary, scores=[artm.TopTokensScore(name='top_tokens_score')]) model.fit_offline(batch_vectorizer=batch_vectorizer, num_collection_passes=10)#10    top_tokens = model.score_tracker['top_tokens_score'] 

 #In [3]: for topic_name in model.topic_names: print (topic_name) for (token, weight) in zip(top_tokens.last_tokens[topic_name], top_tokens.last_weights[topic_name]): print token, '-', round(weight,3)