Популярность текста

29. Hocza A. Noun Phrase Recognition with Tree Patterns //Acta Cybern. - 2004. - Т. 16. - №. 4. - С. 611-623.

30. Kudinov M. S., Romanenko A. A., Piontkovskaja I. I. Conditional random field in segmentation and noun phrase inclination tasks for Russian //Annual International Conference” Dialogue”, Computational Linguistics and Intellectual Technologies. - 2014. - Т. 13. - С. 220.

31. Long A. Calculating Reading Level. Tameri Guide for Writers.

32. McCallum D. R., Peterson J. L. Computer-based readability indexes Proceedings of the ACM'82 Conference. - ACM, 1982. - С. 44-48.

33. McLaughlin H. SMOG grading - a new readability formula // J. of Reading. 1969. - N 22. - P. 639-646.

34. Miles T.H. The fog index: a practical readabilityscale / In Critical Thinking and Writingfor Science and Technology. HarcourtBrace Jovanovich. - 1990. - P. 280-284. -http://www.as.wvu.edu/~tmiles/fog.html.

35. Nivre J., Boguslavsky I. M., Iomdin L. L. Parsing the SynTagRus treebank of Russian //Proceedings of the 22nd International Conference on Computational Linguistics-Volume 1. - Association for Computational Linguistics, 2008. - С. 641-648.

36. Pitler E. and Nenkova A. (2008), "Revisiting readability: A unified framework for predicting text quality." Proceedings of the conference on empirical methods in natural language processing. Association for Computational Linguistics.

37. Powers R.D., Sumner W.A., Kearl B.E. A recalculation of 4 readability formulae // Educational Psychology, University of Birmingham. - 1993. - N 49. - P. 99-105.

38. Raygor A. L. The Raygor readability estimate: A quick and easy way to determine difficulty //Reading: Theory, research, and practice. - 1977. - С. 259-263.

39. Sang E. F. Noun phrase recognition by system combination //Proceedings of the 1st North American chapter of the Association for Computational Linguistics conference. - Association for Computational Linguistics, 2000. - С. 50-55.

40. Segalovich I. A fast morphological algorithm with unknown word guessing induced by a dictionary for a web search engine //MLMTA. - 2003. - С. 273-280.

Приложение

from pymystem3 import Mystem

mystem = Mystem()

import json

from bs4 import BeautifulSoup

from urllib.requestimport Request, urlopen

import urllib

import re

import csv

from nltk.tokenizeimport sent_tokenize

from nltk.tokenizeimport word_tokenize

import pandas

import csv

from pandas import read_csv

import nltk

import numpyas np

from pymystem3 import Mystem

import pickle

import matplotlib.pyplotas plt

import numpyas np

from sklearnimport linear_model

from sklearn.metricsimport mean_squared_error, r2_score

from sklearn.model_selectionimport train_test_split

from sklearn.metricsimport recall_score

from sklearn.metricsimport precision_score

from sklearn.metricsimport f1_score

from sklearn.model_selectionimport cross_val_score

from sklearn.naive_bayesimport GaussianNB

from sklearn.utilsimport shuffle

from sklearn.svmimport SVC

from sklearn.naive_bayesimport MultinomialNB

from sklearn.treeimport DecisionTreeClassifier

from sklearn.ensembleimport AdaBoostClassifier

from sklearn.ensembleimport RandomForestClassifier

from sklearn.linear_modelimport LogisticRegression

def change_page(url): #собираетссылкистраничек-меню

next = []

next.append(url)

for a in range(200):

next.append(re.sub('#result', '', url) + '&p=' + str(a) + '#result')

print ('next', next)

return(next)

def get_html(site): #собираетссылкистраничексфанфикамисоднойстранички-меню

fanfics = []

hdr = {'User-Agent': 'FBSearchBot'}

req = Request(site,headers=hdr)

page = urlopen(req)

soup = BeautifulSoup(page, 'lxml')

content = soup.find('section', class_ = 'content-section')

rows = content.find_all('h3')

for link in soup.find_all('a'):

textlink = link.get('href')

try:

fanfics.append('https://ficbook.net/readfic/'+re.findall('/readfic/(\d+)', textlink)[0])

except:

pass

print('fanfics')

return fanfics

def get_fanfics(next): # перелистываетстраничкиисобираетскаждойстранички-менюссылкинафанфы

fanfics = []

for iin next:

a = get_html(i)

for fanfin a:

fanfics.append(fanf)

print('I work')

return fanfics

fanfics_links = [

'https://ficbook.net/find?title=&fandom_filter=originals&fandom_group_id=1&sizes%5B%5D=1&pages_min=&pages_max=&ratings%5B%5D=5&ratings%5B%5D=6&ratings%5B%5D=7&ratings%5B%5D=8&ratings%5B%5D=9&transl=&status=2&directions%5B%5D=1&directions%5B%5D=2&directions%5B%5D=3&directions%5B%5D=4&directions%5B%5D=7&directions%5B%5D=6&directions%5B%5D=5&likes_min=&likes_max=&sort=marks&rnd=633394924&find=%D0%9D%D0%B0%D0%B9%D1%82%D0%B8%21#result',

'https://ficbook.net/find?title=&fandom_filter=originals&fandom_group_id=1&sizes%5B%5D=2&pages_min=&pages_max=&ratings%5B%5D=5&ratings%5B%5D=6&ratings%5B%5D=7&ratings%5B%5D=8&ratings%5B%5D=9&transl=&status=2&directions%5B%5D=1&directions%5B%5D=2&directions%5B%5D=3&directions%5B%5D=4&directions%5B%5D=7&directions%5B%5D=6&directions%5B%5D=5&likes_min=&likes_max=&sort=marks&rnd=296436895&find=%D0%9D%D0%B0%D0%B9%D1%82%D0%B8%21#result',

'https://ficbook.net/find?title=&fandom_filter=originals&fandom_group_id=1&sizes%5B%5D=3&pages_min=&pages_max=&ratings%5B%5D=5&ratings%5B%5D=6&ratings%5B%5D=7&ratings%5B%5D=8&ratings%5B%5D=9&transl=&status=2&directions%5B%5D=1&directions%5B%5D=2&directions%5B%5D=3&directions%5B%5D=4&directions%5B%5D=7&directions%5B%5D=6&directions%5B%5D=5&likes_min=&likes_max=&sort=marks&rnd=1102828819&find=%D0%9D%D0%B0%D0%B9%D1%82%D0%B8%21#result']

all_fanfics = []

for link in fanfics_links:

fanf = get_fanfics(change_page(link))

for fan in fanf:

print(fan)

all_fanfics.append(fan)

def get_data(fanfics): # собираетданныесоднойстраницысфанфомизаписываетв csv

texts = []

marks = []

genres = []

dates = []

csvfile = open('new_data2.csv', 'w', encoding='utf-8')

spamwriter = csv.writer(csvfile, delimiter=';', quoting=csv.QUOTE_MINIMAL)

spamwriter.writerow(['marks', 'texts', 'collections', 'genres', 'date'])

for fanfin fanfics:

print(fanf)

result = []

hdr = {'User-Agent': 'FBSearchBot'}

try:

req = Request(fanf, headers=hdr)

page = urlopen(req)

soup = BeautifulSoup(page, 'lxml')

except urllib.error.HTTPError:

continue

mark = soup.find('span', class_='js-marks-plus')

if mark != None:

result.append(int(mark.text))

marks.append(int(mark.text))

text = None

text1 = soup.find('div', class_='jsPartTextpart_texturlizepublic_beta')

text2 = soup.find('div', class_='jsPartTextpart_texturlizepublic_beta_disabled')

if text1 != None:

text = text1.text

if text2 != None:

text = text2.text

result.append(text)

texts.append(text)

collection = soup.find('strong', class_='jsCollectionsCounter')

if collection != None:

result.append(int(collection.text))

else:

result.append(None)

genre = soup.find('div', class_='col-md-8')

if genre != None:

genre = genre.text

genre = re.findall('Жанры:(.+)Предупреждения:', genre.replace('\n', ''))

if len(genre) != 0:

result.append(genre[0])

genres.append(genre[0])

else:

result.append(None)

genres.append(None)

else:

result.append(None)

date = soup.find('span', class_='part-date')

if date != None:

date = date.text

result.append(date)

dates.append(date)

spamwriter.writerow(result)

csvfile.close()

return (marks)

return (text)

return (collections)

return (dates)

return (genres)

get_data(all_fanfics)

data = pandas.read_csv('mydata.csv', error_bad_lines=False, delimiter=';', quoting=csv.QUOTE_MINIMAL, encoding='utf-8')

data = data[data.marks.notnull() &data.texts.notnull()]

data.reset_index(inplace=True)

data.drop('index', axis=1, inplace=True)

def split_tokenized_text(text):

sent_tokenize_list=[]

s = re.split(';\r\n \r\n\r\n', text)

try:

for iin s:

st = s[1]

sent_tokenize_list = nltk.sent_tokenize(st)

split_sents=[]

lemmas = []

split_sents = list(map(lambda x: re.findall('\w+', x), sent_tokenize_list))

return split_sents

except IndexError:

return np.nan

data.texts = data.texts.astype(str)

data['split_tokenized_texts'] = data.texts.apply(split_tokenized_text)

mystem = Mystem()

def lemmatize_text(split_tokenized_text):

lemmatized_text_info = []

for sent in split_tokenized_text:

lemmatized_text_info.append([])

for word in sent:

a = mystem.analyze(word)[0]

lemmatized_text_info[-1].append(a)

return lemmatized_text_info

data = data[data.split_tokenized_texts.notnull()]

data['lemmatized_texts'] = data.split_tokenized_texts.apply(lemmatize_text)

data.to_pickle('samsonova.pickle')

with open('samsonova.pickle', 'rb') as file:

data = pickle.load(file)

def feature1(text): # 1. Average number of characters per word

text = [ifor iin re.split('\W+', text) if i != '']

characters = 0

for word in text:

characters += len(word)

feature1 = characters / len(text)

return (feature1)

f1 = []

for text in data.texts:

f1.append(feature1(text))

data['feature1'] = f1

def feature2(split_tokenized_text): # 2. Average number of words per sentence

words = 0

for sent in split_tokenized_text:

words += len(sent)

feature2 = words / len(split_tokenized_text)

return (feature2)

f2 = []

for text in data.split_tokenized_texts:

f2.append(feature2(text))

data['feature2'] = f2

def feature3(split_tokenized_text): #3. Maximum number of words per sentence

length_sent=[]

for sent in split_tokenized_text:

length_sent.append(len(sent))

return(max(length_sent))

f3 = []

for text in data.split_tokenized_texts:

f3.append(feature3(text))

data['feature3'] = f3

def feature4(text): #4. Number of words per text

text = [ifor iin re.split('\W+', text) if i != '']

return(len(text))

f4 = []

for text in data.texts:

f4.append(feature4(text))

data['feature4'] = f4

def feature_5(lemmatized_text): # 9 Number of pronouns per sentence

pronouns = 0

for sent in lemmatized_text:

for word in sent:

try:

for a in word['analysis']:

results = re.findall('SPRO|APRO|ADVPRO', a['gr'])

if len(results) >0:

pronouns += 1

except:

continue

feature_5 = pronouns / len(lemmatized_text)

return (feature_5)

f5 = []

for text in data.lemmatized_texts:

f5.append(feature_5(text))

data['feature5'] = f5

def feature_6(lemmatized_text): # number of words overlap between pairs of adjacent sentences

feature_6 = 0

for sent in range(1, len(lemmatized_text)):

comp_sent = []

new_sent = lemmatized_text[sent - 1] + lemmatized_text[sent]

for word in new_sent:

try:

for a in word['analysis']:

comp_sent.append(a['lex'])

except:

continue

i = len(comp_sent) - len(set(comp_sent))

feature_6 += i

return (feature_6)

f6 = []

for text in data.lemmatized_texts:

f6.append(feature_6(text))

data['feature6'] = f6

def feature_7(lemmatized_text): # number of nouns and pronouns overlap between pairs of adjacent sentences

feature_7 = 0

for sent in range(1, len(lemmatized_text)):

comp_sent = []

new_sent = lemmatized_text[sent - 1] + lemmatized_text[sent]

for word in new_sent:

try:

for a in word['analysis']:

if len(re.findall('S|ADVPRO|APRO|SPRO', a['gr'])) >0:

comp_sent.append(a['lex'])

except:

continue

i = len(comp_sent) - len(set(comp_sent))

feature_7 += i

return (feature_7)

f7 = []

for text in data.lemmatized_texts:

f7.append(feature_7(text))

data['feature7'] = f7

def feature_8(text): #количествослоговвтексте

for letter in text:

a=0

if letter in 'ауеоэёяиюы':

a+=1

return a

f8 = []

for text in data.texts:

f8.append(feature4(text))

data['feature8'] = f8

def feature9(split_tokenized_text): # 2.среднееколичестводлинныхслов

words = 0

for sent in split_tokenized_text:

for word in sent:

slogi = 0

for letter in word:

if letter in 'ауеоэёяиюы':

slogi += 1

if slogi>3:

words += 1

feature9 = words / len(split_tokenized_text)

return (feature9)

f9 = []

for text in data.split_tokenized_texts:

f9.append(feature2(text))

data['feature9'] = f9

def feature10(split_tokenized_text): # 2.среднееколичестводлинныхслов

words = 0

for sent in split_tokenized_text:

for word in sent:

slogi = 0

for letter in word:

if letter in 'ауеоэёяиюы':

slogi += 1

if slogi == 1:

words += 1

feature10 = words / len(split_tokenized_text)

return (feature10)

f10 = []

for text in data.split_tokenized_texts:

f10.append(feature2(text))

data['feature10'] = f10

train, test = train_test_split(data, test_size=0.3)

X_train = train[['feature1','feature2','feature3','feature4', 'feature5', 'feature6', 'feature7','feature8', 'feature9', 'feature10' ]]

X_test = test[['feature1','feature2','feature3','feature4', 'feature5', 'feature6', 'feature7','feature8', 'feature9', 'feature10']]

print(X_test)

y_train = train['marks']

y_test = test['marks']

print(y_test)

regr = linear_model.LinearRegression()

regr.fit(X_train, y_train)

y_pred = regr.predict(X_test)

print('Coefficients: \n', regr.coef_)

print("Mean squared error: %.2f"

% mean_squared_error(y_test, y_pred))

print('Variance score: %.2f' % r2_score(y_test, y_pred))

plt.scatter(X_test['feature1'], y_test, color='black')

plt.plot(X_test['feature1'], y_pred, color='blue', linewidth=3)

plt.xticks(())

plt.yticks(())

plt.show()

regr_1 = DecisionTreeRegressor(max_depth=2)

regr_2 = DecisionTreeRegressor(max_depth=5)

regr_1.fit(X_train, y_train)

regr_2.fit(X_train, y_train)

y_1 = regr_1.predict(X_test)

y_2 = regr_2.predict(X_test)

print("Mean squared error regr_1: %.2f"

% mean_squared_error(y_test, y_1))

print("Mean squared error regr_2: %.2f"

% mean_squared_error(y_test, y_2))

print('Variance score regr_1: %.2f' % r2_score(y_test, y_1))

print('Variance score regr_2: %.2f' % r2_score(y_test, y_2))

a = pd.qcut(data['marks'], 4, labels = False)

data['label'] = a.sort_index()

data = shuffle(data)

train_c, test_c = train_test_split(data, test_size=0.2)

X_train_c = train_c[['feature1','feature2','feature3', 'feature4', 'feature5', 'feature6', 'feature7']]

X_test_c = test_c[['feature1','feature2','feature3', 'feature4', 'feature5', 'feature6', 'feature7']]

y_train_c = train_c['label']

y_test_c = test_c['label']

data = shuffle(data)

n_f = ['feature1','feature2','feature3', 'feature4', 'feature5', 'feature6', 'feature7']

print('SVC')

clf = SVC()

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='f1_macro')

print('f1', scores.mean())

clf = SVC()

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='recall_macro')

print('recall', scores.mean())

clf = SVC()

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='precision_macro')

print('precision', scores.mean())

clf = GaussianNB()

print('GaussianNB')

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='f1_macro')

print('f1', scores.mean())

clf = GaussianNB()

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='recall_macro')

print('recall', scores.mean())

clf = GaussianNB()

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='precision_macro')

print('precision', scores.mean())

clf = MultinomialNB()

print('MultinomialNB')

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='f1_macro')

print('f1', scores.mean())

clf = MultinomialNB()

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='recall_macro')

print('recall', scores.mean())

clf = MultinomialNB()

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='precision_macro')

print('precision', scores.mean())

clf = AdaBoostClassifier(

DecisionTreeClassifier(max_depth=5),

n_estimators=1000,

learning_rate=1)

print('DecisionTreeClassifier')

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='f1_macro')

print('f1', scores.mean())

clf = AdaBoostClassifier(

DecisionTreeClassifier(max_depth=5),

n_estimators=1000,

learning_rate=1)

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='recall_macro')

print('recall', scores.mean())

clf = AdaBoostClassifier(

DecisionTreeClassifier(max_depth=5),

n_estimators=1000,

learning_rate=1)

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='precision_macro')

print('precision', scores.mean())

clf = RandomForestClassifier(n_estimators=100)

print('RandomForestClassifier')

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='f1_macro')

print('f1', scores.mean())

clf = RandomForestClassifier(n_estimators=100)

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='recall_macro')

print('recall', scores.mean())

clf = RandomForestClassifier(n_estimators=100)

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='precision_macro')

print('precision', scores.mean())

clf = LogisticRegression()

print('LogisticRegression')

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='f1_macro')

print('f1', scores.mean())

clf = LogisticRegression()

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='recall_macro')

print('recall', scores.mean())

clf = LogisticRegression()

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='precision_macro')

print('precision', scores.mean())

f = ['feature1', 'feature2', 'feature3', 'feature4', 'feature5', 'feature6', 'feature7', 'feature8', 'feature9',

'feature10']

for iin range(len(f)):

n_f = f[:i] + f[i + 1:]

clf = AdaBoostClassifier(

DecisionTreeClassifier(max_depth=5),

n_estimators=1000,

learning_rate=1)

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='f1_macro')

print('f1', scores.mean())

clf = AdaBoostClassifier(

DecisionTreeClassifier(max_depth=5),

n_estimators=1000,

learning_rate=1)

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='recall_macro')

print('recall', scores.mean())

clf = AdaBoostClassifier(

DecisionTreeClassifier(max_depth=5),

n_estimators=1000,

learning_rate=1)

scores = cross_val_score(clf, data[n_f], data['label'], cv=5, scoring='precision_macro')

print('precision', scores.mean())

Размещено на Allbest.ru

...