self.d = 0.

self.dictionary = dictionary # type: Dictionary

self.frequencies = dictionary.frequencies

self.prototypes = prototypes # type: int

self.dim = dim # type: int

self.n_words = N = len(self.frequencies)

nodes = build_huffman_tree(self.frequencies)

outputs = convert_huffman_tree(nodes, N)

max_length = max(len(x.code) for x in outputs)

self.path = np.zeros((N, max_length), dtype=np.int32)

self.code = np.zeros((N, max_length), dtype=np.int8)

for n, output in enumerate(outputs):

self.code[n] = -1

for i, (c, p) in enumerate(zip(output.code, output.path)):

self.code[n, i] = c

self.path[n, i] = p

def rand_arr(shape, nrm, dtype):

return (np.array(np.random.rand(*shape), dtype=dtype) - 0.5) * nrm

self.In = rand_arr((N, prototypes, dim), 1. / dim, np.float32)

self.Out = rand_arr((N, dim), 1. / dim, np.float32)

self.counts = np.zeros((N, prototypes), np.float32)

@property

def InNorms(self):

if not hasattr(self, '_InNorms'):

self._InNorms = norm(self.In, axis=2)

return self._InNorms

def sense_neighbors(self, word, sense, max_neighbors=10,

min_closeness=None, min_count=1):

""" Nearest neighbors of given sense of the word.

:arg word: word (a string)

:arg sense: integer sense id (starting from 0)

:arg max_neighbors: max number of neighbors returned

:arg min_count: min count of returned neighbors

:arg min_closeness: min closeness of returned neighbors

:return: A list of triples (word, sense, closeness)

"""

if not self.is_valid_sense_vector(word, sense):

return []

word_id = self.dictionary.word2id[word]

s_v = self.In[word_id, sense] / self.InNorms[word_id, sense]

sim_matrix = np.dot(self.In, s_v) / self.InNorms

sim_matrix[np.isnan(sim_matrix)] = -np.inf

most_similar = []

while True:

idx = sim_matrix.argmax()

w_id, s = idx // self.prototypes, idx % self.prototypes

sim = sim_matrix[w_id, s]

sim_matrix[w_id, s] = -np.inf

if ((w_id, s) != (word_id, sense) and

self.counts[w_id, s] >= min_count):

if min_closeness is not None and sim < min_closeness:

break

most_similar.append((self.dictionary.id2word[w_id], s, sim))

if max_neighbors is not None and len(most_similar) >= max_neighbors:

break

return most_similar

def is_valid_sense_vector(self, word, sense):

word_id = self.dictionary.word2id[word]

s_v, s_v_norm = self.In[word_id, sense], self.InNorms[word_id, sense]

return not (np.allclose(s_v, 0) or np.allclose(s_v_norm, 0))

def disambiguate(self, word, context, min_prob=1e-3):

""" Return an array of probabilities for each sense of word in context.

"""

word_idx = self.dictionary.word2id[word]

z = expected_pi(self, word_idx)

z[z < min_prob] = 0

z = np.log(z)

return z

def sense_vector(self, word, sense, normalized=False):

word_id = self.dictionary.word2id[word]

v = self.In[word_id, sense]

if normalized:

nv = self.InNorms[word_id, sense]

if not np.isclose(nv, 0):

v = v / nv

return v

@classmethod

def load(cls, input):

return joblib.load(input)

def save(self, output):

joblib.dump(self, output)

class HierarchicalSoftmaxNode(object):

num_nodes = 0

def __init__(self, parent=-1, branch=False):

HierarchicalSoftmaxNode.num_nodes += 1

self.num = HierarchicalSoftmaxNode.num_nodes

self.parent = parent

self.branch = branch

def is_root(self):

return self.parent == -1

def __repr__(self):

return '<HierarchicalSoftmaxNode {} {}>'.format(

self.parent, self.branch)

def __gt__(self, other):

return self.num > other.num

class HierarchicalOutput(object):

def __init__(self, code, path):

self.code = code

self.path = path

def __repr__(self):

return '<HierarchicalOutput {} {}>'.format(self.code, self.path)

def softmax_path(nodes, N, idx):

while True:

node = nodes[idx]

if node.is_root():

break

assert node.parent >= N

yield node.parent - N, node.branch

idx = node.parent

def build_huffman_tree(freqs):

nodes = [HierarchicalSoftmaxNode() for _ in freqs]

heap = list(zip(freqs, nodes))

heapq.heapify(heap)

def pop_initialize(parent, branch):

freq, node = heapq.heappop(heap)

node.parent = parent

node.branch = branch

return freq

idx = len(nodes) - 1

while len(heap) > 1:

idx += 1

node = HierarchicalSoftmaxNode()

nodes.append(node)

freq = pop_initialize(idx, True) + pop_initialize(idx, False)

heapq.heappush(heap, (freq, node))

assert len(heap) == 1

return nodes

def convert_huffman_tree(nodes, N):

outputs = []

for idx in range(N):

code = []

path = []

for n, branch in softmax_path(nodes, N, idx):

code.append(branch)

path.append(n)

outputs.append(HierarchicalOutput(code, path))

return outputs

def expected_pi(vm, w_idx):

pi = np.zeros(vm.prototypes, dtype=np.float64)

r = 1.

ts = vm.counts[w_idx, :].sum()

for k in range(vm.prototypes - 1):

ts = max(ts - vm.counts[w_idx, k], 0.)

a = 1. + vm.counts[w_idx, k] - vm.d

b = vm.alpha + k * vm.d + ts

pi[k] = (a / (a + b)) * r

r = max(r - pi[k], 0.)

pi[-1] = r

return pi

Листинг скрипта для конвертации модели

#!/usr/bin/env python3

import argparse

import json

import os.path

from adascript import Dictionary, VectorModel

def main():

parser = argparse.ArgumentParser()

parser.add_argument('in_directory', help='Directory with model files in JSON format')

parser.add_argument('out_file', help='File to save model to')

parser.add_argument('add_name', help='Name assigned to JSON files')

args = parser.parse_args()

with open(os.path.join(args.in_directory, args.add_name + 'vm.json'), 'rt') as f:

vm_data = json.load(f)

with open(os.path.join(args.in_directory, args.add_name + 'id2word.json'), 'rt') as f:

id2word_data = json.load(f)

assert len(id2word_data) == len(vm_data['frequencies'])

dictionary = Dictionary(list(zip(id2word_data, vm_data['frequencies'])),

preserve_indices=True)

model = VectorModel(dictionary, dim=len(vm_data['Out'][0]),

prototypes=len(vm_data['In'][0]),

alpha=vm_data['alpha'])

model.counts[:] = vm_data['counts']

model.Out[:] = vm_data['Out']

model.In[:] = vm_data['In']

model.path[:] = vm_data['path']

model.path -= 1

model.code[:] = vm_data['code']

model.save(args.out_file)

if __name__ == '__main__':

main()

Примеры нахождения ближайших соседей слов

Ближайшие соседи слова “воздушный”:

Ближайшие соседи слова “сушка”. Особенное внимание стоит обратить на последний вариант, где слово употребляется в качестве сленгового обозначения самолетов Сухого:

Листинг скрипта для соревнования RUSSE-2018

#!/usr/bin/env python3

# -*- coding: UTF-8 -*-

from __future__ import print_function

import re

import sys

import tqdm

import os.path

import adagram

import argparse

import warnings

import numpy as np

import pandas as pd

from pymystem3 import Mystem

from sklearn.metrics import adjusted_rand_score

mystem = Mystem()

def disambiguate(model, word, context):

word, = lemmatized_context(word)

try:

probs = model.disambiguate(word, lemmatized_context(context))

except KeyError:

# В случае если слова нет в словаре

return 1

return 1 + probs.argmax()

def lemmatized_context(s):

# Используем лемматизацию т.к модель была натренирована с лемматизированными словами

return [w.lower() for w in mystem.lemmatize(s) if re.match('[\w\-]+$', w)]

def main():

parser = argparse.ArgumentParser(description=__doc__)

arg = parser.add_argument

arg('input', help='Path to input file with contexts')

arg('--output', help='Path to output file with predictions')

arg('--model', help='Path to AdaGram model')

arg('--ari-per-word', help='show ARI per-word')

args = parser.parse_args()

df = pd.read_csv(args.input, sep='\t')

if not os.path.exists(args.model):

print('Error: can not find model at {}'.format(args.model),

file=sys.stderr)

print(__doc__, file=sys.stderr)

sys.exit(1)

print('Loading AdaGram model')

model = adagram.VectorModel.load(args.model)

print('done')

with warnings.catch_warnings():

warnings.simplefilter('ignore', RuntimeWarning)

df['predict_sense_id'] = [

disambiguate(model, word, context)

for word, context in tqdm.tqdm(zip(df['word'], df['context']),

total=len(df))]

if df['gold_sense_id'].any():

per_word = df.groupby('word').aggregate(

lambda f: adjusted_rand_score(

f['gold_sense_id'], f['predict_sense_id']))

per_word_ari = per_word['predict_sense_id']

if args.ari_per_word:

for word, ari in zip(per_word.index, per_word_ari):

print('{:<20} {:+.4f}'.format(word, ari))

print('Mean word ARI: {:.4f}'.format(np.mean(per_word_ari)))

if args.output:

print('Saving predictions to {}'.format(args.output))

df.to_csv(args.output, sep='\t', index=None)

if __name__ == '__main__':

main()

Сравнение качества моделей на наборе wiki-wiki

№	Команда	ARI
1	jamstic	0.9625
2	akutuzov	0.7096
3	ezhick179	0.6586
4	akapustin	0.6459
5	aby2s	0.5889
6	bokan	0.5530
7	eugenys	0.4377
8	mikhail	0.4109
9	fogside	0.3958
10	AdaGram	0.3480

Сравнение качества моделей на наборе bts-rnc

№	Команда	ARI
1	jamstic	0.3384
2	ezhick179	0.2818
3	joystick	0.2579
4	Timon	0.2434
5	akutuzov	0.2415
6	AdaGram	0.2267
7	thebestDLSpec	0.2227
8	fogside	0.2154
9	aby2s	0.2102
10	bokan	0.1515

Сравнение качества моделей на наборе active-dict

№	Команда	ARI
1	jamstic	0.2477
2	ezhick179	0.2270
3	AdaGram	0.2227
4	Timon	0.2222
5	thebestDLSpec	0.2194
6	akutuzov	0.2144
7	aby2s	0.1985
8	joystick	0.1939
9	ostruyanskiy	0.1403
10	akapustin	0.1183

Листинг предобработки текста

mystem = Mystem()

def handle_emojis(tweet):

# Smile -- :), : ), :-), (:, ( :, (-:, :')

tweet = re.sub(r'(:\s?\)|:-\)|\(\s?:|\(-:|:\'\))', ' позитивная эмоция ', tweet)

# Laugh -- :D, : D, :-D, xD, x-D, XD, X-D

tweet = re.sub(r'(:\s?d|:-d|x-?d|X-?d)', ' позитивная эмоция ', tweet)

# Love -- <3, :*

tweet = re.sub(r'(<3|:\*)', ' позитивная эмоция ', tweet)

# Wink -- ;-), ;), ;-D, ;D, (;, (-;

tweet = re.sub(r'(;-?\)|;-?d|\(-?;)', ' позитивная эмоция ', tweet)

# Sad -- :-(, : (, :(, ):, )-:

tweet = re.sub(r'(:\s?\(|:-\(|\)\s?:|\)-:)', ' негативная эмоция ', tweet)

# Cry -- :,(, :'(, :"(

tweet = re.sub(r'(:,\(|:\'\(|:"\()', ' негативная эмоция ', tweet)

return tweet

def preprocess_word(word):

word = word.strip('\'"?!,.():;')

word = re.sub(r'(.)\1+', r'\1\1', word)

word = re.sub(r'(-|\')', '', word)

return word

def is_valid_word(word):

return (re.search(r'^[a-zA-Zа-яА-Я][a-zа-я0-9A-ZА-Я\._]*$', word) is not None)

def preprocess_tweet(stemmer, tweet):

processed_tweet = []

tweet = tweet.lower()

tweet = re.sub(r'((www\.[\S]+)|(https?://[\S]+))', ' url ', tweet)

tweet = re.sub(r'@[\S]+', 'упоминание пользователя', tweet)

tweet = re.sub(r'#(\S+)', r' \1 ', tweet)

tweet = re.sub(r'\brt\b', '', tweet)

tweet = re.sub(r'\.{2,}', ' ', tweet)

tweet = tweet.strip(' "\'')

tweet = handle_emojis(tweet)

tweet = re.sub(r'\s+', ' ', tweet)

words = tweet.split()

for word in words:

word = preprocess_word(word)

if is_valid_word(word):

word = stemmer.lemmatize(word)[0]

processed_tweet.append(word)

return ' '.join(processed_tweet)

df['stemmed'] = df['text'].apply(lambda x: preprocess_tweet(mystem, x))

df.reset_index(inplace=True)

Листинг скрипта обучения Word2Vec

#!/usr/bin/env python3

# -*- coding: UTF-8 -*-

import os

import re

import sys

import gensim

import numpy as np

import pandas as pd

class MySentences(object):

def __init__(self, path, filename):

self.filename = filename

self.path = path

def __iter__(self):

for line in open(os.path.join(self.path, self.filename), encoding='utf-8'):

yield line.split()

sentences = MySentences('data/','tokenized_taiga_all.txt') # a memory-friendly iterator

model = gensim.models.Word2Vec(sentences=sentences, size=300, window=5, min_count=50,

workers=16, alpha=0.2, sg=1, hs=1)

model.save('trained_models/w2v_taiga/w2v_taiga_w5_a02_minf50_dim300')

Листинг векторизации с помощью предобученной модели

def vectorize_with_pos(model, text, stopword_set):

text = text.replace(' ', '%20').lower()

adr = urllib.parse.quote("lindat.mff.cuni.cz/services/udpipe/api/process?model=russian-taiga-ud-2.3-181115&tokenizer&tagger&data=" + text)

vectorized = []

try:

with urllib.request.urlopen('https://' + adr) as url:

data = parse(json.loads(url.read().decode('utf-8'))['result'])

except:

data = 0

if data != 0:

for sentence in data:

for word in sentence:

if word not in stopword_set:

if word['upostag'] not in ['PUNCT', 'SYM', 'X']:

try:

vectorized.append(model[word['lemma'] + '_' + word['upostag']])

except KeyError:

pass

return np.mean(vectorized, axis=0)

df['vectorized__rusvec_w2v'] = df['stemmed'].apply(lambda x: vectorize_with_pos(wv, x, russian_stopwords))

Размещено на Allbest.ru

...