Processing Natural Language with Python and Friends

import numpy as np

from spacy.lang.en import English

nlp = English()

import nltk
print( nltk.corpus.gutenberg.fileids())

emma = nltk.corpus.gutenberg.raw('austen-emma.txt')
emma = emma.replace('\n',' ')
docEmma = nlp(emma)

# Process the text
doc = nlp("I like tree kangaroos and narwhals.")

# Select the first token
first_token = doc[0]

# Print the first token's text
print(first_token.text)

# Process the text
doc = nlp("I like tree kangaroos and narwhals.")

# A slice of the Doc for "tree kangaroos"
tree_kangaroos = doc[2:4]
print(tree_kangaroos.text)

# A slice of the Doc for "tree kangaroos and narwhals" (without the ".")
tree_kangaroos_and_narwhals = doc[2:6]
print(tree_kangaroos_and_narwhals.text)

# Process the text
doc = nlp("In 1990, more than 60% of people in East Asia were in extreme poverty. "
    "Now less than 4% are.")

# Iterate over the tokens in the doc
for token in doc:
    # Check if the token resembles a number
    if token.like_num:
        # Get the next token in the document
        next_token = doc[token.i + 1]
        # Check if the next token's text equals '%'
        if next_token.text == "%":
            print("Percentage found:", token.text)

#model package
#$ python -m spacy download en_core_web_sm

#load models
import spacy
nlp = spacy.load('en_core_web_sm')

doc = nlp("She ate the pizza")

#iterate over the tokens
for token in doc:
    #print the text and the predicted part-of-speech tag
    print(token.i, token.text, token.pos_)

#syntatic dependency
for token in doc:
    print(token.text, token.pos_, token.dep_, token.head.text)

#process a text
doc = nlp(u"Apple is looking at buying U.K. startup for $1 billion")

# Iterate over the predicted entities
for ent in doc.ents:
    # Print the entity text and its label
    print(ent.text, ent.label_)

for tok in doc:
    print( tok.text, tok.ent_type_, end=" ")

#common tags and labels
print( spacy.explain('GPE') )
print( spacy.explain('NNP') )
print( spacy.explain('dobj') )

import spacy
from spacy import displacy
nlp = spacy.load("en_core_web_sm")
doc1 = nlp("This is a sentence.")
doc2 = nlp("This is another sentence.")
displacy.render([doc1, doc2], style="dep")

import spacy
from spacy import displacy
nlp = spacy.load("en_core_web_sm")

options = {"compact": True, "bg": "#09a3d5","color": "white", "font": "Source Sans Pro"}

text = """In ancient Rome, some neighbors live in three adjacent houses. In the center is the house of Senex, who lives there with wife Domina, son Hero, and several slaves, including head slave Hysterium and the musical's main character Pseudolus."""
doc = nlp(text)
sentence_spans = list(doc.sents)
displacy.render(sentence_spans, style="dep", options=options, jupyter=True)

import spacy
from spacy import displacy

text = "When Sebastian Thrun started working on self-driving cars at Google in 2007, few people outside of the company took him seriously."

nlp = spacy.load("en_core_web_sm")
doc = nlp(text)
displacy.render(doc, style="ent")

colors = {"ORG": "linear-gradient(90deg, #aa9cfc, #fc9ce7)"}
options = {"ents": ["ORG"], "colors": colors}

displacy.render(doc, style="ent", options=options)

{
    "words": [
        {"text": "This", "tag": "DT"},
        {"text": "is", "tag": "VBZ"},
        {"text": "a", "tag": "DT"},
        {"text": "sentence", "tag": "NN"}
    ],
    "arcs": [
        {"start": 0, "end": 1, "label": "nsubj", "dir": "left"},
        {"start": 2, "end": 3, "label": "det", "dir": "left"},
        {"start": 1, "end": 3, "label": "attr", "dir": "right"}
    ]
}

ex = [{"text": "But Google is starting from behind.",
       "ents": [{"start": 4, "end": 10, "label": "ORG"}],
       "title": None}]
html = displacy.render(ex, style="ent", manual=True)

# Import the Matcher
from spacy.matcher import Matcher

text = "New iPhone X release date leaked as Apple reveals pre-orders by mistake"

# Initialize the matcher with the shared vocab
matcher = Matcher(nlp.vocab)

# Add the pattern to the matcher
pattern = [{'TEXT': 'iPhone'}, {'TEXT': 'X'}]
matcher.add('IPHONE_PATTERN', None, pattern)

# Process some text
doc = nlp("New iPhone X release date leaked")

# Call the matcher on the doc
matches = matcher(doc)

#match_id: hash value of the pattern name
#start: start index of matched span
#end: end index of matched span

# Iterate over the matches
for match_id, start, end in matches:
    # Get the matched span
    matched_span = doc[start:end]
    print(matched_span.text)

# Write a pattern that matches a form of "download" plus proper noun
pattern = [{"LEMMA": "download"}, {"POS": "PROPN"}]
# Write a pattern for adjective plus one or two nouns
pattern = [{"POS": ____}, {"POS": ____}, {"POS": ____, "OP": ____}]

nlp.vocab.length

#Hashes can't be reversed – that's why we need to provide the shared vocab
coffee_hash = nlp.vocab.strings['coffee']
print(coffee_hash)

# Raises an error if we haven't seen the string before
string = nlp.vocab.strings[3197928453018144401]

doc = nlp("I love coffee")
print('hash value:', nlp.vocab.strings['coffee'], doc.vocab.strings['coffee'])
print('string value:', nlp.vocab.strings[3197928453018144401])

#contains the context-independent information
doc = nlp("I love coffee")
lexeme = nlp.vocab['coffee']

# Print the lexical attributes
print(lexeme.text, lexeme.orth, lexeme.is_alpha)

# Create an nlp object
from spacy.lang.en import English
nlp = English()

# Import the Doc class
from spacy.tokens import Doc

# The words and spaces to create the doc from
words = ['Hello', 'world', '!']
spaces = [True, False, False]

# Create a doc manually
doc = Doc(nlp.vocab, words=words, spaces=spaces)
doc

# Import the Doc class
from spacy.tokens import Span

# Create a doc manually
doc = Doc(nlp.vocab, words=words, spaces=spaces)

# Create a span manually
span = Span(doc, 0, 2)

# Create a span with a label
span_with_label = Span(doc, 0, 2, label="GREETING")

# Add span to the doc.ents
doc.ents = [span_with_label]

# Print entities' text and labels
print([(ent.text, ent.label_) for ent in doc.ents])

# Load a larger model with vectors
nlp = spacy.load('en_core_web_lg')

# Compare two documents
doc1 = nlp("I like fast food")
doc2 = nlp("I like pizza")
print(doc1.similarity(doc2))

doc = nlp("I like pizza and pasta")
token1 = doc[2]
token2 = doc[4]
print(token1.similarity(token2))

nlp = spacy.load('en_core_web_sm')
doc1 = nlp("I")
print( doc1.vector.shape )

nlp = spacy.load('en_core_web_lg')
doc1 = nlp("I")
print( doc1.vector.shape )

nlp = spacy.load('en_core_web_sm')

doc1 = nlp("I")
doc2 = nlp("like")
doc3 = nlp("I like")
doc4 = nlp("I like pizza")

print( doc1.vector.shape, ' ', doc1.vector_norm )
print( doc2.vector.shape, ' ', doc2.vector_norm )
print( doc3.vector.shape, ' ', doc3.vector_norm )
print( doc4.vector.shape, ' ', doc4.vector_norm )

doc1[0].vector == doc3[0].vector

print( np.dot(doc1.vector, doc2.vector) )
print( np.linalg.norm(doc2.vector, ord=2) )
print( np.linalg.norm(doc1.vector, ord=2) == np.linalg.norm(doc2.vector, ord=2) )
print( doc2.vector_norm )

doc1 = nlp("pizza like I")
doc2 = nlp("I like pizza")
doc3 = nlp("pizza I like")
doc4 = nlp("like pizza I")
doc5 = nlp("I pizza like")
doc6 = nlp("like I pizza")

lDoc = [doc1, doc2, doc3, doc4, doc5, doc6]
result = np.zeros((6,6))
for i,doc1 in enumerate(lDoc):
    for j, doc2 in enumerate(lDoc):
        result[i,j] = doc1.similarity(doc2)

result

nlp = spacy.load('en_core_web_lg')

doc1 = nlp("apples oranges fruit")
print( doc1[0].vector_norm )
print( doc1[1].vector_norm )
print( doc1[2].vector_norm )

print( doc1[0].similarity(doc1[1]))
print( doc1[0].similarity(doc1[2]))

doc1 = nlp("apples")
doc2 = nlp("apples apples apples apples apples apples apples apples apples apples apples apples apples apples apples apples apples apples")

print(doc1.vector_norm)
print(doc2.vector_norm)
print(doc1.similarity(doc2))

doc1 = nlp("apples fruit")
doc2 = nlp("apples apples apples apples apples apples apples apples apples apples apples apples apples apples apples apples apples apples fruit")

print(doc1.vector_norm)
print(doc2.vector_norm)
print(doc1.similarity(doc2))

# Load a larger model with vectors
#nlp = spacy.load('en_core_web_lg')

doc = nlp("I have a banana")
# Access the word vector via the token.vector attribute
vector = doc[3].vector
print( type(vector) )
print( vector.shape)

vector[0:10]

#no universal definition for similarity 
doc1 = nlp("I like cats")
doc2 = nlp("I hate cats")

print(doc1.similarity(doc2))

import spacy
from spacy.matcher import Matcher

nlp = spacy.load("en_core_web_sm")
doc = nlp(
    "Twitch Prime, the perks program for Amazon Prime members offering free "
    "loot, games and other benefits, is ditching one of its best features: "
    "ad-free viewing. According to an email sent out to Amazon Prime members "
    "today, ad-free viewing will no longer be included as a part of Twitch "
    "Prime for new members, beginning on September 14. However, members with "
    "existing annual subscriptions will be able to continue to enjoy ad-free "
    "viewing until their subscription comes up for renewal. Those with "
    "monthly subscriptions will have access to ad-free viewing until October 15."
)

# Create the match patterns
pattern1 = [{"LOWER": "Amazon"}, {"IS_TITLE": True, "POS": "PROPN"}]
pattern2 = [{"LOWER": "ad-free"}, {"POS": "NOUN"}]

# Initialize the Matcher and add the patterns
matcher = Matcher(nlp.vocab)
matcher.add("PATTERN1", None, pattern1)
matcher.add("PATTERN2", None, pattern2)

# Iterate over the matches
for match_id, start, end in matcher(doc):
    # Print pattern string name and text of matched span
    print(doc.vocab.strings[match_id], doc[start:end].text)

import json
from spacy.lang.en import English

with open("exercises/countries.json") as f:
    COUNTRIES = json.loads(f.read())

nlp = English()
doc = nlp("Czech Republic may help Slovakia protect its airspace")

# Import the PhraseMatcher and initialize it
from spacy.matcher import PhraseMatcher

matcher = PhraseMatcher(nlp.vocab)

# Create pattern Doc objects and add them to the matcher
# This is the faster version of: [nlp(country) for country in COUNTRIES]
patterns = list(nlp.pipe(COUNTRIES))
matcher.add("COUNTRY", None, *patterns)

# Call the matcher on the test document and print the result
matches = matcher(doc)
print([doc[start:end] for match_id, start, end in matches])

#initialize the language, add the pipeline and load in the binary model weights
nlp = spacy.load("en_core_web_sm")

doc = nlp("This is a sentence.")

#list of pipeline component names
print(nlp.pipe_names)

#list of (name, component) tuples
print(nlp.pipeline)

def custom_component(doc, <last,first,before,after> ):
    # Do something to the doc here
    return doc

nlp.add_pipe(custom_component)

#simple component

# Create the nlp object
nlp = spacy.load('en_core_web_sm')

# Define a custom component
def custom_component(doc):
    # Print the doc's length
    print('Doc length:', len(doc))
    # Return the doc object
    return doc

# Add the component first in the pipeline
nlp.add_pipe(custom_component, first=True)

# Print the pipeline component names
print('Pipeline:', nlp.pipe_names)

# Process a text
doc = nlp("Hello world!")

import spacy
from spacy.matcher import PhraseMatcher
from spacy.tokens import Span

nlp = spacy.load("en_core_web_sm")
animals = ["Golden Retriever", "cat", "turtle", "Rattus norvegicus"]
animal_patterns = list(nlp.pipe(animals))
print("animal_patterns:", animal_patterns)
matcher = PhraseMatcher(nlp.vocab)
matcher.add("ANIMAL", None, *animal_patterns)

# Define the custom component
def animal_component(doc):
    # Apply the matcher to the doc
    matches = matcher(doc)
    # Create a Span for each match and assign the label 'ANIMAL'
    spans = [Span(doc, start, end, label="ANIMAL") for match_id, start, end in matches]
    # Overwrite the doc.ents with the matched spans
    doc.ents = spans
    return doc

# Add the component to the pipeline after the 'ner' component
nlp.add_pipe(animal_component, after="ner")
print(nlp.pipe_names)

# Process the text and print the text and label for the doc.ents
doc = nlp("I have a cat and a Golden Retriever")
print([(ent.text, ent.label_) for ent in doc.ents])

doc._.title = 'My document'
token._.is_color = True
span._.has_color = False

# Import global classes
from spacy.tokens import Doc, Token, Span

# Set extensions on the Doc, Token and Span
Doc.set_extension('title', default=None)
Token.set_extension('is_color', default=False)
Span.set_extension('has_color', default=False)

#Attribute extension
from spacy.tokens import Token

# Set extension on the Token with default value
Token.set_extension('is_color', default=False)
doc = nlp("The sky is blue.")
# Overwrite extension attribute value
doc[3]._.is_color = True

#Property (getter/setter) extension: Token
from spacy.tokens import Token

# Define getter function
def get_is_color(token):
    colors = ['red', 'yellow', 'blue']
    return token.text in colors

# Set extension on the Token with getter
Token.set_extension('is_color', getter=get_is_color, force=True)

doc = nlp("The sky is blue.")
print(doc[3]._.is_color, '-', doc[3].text)

#Property (getter/setter) extension: Span
from spacy.tokens import Span

# Define getter function
def get_has_color(span):
    colors = ['red', 'yellow', 'blue']
    return any(token.text in colors for token in span)

# Set extension on the Span with getter
Span.set_extension('has_color', getter=get_has_color)

doc = nlp("The sky is blue.")
print(doc[1:4]._.has_color, '-', doc[1:4].text)
print(doc[0:2]._.has_color, '-', doc[0:2].text)

#Method (pass an argument) extension
from spacy.tokens import Doc

# Define method with arguments
def has_token(doc, token_text):
    in_doc = token_text in [token.text for token in doc]
    return in_doc

# Set extension on the Doc with method
Doc.set_extension('has_token', method=has_token)

doc = nlp("The sky is blue.")
print(doc._.has_token('blue'), '- blue')
print(doc._.has_token('cloud'), '- cloud')

%timeit 
docs = [nlp(text) for text in LOTS_OF_TEXTS]
#bad

%timeit
docs = list(nlp.pipe(LOTS_OF_TEXTS))
#good

#this idiom is useful for associating metadata with the doc
from spacy.tokens import Doc

Doc.set_extension('id', default=None)
Doc.set_extension('page_number', default=None)

data = [
    ('This is a text', {'id': 1, 'page_number': 15}),
    ('And another text', {'id': 2, 'page_number': 16}),
]

for doc, context in nlp.pipe(data, as_tuples=True):
    doc._.id = context['id']
    doc._.page_number = context['page_number']

#slow
doc = nlp("Hello world")

#fast - only runs tokenizer, not all models
doc = nlp.make_doc("Hello world!")

#disable tagger and parser
#restores them after the with block
with nlp.disable_pipes('tagger', 'parser'):
    # Process the text and print the entities
    doc = nlp(text)
    print(doc.ents)

import json
from spacy.matcher import Matcher
from spacy.lang.en import English

TEXTS = "New iPhone X release date leaked as Apple reveals pre-orders by mistake"

nlp = English()
matcher = Matcher(nlp.vocab)

# Two tokens whose lowercase forms match 'iphone' and 'x'
pattern1 = [{"LOWER": "iphone"}, {"LOWER": "x"}]

# Token whose lowercase form matches 'iphone' and an optional digit
pattern2 = [{"LOWER": "iphone"}, {"IS_DIGIT": True, "OP": "?"}]

# Add patterns to the matcher
matcher.add("GADGET", None, pattern1, pattern2)

import json
from spacy.matcher import Matcher
from spacy.lang.en import English

with open("exercises/iphone.json") as f:
    TEXTS = json.loads(f.read())

nlp = English()
matcher = Matcher(nlp.vocab)
pattern1 = [{"LOWER": "iphone"}, {"LOWER": "x"}]
pattern2 = [{"LOWER": "iphone"}, {"IS_DIGIT": True, "OP": "?"}]
matcher.add("GADGET", None, pattern1, pattern2)

TRAINING_DATA = []

# Create a Doc object for each text in TEXTS
for doc in nlp.pipe(TEXTS):
    # Match on the doc and create a list of matched spans
    spans = [doc[start:end] for match_id, start, end in matcher(doc)]
    # Get (start character, end character, label) tuples of matches
    entities = [(span.start_char, span.end_char, "GADGET") for span in spans]
    # Format the matches as a (doc.text, entities) tuple
    training_example = (doc.text, {"entities": entities})
    # Append the example to the training data
    TRAINING_DATA.append(training_example)

print(*TRAINING_DATA, sep="\n")

import spacy
import random
import json

with open("exercises/gadgets.json") as f:
    TRAINING_DATA = json.loads(f.read())

nlp = spacy.blank("en")
ner = nlp.create_pipe("ner")
nlp.add_pipe(ner)
ner.add_label("GADGET")

# Start the training
nlp.begin_training()

# Loop for 10 iterations
for itn in range(10):
    # Shuffle the training data
    random.shuffle(TRAINING_DATA)
    losses = {}

    # Batch the examples and iterate over them
    for batch in spacy.util.minibatch(TRAINING_DATA, size=2):
        texts = [text for text, entities in batch]
        annotations = [entities for text, entities in batch]

        # Update the model
        nlp.update(texts, annotations, losses=losses)
        print(losses)

# Load a larger model with vectors
nlp = spacy.load('en_core_web_lg')

# Compare two documents
doc1 = nlp("I like fast food")
doc2 = nlp("I like pizza")
print(doc1.similarity(doc2))

! ls ../tmp;

import pickle

data_dict = {'doc1':doc1,'nlp':nlp}
filename = '../tmp/spacy-pizza.mdl'
outfile = open(filename,'wb')

pickle.dump(data_dict, outfile)
outfile.close()

infile = open(filename,'rb')
new_dict = pickle.load(infile)
infile.close()

new_dict['doc1'].similarity(doc2)

import bz2
import pickle

filename = '../tmp/spacy-pizza.bz'
#sfile = bz2.BZ2File('smallerfile', 'w')
#pickle.dump(data_dict, sfile)
#outfile.close()


outfile = bz2.BZ2File(filename, 'wb')
pickle.dump(data_dict, outfile, protocol=2)
outfile.close()

! ls ../tmp

infile = bz2.BZ2File(filename, 'rb')
myobj = pickle.load(infile)
infile.close()

myobj

myobj['doc1'].similarity(doc2)

#fast method
from spacy.attrs import LOWER, POS, ENT_TYPE, IS_ALPHA
doc = nlp(text)

# All strings mapped to integers, for easy export to numpy
np_array = doc.to_array([LOWER, POS, ENT_TYPE, IS_ALPHA])
np_array = doc.to_array("POS")

import pickle
serialized = pickle.dumps(np_array, protocol=0) # protocol 0 is printable ASCII
deserialized_array = pickle.loads(serialized)

#comprehensive approach
import spacy
from spacy.tokens import DocBin

doc_bin = DocBin(attrs=["LEMMA", "ENT_IOB", "ENT_TYPE"], store_user_data=True)
texts = ["Some text", "Lots of texts...", "..."]
nlp = spacy.load("en_core_web_sm")
for doc in nlp.pipe(texts):
    doc_bin.add(doc)
bytes_data = doc_bin.to_bytes()    #.to_disk("/path")

# Deserialize later, e.g. in a new process
nlp = spacy.blank("en")
doc_bin = DocBin().from_bytes(bytes_data)    #.from_disk("/path")
docs = list(doc_bin.get_docs(nlp.vocab))

docs = list(doc_bin.get_docs(nlp.vocab))
Doc.set_extension("my_custom_attr", default=None)
print([doc._.my_custom_attr for doc in docs])

doc = nlp("This is a sentence.")
assert len(nlp.vocab) > 0

data = {"doc":doc, "nlp":nlp}