Model Training: Code

#! /usr/bin/env python3


def get_sibling_directory_path(sibling_directory_name):
    '''
    returns path for a specified folder that is in the same parent directory as
        the current working directory
    '''

    import os

    current_path = os.getcwd()
    last_separator_position = current_path.rfind(os.sep)
    parent_directory_path = current_path[0:last_separator_position]
    sibling_directory_path = os.path.join(parent_directory_path,
                                        sibling_directory_name)
    return(sibling_directory_path)


def iter_documents_sqlite(database_name, table_name, col_name, key_col_name):
    '''
    Iterates through database and returns each document (a list of tokens)
    '''

    import sqlite3

    con = sqlite3.connect(database_name)
    cur = con.cursor()

    cur.execute('SELECT COUNT(*) FROM {t}'.format(c=key_col_name, t=table_name))
    response = cur.fetchall()
    num_documents = response[0][0]
    print('Number of documents in database is ', num_documents)

    try:
        for i in range(num_documents):
            cur.execute('SELECT ({c1}) FROM {t} WHERE {c2}=?'
                        .format(c1=col_name, t=table_name, c2=key_col_name),
                        (i, ))
            response = cur.fetchall()
            if response:
                document = response[0][0]
            else:
                document = ''
            yield(document.split())

    finally:
        con.close()


def get_documents(database_path, table_name, col_name, key_col_name):
    '''
    Retrieves documents from database as a list of lists (1 document per list)
    Each list is composed of tokenized, processed words in the document in their
        original order
    'database_path' - path and filename for the database
    'table_name' - table name in the database where documents are stored
    'col_name' - name of column in table where documents are stored
    'key_col_name' - name of column for primary key in table where documents are
        stored
    '''

    import sqlite3

    documents = []

    con = sqlite3.connect(database_path)
    cur = con.cursor()

    cur.execute('SELECT COUNT(*) FROM {t}'.format(c=key_col_name, t=table_name))
    response = cur.fetchall()

    num_documents = response[0][0]

    try:
        for i in range(num_documents):
            cur.execute('SELECT ({c1}) FROM {t} WHERE {c2}=?'
                        .format(c1=col_name, t=table_name, c2=key_col_name),
                        (i, ))
            response = cur.fetchall()
            if response:
                document = response[0][0]
            else:
                document = ''
            documents.append(document.split())
        return(documents)

    finally:
        con.close()


def hms_string(sec_elapsed):
    '''
    # downloaded from:
    # http://www.heatonresearch.com/2017/03/03/python-basic-wikipedia-parsing.html
    # https://github.com/jeffheaton/article-code/blob/master/python/wikipedia/wiki-basic-stream.py
    # Simple example of streaming a Wikipedia
    # Copyright 2017 by Jeff Heaton, released under the The GNU Lesser General Public License (LGPL).
    # http://www.heatonresearch.com
    '''
    # Nicely formatted time string
    h = int(sec_elapsed / (60 * 60))
    m = int((sec_elapsed % (60 * 60)) / 60)
    s = sec_elapsed % 60
    return("{}:{:>02}:{:>05.2f}".format(h, m, s))


def print_models_update(model_id, elapsed_time, start_time, end_time,
                        coherence_id):
    '''
    Prints updates on models and runtimes while program is running
    'model_id' - integer identifying model
    'elapsed_time' - denotes runtime for model in seconds
    'start_time' - denotes starting date and time for model run
    'end_time' - denotes ending date and time for model run
    '''
    print('Model {m} Coherence {c} run started at {st} and ended at {et}'
        .format(m=model_id, c=coherence_id, st=start_time, et=end_time))
    print('Elapsed time: {}'.format(hms_string(elapsed_time)))
    print('\n')


def run_coherences(lda_model, texts, corpus, dictionary, model_id, coh_type,
                topn, processes):
    '''
    Calculates overall coherence and coherence per topic of LDA model and times
        the calculations
    'u_mass' coherence requires only the corpus while the other coherence types
        require the original texts
    See Gensim documentation and Roder, Both, and Hinneburg (2015) Exploring the
        Space of Topic Coherence Measures
    'lda_model' - trained Gensim LDA model
    'texts' - list of lists of texts with 1 document per list; each document is
        a list of processed, tokenized words
    'corpus' - Gensim corpus
    'dictionary' - Gensim dictionary
    'model_id' - integer identifying model
    'coh_type' - type of coherence to calculate:  'u_mass', 'c_uci', 'c_nmpi',
        or 'c_v'
    'topn' - from Gensim documentation:  'Integer corresponding to the number of
        top words to be extracted from each topic'
    'processes' - from Gensim documentation:  'number of processes to use for
        probability estimation phase; any value less than 1 will be interpreted
        to mean num_cpus - 1; default is -1.'
    '''

    from gensim.models.coherencemodel import CoherenceModel
    import time

    local_start_time = time.ctime(int(time.time()))
    start_time = time.time()

    if coh_type == 'u_mass':
        coh_model = CoherenceModel(model=lda_model, corpus=corpus,
                                dictionary=dictionary, coherence=coh_type,
                                topn=topn, processes=processes)
    else:
        coh_model = CoherenceModel(model=lda_model, texts=texts,
                                dictionary=dictionary, coherence=coh_type,
                                topn=topn, processes=processes)

    coherence = coh_model.get_coherence()
    coherence_topics = coh_model.get_coherence_per_topic()

    elapsed_time = time.time() - start_time
    local_end_time = time.ctime(int(time.time()))

    print_models_update(model_id, elapsed_time, local_start_time,
                        local_end_time, coh_type)

    return(coherence, coherence_topics, elapsed_time)


def run_lda_coherence(texts, corpus, dictionary, model_id, num_topics, alpha,
                    eta, chunksize, passes, coherences, coh_topn,
                    coh_processes, load=False):
    '''
    Trains and saves Gensim LDA model, calls function to calculate overall topic
        coherences and coherences per topic, and saves log of the process
    Returns coherences, coherences per topic, and runtimes for training LDA
        model and calculating coherences
    'texts' - list of lists of texts with 1 document per list; each document is
        a list of processed, tokenized words
    'corpus' - Gensim corpus
    'dictionary' - Gensim dictionary
    'model_id' - integer identifying model
    'num_topics' - number of topics for the LDA model to extract
    'alpha' - LDA hyperparameter that smooths document-topic distributions
    'eta' - LDA hyperparameter that smooths topic-word distributions; also known
        as 'beta'
    'chunksize' - number of documents to process per core (for multicore
        LDA training)
    'passes' - number of times to pass over the entire corpus during training
    'coherences' - list of coherences to calculate; can include 'u_mass',
        'c_uci', 'c_nmpi', and/or 'c_v'
    'coh_topn' - from Gensim documentation for coherence calculations:
        'Integer corresponding to the number of top words to be extracted from
        each topic'
    'coh_processes' - from Gensim documentation for coherence calculations:
        'number of processes to use for probability estimation phase; any value
        less than 1 will be interpreted to mean num_cpus - 1; default is -1.'
    'load' - if 'True', load saved model instead of training a new one
    '''

    from gensim.models.ldamulticore import LdaMulticore
    from gensim.models.ldamodel import LdaModel
    import logging
    import time
    import os

    eval_every = None               # do not evaluate perplexity
    save_directory_name = 'model' + str(model_id)
    save_log_directory_name = 'logs'

    if not os.path.exists(save_directory_name):
        os.makedirs(save_directory_name)
    if not os.path.exists(save_log_directory_name):
        os.makedirs(save_log_directory_name)

    returns = [corpus.num_docs, model_id, num_topics, alpha, eta, chunksize,
            passes, eval_every, coh_topn, coh_processes]

    log_filename = os.path.join(save_log_directory_name,
                                'log_lda_model' + str(model_id) + '.txt')
    logging.basicConfig(filename=log_filename,
                        format='%asctimes : %(levelname)s : %(message)s',
                        level=logging.DEBUG)

    local_start_time = time.ctime(int(time.time()))
    start_time = time.time()

    if load:
        lda = LdaModel.load(os.path.join(save_directory_name,
                                        'lda_model' + str(model_id)))

    else:
        if alpha == 'auto':     # alpha 'auto' not implemented for LdaMulticore
            lda = LdaModel(corpus=corpus, id2word=dictionary,
                        num_topics=num_topics, alpha=alpha, eta=eta,
                        chunksize=chunksize, passes=passes,
                        random_state=7111914, eval_every=eval_every)
        else:
            lda = LdaMulticore(corpus=corpus, id2word=dictionary,
                            num_topics=num_topics, alpha=alpha, eta=eta,
                            chunksize=chunksize, passes=passes,
                            random_state=7111914, eval_every=eval_every)

    elapsed_time = time.time() - start_time
    local_end_time = time.ctime(int(time.time()))

    print_models_update(model_id, elapsed_time, local_start_time,
                        local_end_time, 'none')

    if not load:
        lda.save(os.path.join(save_directory_name, 'lda_' + save_directory_name))

    returns.append(elapsed_time)

    for i in range(len(coherences)):
        coh, coh_topics, elapsed_time = run_coherences(lda, texts, corpus,
                dictionary, model_id, coherences[i], coh_topn, coh_processes)
        returns.append(coh)
        returns.append(coh_topics)
        returns.append(elapsed_time)

    return(returns)


def run_lda_models(texts, corpus, dictionary, model_id, topic_nums, alphas,
                etas, chunksizes, passes, coherences, coh_topns,
                coh_processes, load=False):
    '''
    Trains and saves multiple Gensim LDA models with different combinations of
        parameters
    Each Gensim LDA model has coherences and coherences per topic calculated
        along with runtimes for the training and calculations
    These results are saved to a Pandas DataFrame and returned
    Gensim LDA models are trained on different combinations of the parameters
        'topic_nums', 'alphas', 'etas', 'chunksizes', 'passes', and 'coh_topns'
    'texts' - list of lists of texts with 1 document per list; each document is
        a list of processed, tokenized words
    'corpus' - Gensim corpus
    'dictionary' - Gensim dictionary
    'model_id' - integer identifying model
    'topic_nums' - list of numbers of topics for LDA models to extract
    'alphas' - list of LDA hyperparameters 'alpha' that smooths document-topic
        distributions
    'etas' - list of LDA hyperparameters 'eta' that smooths topic-word
        distributions; also known as 'beta'
    'chunksizes' - list of numbers of documents to process per core (for
        multicore LDA training)
    'passes' - list of numbers of times to pass over the entire corpus during
        training
    'coherences' - list of coherences to calculate; can include 'u_mass',
        'c_uci', 'c_nmpi', and/or 'c_v'
    'coh_topns' - list of 'coh_topns'; from Gensim documentation for coherence
        calculations for each 'coh_topn': 'Integer corresponding to the number
        of top words to be extracted from each topic'
    'coh_processes' - from Gensim documentation for coherence calculations:
        'number of processes to use for probability estimation phase; any value
        less than 1 will be interpreted to mean num_cpus - 1; default is -1.'
    'load' - if 'True', load saved model instead of training a new one
    '''

    import pandas as pd
    from itertools import product
    import time

    columns = ['num_documents', 'model_ID', 'num_topics', 'alpha', 'eta',
            'chunksize', 'passes', 'eval_every', 'coh_topn', 'coh_processes',
            'lda_runtime']
    coh_outcomes = ['coherence', 'coh_topics', 'coh_runtime']
    coh_columns = [e[0] + '_' + e[1]
                for e in list(product(coherences, coh_outcomes))]
    columns.extend(coh_columns)
    columns.append('overall_runtime')

    lda_runs = []

    # NOTE TO SELF:  yes, the matryoshka nested loops are kinda ridiculous;
    # need to create lists of different combinations of parameters
    for i in range(len(topic_nums)):
        for j in range(len(alphas)):
            for k in range(len(etas)):
                for l in range(len(chunksizes)):
                    for m in range(len(passes)):
                        for n in range(len(coh_topns)):

                            start_time = time.time()

                            lda_run = run_lda_coherence(texts, corpus,
                                dictionary, model_id, topic_nums[i], alphas[j],
                                etas[k], chunksizes[l], passes[m], coherences,
                                coh_topns[n], coh_processes, load)

                            elapsed_time = time.time() - start_time

                            lda_run.append(elapsed_time)
                            lda_runs.append(lda_run)

                            model_id += 1

    results = pd.DataFrame.from_records(lda_runs, columns=columns)

    return(results)


def main():
    '''
    Testing chunksize and iterations numbers:  more iterations provide greater
        probability that documents will converge (i.e., the LDA model provides
        good quality results) but at the cost of longer runtime
    This testing tries to determine which values of 'chunksize' will adequately
        balance these two outcomes
    '''

    import os
    import gensim

    #wiki_folder = '41_fast_corpus'
    wiki_folder = os.path.join('41_fast_corpus', 'sampling_interval_05_5578331')
    wiki_path = get_sibling_directory_path(wiki_folder)
    dictionary_filename = 'wiki_dictionary.dict'
    corpus_filename = 'wiki_corpus.mm'
    dictionary_filepath = os.path.join(wiki_path, dictionary_filename)
    corpus_filepath = os.path.join(wiki_path, corpus_filename)

    database_name = 'wiki_token_docs.sqlite'
    database_path = os.path.join(wiki_path, database_name)
    table_name = 'articles'
    column_name = 'text'
    key_column_name = 'key'
    print('Retrieving documents')
    documents = iter_documents_sqlite(database_path, table_name, column_name,
                                    key_column_name)
    #documents = get_documents(database_path, table_name, column_name,
    #                          key_column_name)
    print('Documents retrieved')

    wiki_dictionary = gensim.corpora.Dictionary.load(dictionary_filepath)
    wiki_corpus = gensim.corpora.MmCorpus(corpus_filepath)
    print('Dictionary and corpus loaded')

    model_id = 104
    topic_nums = [30]
    alphas = ['auto']                   # 'auto' worked best in prior testing
    etas = [None]                       # 'None' worked best in prior testing
    chunksizes = [100000]
    passes = [1]
    coherences = ['c_v']                # 'c_v' best results:  Roder, Both, Hinneburg 2015
    coh_topns = [10]                    # default value
    coh_processes = -1                  # default value
    load_model = False
    results = run_lda_models(documents, wiki_corpus, wiki_dictionary, model_id,
                            topic_nums, alphas, etas, chunksizes, passes,
                            coherences, coh_topns, coh_processes, load_model)

    csv_filename = 'model_results.csv'
    #results.to_csv(csv_filename)
    # WARNING:  changing coherences will alter the column headings and/or the
    #   number of columns; append to existing 'csv' file only if coherences has
    #   not been altered
    if os.path.isfile(csv_filename):
        results.to_csv(csv_filename, mode='a', header=False)
    else:
        results.to_csv(csv_filename)

    return()


if __name__ == '__main__':
    main()