Model Training Log Extraction: Code

#! /usr/bin/env python3


def listdir_nohidden_nofolder(path):
    '''
    returns list of nonhidden files in the directory of the specified path
        excludes folders
    'os.listdir' includes hidden files, but this function excludes them
    '''

    import os

    file_list = next(os.walk(path))[2]

    for e in file_list:

        if e.startswith('.'):
            file_list.remove(e)

    return(file_list)


def read_text_file(text_filename, as_string=False):
    '''
    reads each line in a text file as a list item and returns list by default
    if 'as_string' is 'True', reads entire text file as a single string
    '''

    text_list = []

    try:
        with open(text_filename) as text:
            if as_string:
                # reads text file as single string
                text_list = text.read().replace('\n', '')
            else:
                # reads each line of text file as item in a list
                for line in text:
                    text_list.append(line.rstrip('\n'))
            text.close()
        return(text_list)

    except:
        return('There was an error while trying to read the file')


def write_list_to_text_file(a_list, text_file_name, overwrite_or_append='a'):
    '''
    writes a list to a text file with one list element per line in the file
    appends by default; change to overwriting by setting to 'w' instead of 'a'
    '''

    try:
        textfile = open(text_file_name, overwrite_or_append, encoding='utf-8')
        for element in a_list:
            textfile.write('{e}'.format(e=element))
            textfile.write('\n')

    finally:
        textfile.close()


def get_log_topic_diff(log):
    '''
    Identifies and returns topic difference calculations ('topic diff') in
        Gensim LDA model training log
    'log' - a list where each element is a line from the Gensim LDA model
        training log
    '''
    identifier = 'topic diff='
    diff_messages = [e for e in log if identifier in e]

    topic_diffs = [e.rsplit(identifier, 1)[1] for e in diff_messages]
    topic_diffs = [float(e.split(',', 1)[0]) for e in topic_diffs]

    return(topic_diffs)


def get_log_documents_converged(log):
    '''
    Identifies numbers of documents that converge in Gensim LDA model training
        log and returns these numbers as proportions (e.g., 49 out of 100
        documents converged is returned as 0.49)
    Note that most of the denominators (i.e., the size of the mini-batch in
        online training) are the same, but when the trainer reaches the end of
        a pass through a corpus, the last mini-batch size may be smaller
    'log' - a list where each element is a line from the Gensim LDA model
        training log
    '''

    identifier = 'documents converged within'
    converged_messages = [e for e in log if identifier in e]
    fraction = [e.rsplit(identifier, 1)[0] for e in converged_messages]

    debug_splitter = 'DEBUG :'
    fraction = [e.rsplit(debug_splitter, 1)[1].strip() for e in fraction]
    slash = '/'

    numerators = [int(e.split(slash)[0]) for e in fraction]
    denominators = [int(e.split(slash)[1]) for e in fraction]
    proportions = [n / d for n, d in zip(numerators, denominators)]

    return(proportions)


def divide_log_by_model(log):
    '''
    Divides log of Gensim LDA models training into multiple logs -- 1 for each
        LDA model in the original log
    Returns logs as a list of the logs
    'log' - a list where each element is a line from the Gensim LDA model
        training log
    '''
    identifier = "'init_dir_prior', 'created':"
    identifier_idx = [i for i, e in enumerate(log) if identifier in e]

    # identifier occurs in 1st 2 lines of every model run, so remove 2nd lines
    start_idx = [e for i, e in enumerate(identifier_idx) if i % 2 == 0]

    model_logs = [log[s:e] for s, e in zip(start_idx, start_idx[1:] + [None])]
    return(model_logs)


def get_end_digits_from_filename(filename):
    '''
    Returns any consecutive digits at the end of a filename, excluding the
        filename extension
    If no digits are present at the end of the filename, returns empty string
    '''
    from re import search as re_search

    end_digits = re_search(r'\d+$', filename.rsplit('.', 1)[0])

    if end_digits == None:
        return('')
    else:
        return(int(end_digits.group()))


def process_log(log_filename):
    '''
    Reads log of Gensim LDA models training from file, divides it into multiple
        logs if multiple LDA models are recorded in the log, extracts 'topic
        diff' and proportions of converged documents per mini-batch from the
        logs, and saves them to text files
    'log_filename' - filename for log of Gensim LDA models training; the model
        ID number for the first model in the log is assumed to be at the end of
        the filename (excluding the file extension); subsequent model ID
        numbers are assumed to increment by 1 (e.g., model 1, model 2,
        model 3, etc.)
    '''

    import os

    log_path = os.path.dirname(log_filename)
    log = read_text_file(log_filename)
    logs = divide_log_by_model(log)

    topic_diffs = [get_log_topic_diff(e) for e in logs]
    converged_proportions = [get_log_documents_converged(e) for e in logs]
    model_id = get_end_digits_from_filename(log_filename)

    for i in range(len(logs)):

        td_filename = 'topic_diffs' + str(model_id + i) + '.txt'
        td_filepath = os.path.join(log_path, td_filename)
        write_list_to_text_file(topic_diffs[i], td_filepath, 'w')

        cp_filename = 'converged_props' + str(model_id + i) + '.txt'
        cp_filepath = os.path.join(log_path, cp_filename)
        write_list_to_text_file(converged_proportions[i], cp_filepath, 'w')


def process_logs(log_filenames):
    '''
    Calls function 'process_log' once for each filename in 'log_filenames'
    'log_filenames' - list of filenames
    '''
    for i in range(len(log_filenames)):
        process_log(log_filenames[i])


def main():
    '''
    From each file named in 'log_filenames', reads log of Gensim LDA models
        training from file, divides it into multiple logs if multiple LDA models
        are recorded in the log, extracts 'topic diff' and proportions of
        converged documents per mini-batch from the logs, and saves them to text
        files
    '''

    import os

    logs_folder = 'logs'
    path = os.path.join(os.getcwd(), logs_folder)
    filenames = listdir_nohidden_nofolder(path)

    log_stem = 'log_lda_model'
    log_stem_len = len(log_stem)
    log_filenames = [e for e in filenames if e[:log_stem_len] == log_stem]
    log_filepaths = [os.path.join(path, e) for e in log_filenames]

    process_logs(log_filepaths)


if __name__ == '__main__':
    main()