# AUTOGENERATED! DO NOT EDIT! File to edit: 02_collect-data.ipynb (unless otherwise specified).

__all__ = ['query_lexeme', 'query_subr', 'query_subr_year', 'get_results', 'conv_results_to_df', 'comm_subr_to_csv',
           'get_subr_year']

# Cell
from psaw import PushshiftAPI
from tqdm import tqdm
import datetime as dt
import json
import pandas as pd
import os

# Cell
def query_lexeme(lex, year):
    api = PushshiftAPI()
    gen = api.search_comments(
        q = lex,
        after = int(dt.datetime(year, 1, 1).timestamp()),
        before = int(dt.datetime(year, 12, 31).timestamp())
    )
    return gen

# Cell
def query_subr(subreddit):
    api = PushshiftAPI()
    gen = api.search_comments(
        subreddit=subreddit
    )
    return gen

# Cell
def query_subr_year(subreddit, year):
    api = PushshiftAPI()
    gen = api.search_comments(
        subreddit = subreddit,
        after = int(dt.datetime(int(year), 1, 1).timestamp()),
        before = int(dt.datetime(int(year), 12, 31).timestamp())
    )
    return gen

# Cell
def get_results(gen, limit):
    cache = []
    for c in tqdm(gen, total=limit):
        cache.append(c)
        if len(cache) >= limit:
            break
    return cache

# Cell
def conv_results_to_df(results):
    df = pd.DataFrame([thing.d_ for thing in results])
    return df

# Cell
def comm_subr_to_csv(comments, subreddit='NaN', limit='NaN', year='NaN'):
    dir_out = f'data/subreddit/{subreddit}'
    if not os.path.exists(dir_out):
        os.makedirs(dir_out)
    comments.to_csv(
        f'{dir_out}/{limit}_{year}.csv',
        index=False
    )

# Cell
def get_subr_year(subreddit, year, limit):
    query_gen = query_subr(subreddit, year)
    results = get_results(query_gen, limit)
    comments = conv_results_to_df(results)
    comm_subr_to_csv(comments, subreddit, limit, year)
# AUTOGENERATED! DO NOT EDIT! File to edit: 02_collect-data.ipynb (unless otherwise specified).

__all__ = ['query_lexeme', 'query_subr', 'query_subr_year', 'get_results', 'conv_results_to_df', 'comm_subr_to_csv',
           'get_subr_year']

# Cell
from psaw import PushshiftAPI
from tqdm import tqdm
import datetime as dt
import json
import pandas as pd
import os

# Cell
def query_lexeme(lex, year):
    api = PushshiftAPI()
    gen = api.search_comments(
        q = lex,
        after = int(dt.datetime(year, 1, 1).timestamp()),
        before = int(dt.datetime(year, 12, 31).timestamp())
    )
    return gen

# Cell
def query_subr(subreddit):
    api = PushshiftAPI()
    gen = api.search_comments(
        subreddit=subreddit
    )
    return gen

# Cell
def query_subr_year(subreddit, year):
    api = PushshiftAPI()
    gen = api.search_comments(
        subreddit = subreddit,
        after = int(dt.datetime(int(year), 1, 1).timestamp()),
        before = int(dt.datetime(int(year), 12, 31).timestamp())
    )
    return gen

# Cell
def get_results(gen, limit):
    cache = []
    for c in tqdm(gen, total=limit):
        cache.append(c)
        if len(cache) >= limit:
            break
    return cache

# Cell
def conv_results_to_df(results):
    df = pd.DataFrame([thing.d_ for thing in results])
    return df

# Cell
def comm_subr_to_csv(comments, subreddit='NaN', limit='NaN', year='NaN'):
    dir_out = f'data/subreddit/{subreddit}'
    if not os.path.exists(dir_out):
        os.makedirs(dir_out)
    comments.to_csv(
        f'{dir_out}/{limit}_{year}.csv',
        index=False
    )

# Cell
def get_subr_year(subreddit, year, limit):
    query_gen = query_subr(subreddit, year)
    results = get_results(query_gen, limit)
    comments = conv_results_to_df(results)
    comm_subr_to_csv(comments, subreddit, limit, year)
# AUTOGENERATED! DO NOT EDIT! File to edit: 02_collect-data.ipynb (unless otherwise specified). # Cell # Cell # Cell # Cell # Cell # Cell # Cell # Cell