# script to get a list of all DOIs with mml markup
import gzip, json, sys, time
sys.path.insert(0, "comparisonTest")
from sanitise import clear_math_ml_tags

CROSSREF_PATH = "/Scratch/jc550/April 2023 Public Data File from Crossref/"

VALID_TYPES = ['journal-article', 'proceedings-article', 'book-chapter']

NUM_FILES_TO_RUN_THROUGH = 10000

mathMMLTitles = []

start_time = time.time()

for numberFile in range(NUM_FILES_TO_RUN_THROUGH):
    with gzip.open(CROSSREF_PATH + str(numberFile) + ".json.gz", "r") as gzippedFile:
        jsonDictionary = json.load(gzippedFile)
        for item in jsonDictionary["items"]:
            if "title" not in item.keys():
                continue
            title = item["title"][0]
            if title == clear_math_ml_tags(title):
                continue
            mathMMLTitles.append({"title" : title, "DOI" : item["DOI"]})

with open("mmlOut.json", "w", encoding="utf-8") as file:
    json.dump(mathMMLTitles, file)

end_time = time.time()

elapsed_time = end_time - start_time
estimated_time = (10000 / NUM_FILES_TO_RUN_THROUGH) * elapsed_time

print("Elapsed Time: " + str(elapsed_time))
print("Estimated Time for whole batch: " + str(estimated_time))