Lot Watch

def get_listings(zip_path, deleteFile=True):
    """return dict of listings given zip file path"""
    listings = []

    with ZipFile(zip_path, 'r') as archive:
        print(f'Received archive with: {archive.namelist()}')
        for html_file_name in archive.namelist():
            with archive.open(html_file_name) as html_file:
                html_contents = html_file.read().decode('utf-8')
            soup = BeautifulSoup(html_contents, 'html.parser')
            listing_divs = soup.find_all("div", class_="CLASS_NAME_HERE")
            for listing in listing_divs:
                try:
                    listings.append(json.loads(listing.text))
                except json.decoder.JSONDecodeError as e:
                    print("Invalid JSON:", str(e) + "\n Div contents:" + listing.text)
                    return None
    # remove upload
    if deleteFile: os.remove(zip_path)
    return listings, html_file_name

def filter_listings(listings, keys):
    # extract and format important data
    export_list = []
    for listing in listings:
        export_dict = {key: value for key, value in listing.items() if key in keys}
        index_map = {v: i for i, v in enumerate(keys)}
        ordered_dict = dict(sorted(export_dict.items(), key=lambda pair: index_map[pair[0]]))
        export_list.append(ordered_dict)
    return export_list

def generate_csv(listings, csv_filename):
    export_keys = ['list', 'of', 'json', 'keys']

    filtered = filter_listings(listings, export_keys)

    # Write to CSV
    export_list_values = [item.values() for item in filtered]
    # Create the uploads directory if it doesn't exist
    if not os.path.exists('processed'): os.makedirs('processed')
    with open('processed/' + csv_filename, 'w') as csvfile:
        csvwriter = csv.writer(csvfile)
        csvwriter.writerow(export_keys)
        csvwriter.writerows(export_list_values)

    print(f'Generated "{csvfile.name}" with {len(filtered)} lines"')
    return csvfile.name

def send_csv(filename):
  # validate path
  if re.match('^processed/([\w\-.]+)\.csv$', filename):
      # load into memory and delete from disk
      file_buffer = io.BytesIO()
      with open(filename, 'rb') as file:
          file_buffer.write(file.read())
      file_buffer.seek(0)
      os.remove(filename)

      # Create response with CSV download
      bare_filename = filename.replace('processed/', '')
      response = send_file(file_buffer, mimetype='text/csv', as_attachment=True, download_name=bare_filename)
      return response
  else:
      return 'Invalid file request', 403

<div class="vertical-center">
    <h1>Lot Watch</h1>
    <form>
        <h3>Upload HTML</h3>
        <label for="fileUpload">Select a file to upload:</label>
        <input type="file" name="fileUpload" id="fileUpload">
        <input type="button" value="Upload File" onclick="uploadFile()">
        <div class="upload-status" id="uploadStatus">
            <div class="lds-ellipsis">
                <div></div>
                <div></div>
                <div></div>
                <div></div>
            </div>
        </div>
        <div id="uploadStatusMessage"></div>
    </form>

    function downloadCsvResponse(xhr) {
        // Create a URL for the file response
        const blob = xhr.response;
        const url = URL.createObjectURL(blob);

        // Extract the filename from the Content-Disposition header
        const filenameRegex = /filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/;
        const filenameMatch = xhr.getResponseHeader('Content-Disposition').match(filenameRegex);
        const filename = filenameMatch && filenameMatch.length > 1 ? filenameMatch[1] : 'download';

        // Create an <a> tag with the URL as the href attribute and the filename as the download attribute
        const a = document.createElement('a');
        a.href = url;
        a.download = filename;

        // Simulate a click on the <a> tag to trigger the file download
        document.body.appendChild(a);
        a.click();
        document.body.removeChild(a);

        // Clean up the URL object to free up memory
        URL.revokeObjectURL(url);
    }

    function uploadFile() {
        document.getElementById("uploadStatusMessage").innerHTML = "";

        var fileInput = document.getElementById("fileUpload");
        var file = fileInput.files[0];
        var formData = new FormData();
        formData.append("file", file, file.name);
        var xhr = new XMLHttpRequest();
        xhr.open("POST", "/csv");
        xhr.responseType = 'blob';

        xhr.onloadstart = function () {
            document.getElementById("uploadStatus").classList.add("active");
        };

        xhr.onload = function () {
            downloadCsvResponse(xhr);
        };

        xhr.onloadend = function () {
            document.getElementById("uploadStatus").classList.remove("active");

            if (xhr.status !== 200) {
                document.getElementById("uploadStatusMessage").innerHTML = "Error processing file.";
            }
        };

        xhr.send(formData);
    }

# Flask endpoint
@app.route('/')
def home():
    return current_app.send_static_file('index.html')

# Use the official lightweight Python image.
# https://hub.docker.com/_/python
FROM python:3.10-slim

# Allow statements and log messages to immediately appear in the Knative logs
ENV PYTHONUNBUFFERED True

# Copy local code to the container image.
ENV APP_HOME /app
WORKDIR $APP_HOME
COPY . ./

# create working directories
RUN mkdir -p /uploads /processed

# Install production dependencies.
RUN pip install --no-cache-dir -r requirements.txt

# Run the static service on container startup. Here we use the gunicorn
# webserver, with one worker process and 8 threads.
# For environments with multiple CPU cores, increase the number of workers
# to be equal to the cores available.
# Timeout is set to 0 to disable the timeouts of the workers to allow Cloud Run to handle instance scaling.
CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app

def get_page(search_url, useCache=True):
  """takes a url and returns content from site or pre-existing cached version"""
  # create hash from url
  urlHash = int(hashlib.md5(search_url.encode('utf-8')).hexdigest(), 16)
  filePath = f'./pagecache/{urlHash}.json'
  # retrieve from cache if exists
  if (os.path.exists(filePath) and useCache):
      print(f'retrieved cached page for {search_url} from {filePath}')
      with open(filePath, "r") as f:
          return f.read()

  ## NORMAL WEB REQUEST HERE ##

  # cache page content for repeated calls
  if useCache:
      with open(filePath, "w") as f:
          f.write(response.text)

import os
from google.cloud import secretmanager

class Creds:
    PROJECT_ID = 'lot-watch'
    VERSION = 'latest'

    secret_prefix = f'projects/lot-watch/secrets/'
    secrets = ['app_user', 'app_password', 'service_password']
    secret_version = '/versions/latest'
    def __init__(self):
        client = secretmanager.SecretManagerServiceClient()
        for secret in self.secrets:
            # secret_req = client.secret_version_path(PROJECT_ID, secret, VERSION)
            # get secret from GCP
            response = client.access_secret_version(request={"name": self.secret_prefix + secret + self.secret_version})
            secret_resp = response.payload.data.decode('UTF-8')
            # Set secret environment variable
            os.environ[secret.upper()] = secret_resp

def store_listings(listings):
    # could be optimized with generator/yield
    print(f'writing {len(listings)} listings to db...')
    # convert to datetime
    time_format = "%Y-%m-%dT%H:%M:%SZ"
    for listing in listings:
        auction_start = datetime.datetime.strptime(listing['auctionStartTime'], time_format)
        auction_end = datetime.datetime.strptime(listing['auctionEndTime'], time_format)
        listing['auctionStartTime'] = auction_start
        listing['auctionEndTime'] = auction_end
    # split list into chunks of 100 for batch operations
    listing_batches = [listings[i:i + 100] for i in range(0, len(listings), 100)]
    batch = db.batch()
    for listing_batch in listing_batches:
        # generate doc id and add listing
        for listing in listing_batch:
            doc_ref = db.collection("listingsbyid").document(listing['id'])
            # print(listing['id'])
            batch.set(doc_ref, listing)
        # commit batch
        batch.commit()

@app.route('/updatedb', methods=['GET'])
def update_db():
    print('activated update_db() hook')
    listings = json.loads(listing_source.get_results_json('https://api.url.com', useCache=offline))['items']
    store_listings(listings)
    return 'OK', 200

@app.route('/ended', methods=['POST'])
def ended_listings():
    # get current timestamp
    now = datetime.datetime.now()
    # run query for set number of auctions which have an end time in the past
    if (request.form['password'] != os.environ.get('SERVICE_PASSWORD')):
        return 'Incorrect Password', 401

    # DESCENDING is future->past
    # >= is future listings, <= is past listings
    listingsbyid_ref = db.collection("listingsbyid")
    query = listingsbyid_ref.where('auctionEndTime', "<=", now)\
        .order_by('auctionEndTime', direction=firestore.Query.DESCENDING)\
        .order_by('id').limit(50) # past listings

    listings = []
    for pages in range(ended_listings_count):
        query_results = query.get()

        listings_page = [doc.to_dict() for doc in query_results]
        listings += listings_page
        print(len(listings))

    csv_path = f'{datetime.date.today()}.csv'
    filename = generate_csv(listings, csv_path)

    # respond to client with CSV
    response = send_csv(filename)

    # generate CSV and send to client
    print(str(now))
    return response