Added possibility to do authentication with AWS Cognito on load. Other minor changes.

.dockerignore

@@ -13,4 +13,5 @@ tesseract/*
 poppler/*
 build/*
 dist/*
-build_deps/*
+build_deps/*
+doc_redaction_amplify_app/*

.gitignore

@@ -13,4 +13,5 @@ tesseract/*
 poppler/*
 build/*
 dist/*
-build_deps/*
+build_deps/*
+doc_redaction_amplify_app/*

DocRedactApp_0.2.spec

@@ -0,0 +1,52 @@
+# -*- mode: python ; coding: utf-8 -*-
+from PyInstaller.utils.hooks import collect_data_files
+
+datas = [('tesseract/', 'tesseract/'), ('poppler/poppler-24.02.0/', 'poppler/poppler-24.02.0/')]
+datas += collect_data_files('gradio_client')
+datas += collect_data_files('gradio')
+
+
+a = Analysis(
+    ['app.py'],
+    pathex=[],
+    binaries=[],
+    datas=datas,
+    hiddenimports=['pyarrow.vendored.version', 'pydicom.encoders'],
+    hookspath=['build_deps'],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=[],
+    noarchive=False,
+    optimize=0,
+    module_collection_mode={
+        'gradio': 'py',  # Collect gradio package as source .py files
+    }
+)
+pyz = PYZ(a.pure)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    [],
+    exclude_binaries=True,
+    name='DocRedactApp_0.2',
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=True,
+    console=True,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+)
+coll = COLLECT(
+    exe,
+    a.binaries,
+    a.datas,
+    strip=False,
+    upx=True,
+    upx_exclude=[],
+    name='DocRedactApp_0.2',
+)

README.md

@@ -10,9 +10,7 @@ license: mit
 ---
 
 # Introduction 
-Redact PDF files using image-based OCR or direct text analysis from pdfminer.six. Personal information identification performed using Microsoft Presidio.
-
-Take an image-based or text-based PDF document and redact any personal information. 'Image analysis' will convert PDF pages to image and the identify text via OCR methods before redaction. 'Text analysis' will analyse only selectable text that exists in the original PDF before redaction. Choose 'Image analysis' if you are not sure of the type of PDF document you are working with.
+Redact personal information from documents, open text, or xlsx/csv tabular data. See the 'Redaction settings' to change various settings such as which types of information to redact (e.g. people, places), or terms to exclude from redaction.
 
 WARNING: This is a beta product. It is not 100% accurate, and it will miss some personal information. It is essential that all outputs are checked **by a human** to ensure that all personal information has been removed.
 

app.py

@@ -3,15 +3,16 @@ import os
 # By default TLDExtract will try to pull files from the internet. I have instead downloaded this file locally to avoid the requirement for an internet connection.
 os.environ['TLDEXTRACT_CACHE'] = 'tld/.tld_set_snapshot'
 
-from tools.helper_functions import ensure_output_folder_exists, add_folder_to_path, put_columns_in_df
+from tools.helper_functions import ensure_output_folder_exists, add_folder_to_path, put_columns_in_df, get_connection_params, output_folder, get_or_create_env_var
 from tools.file_redaction import choose_and_run_redactor
 from tools.file_conversion import prepare_image_or_text_pdf
 from tools.data_anonymise import do_anonymise
+from tools.auth import authenticate_user
 #from tools.aws_functions import load_data_from_aws
 import gradio as gr
 
-add_folder_to_path("_internal/tesseract/")
-add_folder_to_path("_internal/poppler/poppler-24.02.0/Library/bin/")
+add_folder_to_path("tesseract/")
+add_folder_to_path("poppler/poppler-24.02.0/Library/bin/")
 
 ensure_output_folder_exists()
 
@@ -20,14 +21,17 @@ full_entity_list = ["TITLES", "PERSON", "PHONE_NUMBER", "EMAIL_ADDRESS", "STREET
 language = 'en'
 
 # Create the gradio interface
-block = gr.Blocks(theme = gr.themes.Base())
+app = gr.Blocks(theme = gr.themes.Base())
 
-with block:
+with app:
 
     prepared_pdf_state = gr.State([])
     output_image_files_state = gr.State([])
     output_file_list_state = gr.State([])
 
+    session_hash_state = gr.State()
+    s3_output_folder_state = gr.State()
+
     gr.Markdown(
     """
     # Document redaction
@@ -116,7 +120,15 @@ with block:
     in_file_text.upload(fn=put_columns_in_df, inputs=[in_file_text], outputs=[in_colnames])    
     match_btn.click(fn=do_anonymise, inputs=[in_file_text, in_text, anon_strat, in_colnames, in_redact_language, in_redact_entities, in_allow_list], outputs=[text_output_summary, text_output_file], api_name="redact_text")
 
+    app.load(get_connection_params, inputs=None, outputs=[session_hash_state, s3_output_folder_state])
 
 # Launch the Gradio app
+COGNITO_AUTH = get_or_create_env_var('COGNITO_AUTH', '0')
+print(f'The value of COGNITO_AUTH is {COGNITO_AUTH}')
+
 if __name__ == "__main__":
-    block.queue().launch(show_error=True) # root_path="/address-match", debug=True, server_name="0.0.0.0", server_port=7861
+
+    if os.environ['COGNITO_AUTH'] == "1":
+        app.queue().launch(show_error=True, auth=authenticate_user)
+    else:
+        app.queue().launch(show_error=True, inbrowser=True)

how_to_create_exe_dist.txt

@@ -14,7 +14,7 @@ NOTE: for ensuring that spaCy models are loaded into the program correctly in re
 
 9.Run the following, assuming you want to make one single .exe file (This helped me: https://github.com/pyinstaller/pyinstaller/issues/8108):
 
-a) In command line: pyi-makespec --additional-hooks-dir="build_deps" --add-data "tesseract/:tesseract/" --add-data "poppler/poppler-24.02.0/:poppler/poppler-24.02.0/" --collect-data=gradio_client  --collect-data=gradio  --hidden-import pyarrow.vendored.version --hidden-import pydicom.encoders  --name DocRedactApp_0.1 app.py 
+a) In command line: pyi-makespec --additional-hooks-dir="build_deps" --add-data "tesseract/:tesseract/" --add-data "poppler/poppler-24.02.0/:poppler/poppler-24.02.0/" --collect-data=gradio_client  --collect-data=gradio  --hidden-import pyarrow.vendored.version --hidden-import pydicom.encoders  --name DocRedactApp_0.2 app.py 
 
 # Add --onefile  to the above if you would like everything packaged as a single exe, although this will need to be extracted upon starting the app, slowing down initialisation time significantly.
 
@@ -28,7 +28,7 @@ a = Analysis(
     }
 )
 
-c) Back in command line, run this: pyinstaller --clean --noconfirm DocRedactApp_0.1.spec
+c) Back in command line, run this: pyinstaller --clean --noconfirm DocRedactApp_0.2.spec
 
 
 9. A 'dist' folder will be created with the executable inside along with all dependencies('dist\data_text_search'). 

tools/auth.py

@@ -0,0 +1,48 @@
+
+import boto3
+from tools.helper_functions import get_or_create_env_var
+
+client_id = get_or_create_env_var('AWS_CLIENT_ID', 'l762du1rg94e1r2q0ii7ls0ef') # This client id is borrowed from async gradio app client
+print(f'The value of AWS_CLIENT_ID is {client_id}')
+
+user_pool_id = get_or_create_env_var('AWS_USER_POOL_ID', 'eu-west-2_8fCzl8qej')
+print(f'The value of AWS_USER_POOL_ID is {user_pool_id}')
+
+def authenticate_user(username, password, user_pool_id=user_pool_id, client_id=client_id):
+    """Authenticates a user against an AWS Cognito user pool.
+
+    Args:
+        user_pool_id (str): The ID of the Cognito user pool.
+        client_id (str): The ID of the Cognito user pool client.
+        username (str): The username of the user.
+        password (str): The password of the user.
+
+    Returns:
+        bool: True if the user is authenticated, False otherwise.
+    """
+
+    client = boto3.client('cognito-idp')  # Cognito Identity Provider client
+
+    try:
+        response = client.initiate_auth(
+            AuthFlow='USER_PASSWORD_AUTH',
+            AuthParameters={
+                'USERNAME': username,
+                'PASSWORD': password,
+            },
+            ClientId=client_id
+        )
+
+        # If successful, you'll receive an AuthenticationResult in the response
+        if response.get('AuthenticationResult'):
+            return True
+        else:
+            return False
+
+    except client.exceptions.NotAuthorizedException:
+        return False
+    except client.exceptions.UserNotFoundException:
+        return False
+    except Exception as e:
+        print(f"An error occurred: {e}")
+        return False

tools/helper_functions.py

@@ -109,3 +109,62 @@ def add_folder_to_path(folder_path: str):
             print(f"Directory {folder_path} already exists in PATH.")
     else:
         print(f"Folder not found at {folder_path} - not added to PATH")
+
+async def get_connection_params(request: gr.Request):
+    if request:
+        #print("request user:", request.username)
+
+        #request_data = await request.json()  # Parse JSON body
+        #print("All request data:", request_data)
+        #context_value = request_data.get('context') 
+        #if 'context' in request_data:
+        #     print("Request context dictionary:", request_data['context'])
+
+        # print("Request headers dictionary:", request.headers)
+        # print("All host elements", request.client)           
+        # print("IP address:", request.client.host)
+        # print("Query parameters:", dict(request.query_params))
+        # To get the underlying FastAPI items you would need to use await and some fancy @ stuff for a live query: https://fastapi.tiangolo.com/vi/reference/request/
+        #print("Request dictionary to object:", request.request.body())
+        print("Session hash:", request.session_hash)
+
+        # Retrieving or setting CUSTOM_CLOUDFRONT_HEADER
+        CUSTOM_CLOUDFRONT_HEADER_var = get_or_create_env_var('CUSTOM_CLOUDFRONT_HEADER', '')
+        #print(f'The value of CUSTOM_CLOUDFRONT_HEADER is {CUSTOM_CLOUDFRONT_HEADER_var}')
+
+        # Retrieving or setting CUSTOM_CLOUDFRONT_HEADER_VALUE
+        CUSTOM_CLOUDFRONT_HEADER_VALUE_var = get_or_create_env_var('CUSTOM_CLOUDFRONT_HEADER_VALUE', '')
+        #print(f'The value of CUSTOM_CLOUDFRONT_HEADER_VALUE_var is {CUSTOM_CLOUDFRONT_HEADER_VALUE_var}')
+
+        if CUSTOM_CLOUDFRONT_HEADER_var and CUSTOM_CLOUDFRONT_HEADER_VALUE_var:
+            if CUSTOM_CLOUDFRONT_HEADER_var in request.headers:
+                supplied_cloudfront_custom_value = request.headers[CUSTOM_CLOUDFRONT_HEADER_var]
+                if supplied_cloudfront_custom_value == CUSTOM_CLOUDFRONT_HEADER_VALUE_var:
+                    print("Custom Cloudfront header found:", supplied_cloudfront_custom_value)
+                else:
+                    raise(ValueError, "Custom Cloudfront header value does not match expected value.")
+
+        # Get output save folder from 1 - username passed in from direct Cognito login, 2 - Cognito ID header passed through a Lambda authenticator, 3 - the session hash.
+
+        if request.username:
+            out_session_hash = request.username
+            print("Request username found:", out_session_hash)
+
+        elif 'x-cognito-id' in request.headers:
+            out_session_hash = request.headers['x-cognito-id']
+            base_folder = "user-files/"
+            print("Cognito ID found:", out_session_hash)
+
+        else:
+            out_session_hash = request.session_hash
+            base_folder = "temp-files/"
+            # print("Cognito ID not found. Using session hash as save folder:", out_session_hash)
+
+        output_folder = base_folder + out_session_hash + "/"
+        #if bucket_name:
+        #    print("S3 output folder is: " + "s3://" + bucket_name + "/" + output_folder)
+
+        return out_session_hash, output_folder
+    else:
+        print("No session parameters found.")
+        return "",""