Upload 337 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+obsei_module/images/obsei_flow.gif filter=lfs diff=lfs merge=lfs -text
+obsei_module/obsei-master/images/obsei_flow.gif filter=lfs diff=lfs merge=lfs -text

obsei_module/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,27 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: "[BUG]"
+labels: bug
+assignees: lalitpagaria
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Stacktrace**
+If applicable, add stacktrace to help explain your problem.
+
+**Please complete the following information:**
+ - OS:
+ - Version:
+
+**Additional context**
+Add any other context about the problem here.

obsei_module/.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

obsei_module/.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

obsei_module/.github/release-drafter.yml

@@ -0,0 +1,33 @@
+name-template: 'v$RESOLVED_VERSION 🌈'
+tag-template: 'v$RESOLVED_VERSION'
+categories:
+  - title: '🚀 Features'
+    labels:
+      - 'feature'
+      - 'enhancement'
+  - title: '🐛 Bug Fixes'
+    labels:
+      - 'fix'
+      - 'bugfix'
+      - 'bug'
+  - title: '🧰 Maintenance'
+    label: 'chore'
+  - title: '⚠️Breaking Changes'
+    label: 'breaking changes'
+change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
+change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks.
+version-resolver:
+  major:
+    labels:
+      - 'major'
+  minor:
+    labels:
+      - 'minor'
+  patch:
+    labels:
+      - 'patch'
+  default: patch
+template: |
+  ## Changes
+
+  $CHANGES

obsei_module/.github/workflows/build.yml

@@ -0,0 +1,54 @@
+# This workflow will install Python dependencies, run test and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: CI
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  type-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Test with mypy
+        run: |
+          pip install mypy
+          # Refer http://mypy-lang.blogspot.com/2021/06/mypy-0900-released.html
+          pip install mypy types-requests types-python-dateutil types-PyYAML types-dateparser types-protobuf types-pytz
+          mypy obsei
+
+  build-and-test:
+    needs: type-check
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
+        python-version: ['3.8', '3.9', '3.10', '3.11']
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install '.[dev,all]'
+          pip install --upgrade --upgrade-strategy eager trafilatura
+          python -m spacy download en_core_web_lg
+          python -m spacy download en_core_web_sm
+
+      - name: Test with pytest
+        run: |
+          coverage run -m pytest
+          coverage report -m

obsei_module/.github/workflows/pypi_publish.yml

@@ -0,0 +1,35 @@
+# This workflows will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Upload Python Package
+
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+
+jobs:
+  deploy-pypi-artifact:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.8'
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel twine hatch
+
+    - name: publish to PyPI
+      if: github.event_name != 'pull_request'
+      env:
+        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      run: |
+        hatch build
+        twine upload dist/*

obsei_module/.github/workflows/release_draft.yml

@@ -0,0 +1,15 @@
+name: release draft
+
+on:
+  workflow_dispatch:
+
+jobs:
+  draft-release:
+#    if: startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    steps:
+      - uses: release-drafter/release-drafter@v6
+        with:
+          config-name: release-drafter.yml
+        env:
+          GITHUB_TOKEN: ${{ secrets.RELEASE_DRAFT_TOKEN }}

obsei_module/.github/workflows/sdk_docker_publish.yml

@@ -0,0 +1,50 @@
+# This workflows will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Publish SDK docker image
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Image tag'
+        required: true
+  release:
+    types: [published]
+
+jobs:
+  deploy-sdk-docker:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Docker meta
+      id: docker_meta
+      uses: docker/metadata-action@v5
+      with:
+        images: obsei/obsei-sdk
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Login to DockerHub
+      if: github.event_name != 'pull_request'
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+    - name: Build and push
+      uses: docker/build-push-action@v5
+      with:
+        context: ./
+        file: ./Dockerfile
+        push: ${{ github.event_name != 'pull_request' }}
+        tags: ${{ steps.docker_meta.outputs.tags }}
+        labels: ${{ steps.docker_meta.outputs.labels }}
+
+    - name: Image digest
+      run: echo ${{ steps.docker_build.outputs.digest }}

obsei_module/.github/workflows/ui_docker_publish.yml

@@ -0,0 +1,50 @@
+# This workflows will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Publish UI Docker image
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Image tag'
+        required: true
+  release:
+    types: [published]
+
+jobs:
+  deploy-ui-docker:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Docker meta
+      id: docker_meta
+      uses: docker/metadata-action@v5
+      with:
+        images: obsei/obsei-ui-demo
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Login to DockerHub
+      if: github.event_name != 'pull_request'
+      uses: docker/login-action@v3
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+    - name: Build and push
+      uses: docker/build-push-action@v5
+      with:
+        context: "{{defaultContext}}:sample-ui"
+        file: Dockerfile
+        push: ${{ github.event_name != 'pull_request' }}
+        tags: ${{ steps.docker_meta.outputs.tags }}
+        labels: ${{ steps.docker_meta.outputs.labels }}
+
+    - name: Image digest
+      run: echo ${{ steps.docker_build.outputs.digest }}

obsei_module/.gitignore

@@ -0,0 +1,148 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+/.idea/*
+*.db
+models*
+
+# OSX custom attributes
+.DS_Store
+
+# VS code configuration
+.vscode/*

obsei_module/.pre-commit-config.yaml

@@ -0,0 +1,21 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-yaml
+      - id: trailing-whitespace
+      - id: requirements-txt-fixer
+      - id: end-of-file-fixer
+
+  - repo: https://github.com/psf/black
+    rev: 22.10.0
+    hooks:
+      - id: black
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.991
+    hooks:
+      - id: mypy
+        args: [--ignore-missing-imports]
+        additional_dependencies: [types-all]
+        files: ^obsei/

obsei_module/.pyup.yml

@@ -0,0 +1,5 @@
+# autogenerated pyup.io config file
+# see https://pyup.io/docs/configuration/ for all available options
+
+schedule: ''
+update: insecure

obsei_module/ATTRIBUTION.md

@@ -0,0 +1,18 @@
+This could not have been possible without following open source software -
+- [searchtweets-v2](https://github.com/twitterdev/search-tweets-python): For Twitter's API v2 wrapper
+- [vaderSentiment](https://github.com/cjhutto/vaderSentiment): For rule-based sentiment analysis
+- [transformers](https://github.com/huggingface/transformers): For text-classification pipeline
+- [atlassian-python-api](https://github.com/atlassian-api/atlassian-python-api): To interact with Jira
+- [elasticsearch](https://github.com/elastic/elasticsearch-py): To interact with Elasticsearch
+- [pydantic](https://github.com/samuelcolvin/pydantic): For data validation
+- [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy): As SQL toolkit to access DB storage
+- [google-play-scraper](https://github.com/JoMingyu/google-play-scraper): To fetch the Google Play Store review without authentication
+- [praw](https://github.com/praw-dev/praw): For Reddit client
+- [reddit-rss-reader](https://github.com/lalitpagaria/reddit-rss-reader): For Reddit scrapping
+- [app-store-reviews-reader](https://github.com/lalitpagaria/app_store_reviews_reader): For App Store reviews scrapping
+- [slack-sdk](https://github.com/slackapi/python-slack-sdk): For slack integration
+- [presidio-anonymizer](https://github.com/microsoft/presidio): Personal information anonymizer
+- [GoogleNews](https://github.com/Iceloof/GoogleNews): For Google News integration
+- [python-facebook-api](https://github.com/sns-sdks/python-facebook): For facebook integration
+- [youtube-comment-downloader](https://github.com/egbertbouman/youtube-comment-downloader): For Youtube video comments extraction code
+- [dateparser](https://github.com/scrapinghub/dateparser): To parse date properly (where format is ambiguous)

obsei_module/CITATION.cff

@@ -0,0 +1,14 @@
+# YAML 1.2
+---
+authors: 
+  -
+    family-names: Pagaria
+    given-names: Lalit
+
+cff-version: "1.1.0"
+license: "Apache-2.0"
+message: "If you use this software, please cite it using this metadata."
+repository-code: "https://github.com/obsei/obsei"
+title: "Obsei - a low code AI powered automation tool"
+version: "0.0.10"
+...

obsei_module/CNAME

@@ -0,0 +1 @@
+www.obsei.com

obsei_module/CODE_OF_CONDUCT.md

@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+- Demonstrating empathy and kindness toward other people
+- Being respectful of differing opinions, viewpoints, and experiences
+- Giving and gracefully accepting constructive feedback
+- Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+- Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+- The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+- Trolling, insulting or derogatory comments, and personal or political attacks
+- Public or private harassment
+- Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+- Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+obsei.tool@gmail.com
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior, harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

obsei_module/CONTRIBUTING.md

@@ -0,0 +1,103 @@
+# 👐 Contributing to Obsei
+
+First off, thank you for even considering contributing to this package, every contribution big or small is greatly appreciated.
+Community contributions are what keep projects like this fueled and constantly improving, so a big thanks to you!
+
+Below are some sections detailing the guidelines we'd like you to follow to make your contribution as seamless as possible.
+
+- [Code of Conduct](#coc)
+- [Asking a Question and Discussions](#question)
+- [Issues, Bugs, and Feature Requests](#issue)
+- [Submission Guidelines](#submit)
+- [Code Style and Formatting](#code)
+- [Contributor License Agreement](#cla)
+
+## 📜 <a name="coc"></a> Code of Conduct
+
+The [Code of Conduct](https://github.com/obsei/obsei/blob/master/CODE_OF_CONDUCT.md) applies within all community spaces.
+If you are not familiar with our Code of Conduct policy, take a minute to read the policy before starting with your first contribution.
+
+## 🗣️ <a name="question"></a> Query or Discussion
+
+We would like to use [Github discussions](https://github.com/obsei/obsei/discussions) as the central hub for all
+community discussions, questions, and everything else in between. While Github discussions is a new service (as of 2021)
+we believe that it really helps keep this repo as one single source to find all relevant information. Our hope is that
+discussion page functions as a record of all the conversations that help contribute to the project's development.
+
+If you are new to [Github discussions](https://github.com/obsei/obsei/discussions) it is a very similar experience
+to Stack Overflow with an added element of general discussion and discourse rather than solely being question and answer based.
+
+## 🪲 <a name="issue"></a> Issues, Bugs, and Feature Requests
+
+We are very open to community contributions and appreciate anything that improves **Obsei**. This includes fixings typos, adding missing documentation, fixing bugs or adding new features.
+To avoid unnecessary work on either side, please stick to the following process:
+
+1. If you feel like your issue is not specific and more of a general question about a design decision, or algorithm implementation maybe start a [discussion](https://github.com/obsei/obsei/discussions) instead, this helps keep the issues less cluttered and encourages more open-ended conversation.
+2. Check if there is already [an related issue](https://github.com/obsei/obsei/issues).
+3. If there is not, open a new one to start a discussion. Some features might be a nice idea, but don't fit in the scope of Obsei and we hate to close finished PRs.
+4. If we came to the conclusion to move forward with your issue, we will be happy to accept a pull request. Make sure you create a pull request in an early draft version and ask for feedback.
+5. Verify that all tests in the CI pass (and add new ones if you implement anything new)
+
+See [below](#submit) for some guidelines.
+
+## ✉️ <a name="submit"></a> Submission Guidelines
+
+### Submitting an Issue
+
+Before you submit your issue search the archive, maybe your question was already answered.
+
+If your issue appears to be a bug, and hasn't been reported, open a new issue.
+Help us to maximize the effort we can spend fixing issues and adding new
+features, by not reporting duplicate issues. Providing the following information will increase the
+chances of your issue being dealt with quickly:
+
+- **Describe the bug** - A clear and concise description of what the bug is.
+- **To Reproduce**- Steps to reproduce the behavior.
+- **Expected behavior** - A clear and concise description of what you expected to happen.
+- **Environment**
+  - Obsei version
+  - Python version
+  - OS
+- **Suggest a Fix** - if you can't fix the bug yourself, perhaps you can point to what might be
+  causing the problem (line of code or commit)
+
+When you submit a PR you will be presented with a PR template, please fill this in as best you can.
+
+### Submitting a Pull Request
+
+Before you submit your pull request consider the following guidelines:
+
+- Search [GitHub](https://github.com/obsei/obsei/pulls) for an open or closed Pull Request
+  that relates to your submission. You don't want to duplicate effort.
+- Fork the main repo if not already done
+- Rebase fork with `upstream master`
+- Create new branch and add the changes in that branch
+- Add supporting test cases
+- Follow our [Coding Rules](#rules).
+- Avoid checking in files that shouldn't be tracked (e.g `dist`, `build`, `.tmp`, `.idea`).
+  We recommend using a [global](#global-gitignore) gitignore for this.
+- Before you commit please run the test suite and make sure all tests are passing.
+- Format your code appropriately:
+  - This package uses [black](https://black.readthedocs.io/en/stable/) as its formatter.
+    In order to format your code with black run `black . ` from the root of the package.
+- Run `pre-commit run --all-files` if you're adding new hooks to pre-commit config file. By default, pre-commit will run on modified files when commiting changes.
+- Commit your changes using a descriptive commit message.
+- In GitHub, send a pull request to `obsei:master`.
+- If we suggest changes then:
+  - Make the required updates.
+  - Rebase your branch and force push to your GitHub repository (this will update your Pull Request):
+
+That's it! Thank you for your contribution!
+
+## ✅ <a name="rules"></a> Coding Rules
+
+We generally follow the [Google Python style guide](http://google.github.io/styleguide/pyguide.html).
+
+## 📝 <a name="cla"></a> Contributor License Agreement
+
+That we do not have any potential problems later it is sadly necessary to sign a [Contributor License Agreement](CONTRIBUTOR_LICENSE_AGREEMENT.md). That can be done literally with the push of a button.
+
+---
+
+_This guide was inspired by the [transformers-interpret](https://github.com/cdpierse/transformers-interpret/blob/master/CONTRIBUTING.md), 
+[Haystack](https://github.com/deepset-ai/haystack/blob/master/CONTRIBUTING.md) and [n8n](https://github.com/n8n-io/n8n/blob/master/CONTRIBUTOR_LICENSE_AGREEMENT.md)_

obsei_module/CONTRIBUTOR_LICENSE_AGREEMENT.md

@@ -0,0 +1,3 @@
+# Obsei Contributor License Agreement
+
+I give Obsei's Creator permission to license my contributions to any terms they like. I am giving them this license in order to make it possible for them to accept my contributions into their project.

obsei_module/Dockerfile

@@ -0,0 +1,38 @@
+# This is Docker file to Obsei SDK with dependencies installed
+FROM python:3.10-slim-bullseye
+
+RUN useradd --create-home user
+WORKDIR /home/user
+
+# env variable
+ENV PIP_DISABLE_PIP_VERSION_CHECK 1
+ENV PIP_NO_CACHE_DIR 1
+ENV WORKFLOW_SCRIPT '/home/user/obsei/process_workflow.py'
+ENV OBSEI_CONFIG_PATH ""
+ENV OBSEI_CONFIG_FILENAME ""
+
+
+# Hack to install jre on debian
+RUN mkdir -p /usr/share/man/man1
+
+# install few required tools
+RUN apt-get update && apt-get install -y --no-install-recommends curl git pkg-config cmake libncurses5 g++ \
+    && apt-get clean autoclean && apt-get autoremove -y \
+    && rm -rf /var/lib/{apt,dpkg,cache,log}/
+
+# install as a package
+COPY pyproject.toml README.md /home/user/
+RUN pip install --upgrade pip
+
+# copy README
+COPY README.md /home/user/
+
+# copy code
+COPY obsei /home/user/obsei
+RUN pip install -e .[all]
+
+
+USER user
+
+# cmd for running the API
+CMD ["sh", "-c", "python ${WORKFLOW_SCRIPT}"]

obsei_module/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2020-2022 Oraika Technologies Private Limited (https://www.oraika.com)
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

obsei_module/MANIFEST.in

@@ -0,0 +1,3 @@
+include LICENSE
+include requirements.txt
+include README.md

obsei_module/README.md

@@ -0,0 +1,1067 @@
+<p align="center">
+    <img src="https://raw.githubusercontent.com/obsei/obsei-resources/master/images/obsei-flyer.png" />
+</p>
+
+---
+<p align="center">
+    <a href="https://www.oraika.com">
+            <img src="https://static.wixstatic.com/media/59bc4e_971f153f107e48c7912b9b2d4cd1b1a4~mv2.png/v1/fill/w_177,h_49,al_c,q_85,usm_0.66_1.00_0.01,enc_auto/3_edited.png" />
+    </a>
+</p>
+<p align="center">
+    <a href="https://github.com/obsei/obsei/actions">
+        <img alt="Test" src="https://github.com/obsei/obsei/workflows/CI/badge.svg?branch=master">
+    </a>
+    <a href="https://github.com/obsei/obsei/blob/master/LICENSE">
+        <img alt="License" src="https://img.shields.io/pypi/l/obsei">
+    </a>
+    <a href="https://pypi.org/project/obsei">
+        <img src="https://img.shields.io/pypi/pyversions/obsei" alt="PyPI - Python Version" />
+    </a>
+    <a href="https://pypi.org/project/obsei/">
+        <img alt="Release" src="https://img.shields.io/pypi/v/obsei">
+    </a>
+    <a href="https://pepy.tech/project/obsei">
+        <img src="https://pepy.tech/badge/obsei/month" alt="Downloads" />
+    </a>
+    <a href="https://huggingface.co/spaces/obsei/obsei-demo">
+        <img src="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue" alt="HF Spaces" />
+    </a>
+    <a href="https://github.com/obsei/obsei/commits/master">
+        <img alt="Last commit" src="https://img.shields.io/github/last-commit/obsei/obsei">
+    </a>
+    <a href="https://github.com/obsei/obsei">
+        <img alt="Github stars" src="https://img.shields.io/github/stars/obsei/obsei?style=social">
+    </a>
+    <a href="https://www.youtube.com/channel/UCqdvgro1BzU13tkAfX3jCJA">
+        <img alt="YouTube Channel Subscribers" src="https://img.shields.io/youtube/channel/subscribers/UCqdvgro1BzU13tkAfX3jCJA?style=social">
+    </a>
+    <a href="https://join.slack.com/t/obsei-community/shared_invite/zt-r0wnuz02-FAkAmhTAUoc6pD4SLB9Ikg">
+        <img src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/Slack_join.svg" height="30">
+    </a>
+    <a href="https://twitter.com/ObseiAI">
+        <img src="https://img.shields.io/twitter/follow/ObseiAI?style=social">
+    </a>
+</p>
+
+---
+
+![](https://raw.githubusercontent.com/obsei/obsei-resources/master/gifs/obsei_flow.gif)
+
+---
+
+<span style="color:red">
+<b>Note</b>: Obsei is still in alpha stage hence carefully use it in Production. Also, as it is constantly undergoing development hence master branch may contain many breaking changes. Please use released version.
+</span>
+
+---
+
+**Obsei** (pronounced "Ob see" | /əb-'sē/) is an open-source, low-code, AI powered automation tool. _Obsei_ consists of -
+
+- **Observer**: Collect unstructured data from various sources like tweets from Twitter, Subreddit comments on Reddit, page post's comments from Facebook, App Stores reviews, Google reviews, Amazon reviews, News, Website, etc.
+- **Analyzer**: Analyze unstructured data collected with various AI tasks like classification, sentiment analysis, translation, PII, etc.
+- **Informer**: Send analyzed data to various destinations like ticketing platforms, data storage, dataframe, etc so that the user can take further actions and perform analysis on the data.
+
+All the Observers can store their state in databases (Sqlite, Postgres, MySQL, etc.), making Obsei suitable for scheduled jobs or serverless applications.
+
+![Obsei diagram](https://raw.githubusercontent.com/obsei/obsei-resources/master/images/Obsei_diagram.png)
+
+### Future direction -
+
+- Text, Image, Audio, Documents and Video oriented workflows
+- Collect data from every possible private and public channels
+- Add every possible workflow to an AI downstream application to automate manual cognitive workflows
+
+## Use cases
+
+_Obsei_ use cases are following, but not limited to -
+
+- Social listening: Listening about social media posts, comments, customer feedback, etc.
+- Alerting/Notification: To get auto-alerts for events such as customer complaints, qualified sales leads, etc.
+- Automatic customer issue creation based on customer complaints on Social Media, Email, etc.
+- Automatic assignment of proper tags to tickets based content of customer complaint for example login issue, sign up issue, delivery issue, etc.
+- Extraction of deeper insight from feedbacks on various platforms
+- Market research
+- Creation of dataset for various AI tasks
+- Many more based on creativity 💡
+
+## Installation
+
+### Prerequisite
+
+Install the following (if not present already) -
+
+- Install [Python 3.7+](https://www.python.org/downloads/)
+- Install [PIP](https://pip.pypa.io/en/stable/installing/)
+
+### Install Obsei
+
+You can install Obsei either via PIP or Conda based on your preference.
+To install latest released version -
+
+```shell
+pip install obsei[all]
+```
+
+Install from master branch (if you want to try the latest features) -
+
+```shell
+git clone https://github.com/obsei/obsei.git
+cd obsei
+pip install --editable .[all]
+```
+  
+Note: `all` option will install all the dependencies which might not be needed for your workflow, alternatively 
+following options are available to install minimal dependencies as per need -
+ - `pip install obsei[source]`: To install dependencies related to all observers
+ - `pip install obsei[sink]`: To install dependencies related to all informers
+ - `pip install obsei[analyzer]`:  To install dependencies related to all analyzers, it will install pytorch as well
+ - `pip install obsei[twitter-api]`: To install dependencies related to Twitter observer
+ - `pip install obsei[google-play-scraper]`: To install dependencies related to Play Store review scrapper observer
+ - `pip install obsei[google-play-api]`: To install dependencies related to Google official play store review API based observer
+ - `pip install obsei[app-store-scraper]`: To install dependencies related to Apple App Store review scrapper observer
+ - `pip install obsei[reddit-scraper]`: To install dependencies related to Reddit post and comment scrapper observer
+ - `pip install obsei[reddit-api]`: To install dependencies related to Reddit official api based observer
+ - `pip install obsei[pandas]`: To install dependencies related to TSV/CSV/Pandas based observer and informer
+ - `pip install obsei[google-news-scraper]`: To install dependencies related to Google news scrapper observer
+ - `pip install obsei[facebook-api]`: To install dependencies related to Facebook official page post and comments api based observer
+ - `pip install obsei[atlassian-api]`: To install dependencies related to Jira official api based informer
+ - `pip install obsei[elasticsearch]`: To install dependencies related to elasticsearch informer
+ - `pip install obsei[slack-api]`:To install dependencies related to Slack official api based informer
+
+You can also mix multiple dependencies together in single installation command. For example to install dependencies 
+Twitter observer, all analyzer, and Slack informer use following command -
+```shell
+pip install obsei[twitter-api, analyzer, slack-api]
+```
+
+
+## How to use
+
+Expand the following steps and create a workflow -
+
+<details><summary><b>Step 1: Configure Source/Observer</b></summary>
+
+<table ><tbody ><tr></tr><tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/twitter.png" width="20" height="20"><b>Twitter</b></summary><hr>
+
+```python
+from obsei.source.twitter_source import TwitterCredentials, TwitterSource, TwitterSourceConfig
+
+# initialize twitter source config
+source_config = TwitterSourceConfig(
+   keywords=["issue"], # Keywords, @user or #hashtags
+   lookup_period="1h", # Lookup period from current time, format: `<number><d|h|m>` (day|hour|minute)
+   cred_info=TwitterCredentials(
+       # Enter your twitter consumer key and secret. Get it from https://developer.twitter.com/en/apply-for-access
+       consumer_key="<twitter_consumer_key>",
+       consumer_secret="<twitter_consumer_secret>",
+       bearer_token='<ENTER BEARER TOKEN>',
+   )
+)
+
+# initialize tweets retriever
+source = TwitterSource()
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/Youtube.png" width="20" height="20"><b>Youtube Scrapper</b></summary><hr>
+
+```python
+from obsei.source.youtube_scrapper import YoutubeScrapperSource, YoutubeScrapperConfig
+
+# initialize Youtube source config
+source_config = YoutubeScrapperConfig(
+    video_url="https://www.youtube.com/watch?v=uZfns0JIlFk", # Youtube video URL
+    fetch_replies=True, # Fetch replies to comments
+    max_comments=10, # Total number of comments and replies to fetch
+    lookup_period="1Y", # Lookup period from current time, format: `<number><d|h|m|M|Y>` (day|hour|minute|month|year)
+)
+
+# initialize Youtube comments retriever
+source = YoutubeScrapperSource()
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/facebook.png" width="20" height="20"><b>Facebook</b></summary><hr>
+
+```python
+from obsei.source.facebook_source import FacebookCredentials, FacebookSource, FacebookSourceConfig
+
+# initialize facebook source config
+source_config = FacebookSourceConfig(
+   page_id="110844591144719", # Facebook page id, for example this one for Obsei
+   lookup_period="1h", # Lookup period from current time, format: `<number><d|h|m>` (day|hour|minute)
+   cred_info=FacebookCredentials(
+       # Enter your facebook app_id, app_secret and long_term_token. Get it from https://developers.facebook.com/apps/
+       app_id="<facebook_app_id>",
+       app_secret="<facebook_app_secret>",
+       long_term_token="<facebook_long_term_token>",
+   )
+)
+
+# initialize facebook post comments retriever
+source = FacebookSource()
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/gmail.png" width="20" height="20"><b>Email</b></summary><hr>
+
+```python
+from obsei.source.email_source import EmailConfig, EmailCredInfo, EmailSource
+
+# initialize email source config
+source_config = EmailConfig(
+   # List of IMAP servers for most commonly used email providers
+   # https://www.systoolsgroup.com/imap/
+   # Also, if you're using a Gmail account then make sure you allow less secure apps on your account -
+   # https://myaccount.google.com/lesssecureapps?pli=1
+   # Also enable IMAP access -
+   # https://mail.google.com/mail/u/0/#settings/fwdandpop
+   imap_server="imap.gmail.com", # Enter IMAP server
+   cred_info=EmailCredInfo(
+       # Enter your email account username and password
+       username="<email_username>",
+       password="<email_password>"
+   ),
+   lookup_period="1h" # Lookup period from current time, format: `<number><d|h|m>` (day|hour|minute)
+)
+
+# initialize email retriever
+source = EmailSource()
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/google_maps.png" width="20" height="20"><b>Google Maps Reviews Scrapper</b></summary><hr>
+
+```python
+from obsei.source.google_maps_reviews import OSGoogleMapsReviewsSource, OSGoogleMapsReviewsConfig
+
+# initialize Outscrapper Maps review source config
+source_config = OSGoogleMapsReviewsConfig(
+   # Collect API key from https://outscraper.com/
+   api_key="<Enter Your API Key>",
+   # Enter Google Maps link or place id
+   # For example below is for the "Taj Mahal"
+   queries=["https://www.google.co.in/maps/place/Taj+Mahal/@27.1751496,78.0399535,17z/data=!4m5!3m4!1s0x39747121d702ff6d:0xdd2ae4803f767dde!8m2!3d27.1751448!4d78.0421422"],
+   number_of_reviews=10,
+)
+
+
+# initialize Outscrapper Maps review retriever
+source = OSGoogleMapsReviewsSource()
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/appstore.png" width="20" height="20"><b>AppStore Reviews Scrapper</b></summary><hr>
+
+```python
+from obsei.source.appstore_scrapper import AppStoreScrapperConfig, AppStoreScrapperSource
+
+# initialize app store source config
+source_config = AppStoreScrapperConfig(
+   # Need two parameters app_id and country.
+   # `app_id` can be found at the end of the url of app in app store.
+   # For example - https://apps.apple.com/us/app/xcode/id497799835
+   # `310633997` is the app_id for xcode and `us` is country.
+   countries=["us"],
+   app_id="310633997",
+   lookup_period="1h" # Lookup period from current time, format: `<number><d|h|m>` (day|hour|minute)
+)
+
+
+# initialize app store reviews retriever
+source = AppStoreScrapperSource()
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/playstore.png" width="20" height="20"><b>Play Store Reviews Scrapper</b></summary><hr>
+
+```python
+from obsei.source.playstore_scrapper import PlayStoreScrapperConfig, PlayStoreScrapperSource
+
+# initialize play store source config
+source_config = PlayStoreScrapperConfig(
+   # Need two parameters package_name and country.
+   # `package_name` can be found at the end of the url of app in play store.
+   # For example - https://play.google.com/store/apps/details?id=com.google.android.gm&hl=en&gl=US
+   # `com.google.android.gm` is the package_name for xcode and `us` is country.
+   countries=["us"],
+   package_name="com.google.android.gm",
+   lookup_period="1h" # Lookup period from current time, format: `<number><d|h|m>` (day|hour|minute)
+)
+
+# initialize play store reviews retriever
+source = PlayStoreScrapperSource()
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/reddit.png" width="20" height="20"><b>Reddit</b></summary><hr>
+
+```python
+from obsei.source.reddit_source import RedditConfig, RedditSource, RedditCredInfo
+
+# initialize reddit source config
+source_config = RedditConfig(
+   subreddits=["wallstreetbets"], # List of subreddits
+   # Reddit account username and password
+   # You can also enter reddit client_id and client_secret or refresh_token
+   # Create credential at https://www.reddit.com/prefs/apps
+   # Also refer https://praw.readthedocs.io/en/latest/getting_started/authentication.html
+   # Currently Password Flow, Read Only Mode and Saved Refresh Token Mode are supported
+   cred_info=RedditCredInfo(
+       username="<reddit_username>",
+       password="<reddit_password>"
+   ),
+   lookup_period="1h" # Lookup period from current time, format: `<number><d|h|m>` (day|hour|minute)
+)
+
+# initialize reddit retriever
+source = RedditSource()
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/reddit.png" width="20" height="20"><b>Reddit Scrapper</b></summary><hr>
+
+<i>Note: Reddit heavily rate limit scrappers, hence use it to fetch small data during long period</i>
+
+```python
+from obsei.source.reddit_scrapper import RedditScrapperConfig, RedditScrapperSource
+
+# initialize reddit scrapper source config
+source_config = RedditScrapperConfig(
+   # Reddit subreddit, search etc rss url. For proper url refer following link -
+   # Refer https://www.reddit.com/r/pathogendavid/comments/tv8m9/pathogendavids_guide_to_rss_and_reddit/
+   url="https://www.reddit.com/r/wallstreetbets/comments/.rss?sort=new",
+   lookup_period="1h" # Lookup period from current time, format: `<number><d|h|m>` (day|hour|minute)
+)
+
+# initialize reddit retriever
+source = RedditScrapperSource()
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/googlenews.png" width="20" height="20"><b>Google News</b></summary><hr>
+
+```python
+from obsei.source.google_news_source import GoogleNewsConfig, GoogleNewsSource
+
+# initialize Google News source config
+source_config = GoogleNewsConfig(
+   query='bitcoin',
+   max_results=5,
+   # To fetch full article text enable `fetch_article` flag
+   # By default google news gives title and highlight
+   fetch_article=True,
+   # proxy='http://127.0.0.1:8080'
+)
+
+# initialize Google News retriever
+source = GoogleNewsSource()
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/webcrawler.png" width="20" height="20"><b>Web Crawler</b></summary><hr>
+
+```python
+from obsei.source.website_crawler_source import TrafilaturaCrawlerConfig, TrafilaturaCrawlerSource
+
+# initialize website crawler source config
+source_config = TrafilaturaCrawlerConfig(
+   urls=['https://obsei.github.io/obsei/']
+)
+
+# initialize website text retriever
+source = TrafilaturaCrawlerSource()
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/pandas.svg" width="20" height="20"><b>Pandas DataFrame</b></summary><hr>
+
+```python
+import pandas as pd
+from obsei.source.pandas_source import PandasSource, PandasSourceConfig
+
+# Initialize your Pandas DataFrame from your sources like csv, excel, sql etc
+# In following example we are reading csv which have two columns title and text
+csv_file = "https://raw.githubusercontent.com/deepset-ai/haystack/master/tutorials/small_generator_dataset.csv"
+dataframe = pd.read_csv(csv_file)
+
+# initialize pandas sink config
+sink_config = PandasSourceConfig(
+   dataframe=dataframe,
+   include_columns=["score"],
+   text_columns=["name", "degree"],
+)
+
+# initialize pandas sink
+sink = PandasSource()
+```
+
+</details>
+</td>
+</tr>
+</tbody>
+</table>
+
+</details>
+
+<details><summary><b>Step 2: Configure Analyzer</b></summary>
+
+<i>Note: To run transformers in an offline mode, check [transformers offline mode](https://huggingface.co/transformers/installation.html#offline-mode).</i>
+
+<p>Some analyzer support GPU and to utilize pass <b>device</b> parameter.
+List of possible values of <b>device</b> parameter (default value <i>auto</i>):
+<ol>
+    <li> <b>auto</b>: GPU (cuda:0) will be used if available otherwise CPU will be used
+    <li> <b>cpu</b>: CPU will be used
+    <li> <b>cuda:{id}</b> - GPU will be used with provided CUDA device id
+</ol>
+</p>
+
+<table ><tbody ><tr></tr><tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/classification.png" width="20" height="20"><b>Text Classification</b></summary><hr>
+
+Text classification: Classify text into user provided categories.
+
+```python
+from obsei.analyzer.classification_analyzer import ClassificationAnalyzerConfig, ZeroShotClassificationAnalyzer
+
+# initialize classification analyzer config
+# It can also detect sentiments if "positive" and "negative" labels are added.
+analyzer_config=ClassificationAnalyzerConfig(
+   labels=["service", "delay", "performance"],
+)
+
+# initialize classification analyzer
+# For supported models refer https://huggingface.co/models?filter=zero-shot-classification
+text_analyzer = ZeroShotClassificationAnalyzer(
+   model_name_or_path="typeform/mobilebert-uncased-mnli",
+   device="auto"
+)
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/sentiment.png" width="20" height="20"><b>Sentiment Analyzer</b></summary><hr>
+
+Sentiment Analyzer: Detect the sentiment of the text. Text classification can also perform sentiment analysis but if you don't want to use heavy-duty NLP model then use less resource hungry dictionary based Vader Sentiment detector.
+
+```python
+from obsei.analyzer.sentiment_analyzer import VaderSentimentAnalyzer
+
+# Vader does not need any configuration settings
+analyzer_config=None
+
+# initialize vader sentiment analyzer
+text_analyzer = VaderSentimentAnalyzer()
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/ner.png" width="20" height="20"><b>NER Analyzer</b></summary><hr>
+
+NER (Named-Entity Recognition) Analyzer: Extract information and classify named entities mentioned in text into pre-defined categories such as person names, organizations, locations, medical codes, time expressions, quantities, monetary values, percentages, etc
+
+```python
+from obsei.analyzer.ner_analyzer import NERAnalyzer
+
+# NER analyzer does not need configuration settings
+analyzer_config=None
+
+# initialize ner analyzer
+# For supported models refer https://huggingface.co/models?filter=token-classification
+text_analyzer = NERAnalyzer(
+   model_name_or_path="elastic/distilbert-base-cased-finetuned-conll03-english",
+   device = "auto"
+)
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/translator.png" width="20" height="20"><b>Translator</b></summary><hr>
+
+```python
+from obsei.analyzer.translation_analyzer import TranslationAnalyzer
+
+# Translator does not need analyzer config
+analyzer_config = None
+
+# initialize translator
+# For supported models refer https://huggingface.co/models?pipeline_tag=translation
+analyzer = TranslationAnalyzer(
+   model_name_or_path="Helsinki-NLP/opus-mt-hi-en",
+   device = "auto"
+)
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/pii.png" width="20" height="20"><b>PII Anonymizer</b></summary><hr>
+
+```python
+from obsei.analyzer.pii_analyzer import PresidioEngineConfig, PresidioModelConfig, \
+   PresidioPIIAnalyzer, PresidioPIIAnalyzerConfig
+
+# initialize pii analyzer's config
+analyzer_config = PresidioPIIAnalyzerConfig(
+   # Whether to return only pii analysis or anonymize text
+   analyze_only=False,
+   # Whether to return detail information about anonymization decision
+   return_decision_process=True
+)
+
+# initialize pii analyzer
+analyzer = PresidioPIIAnalyzer(
+   engine_config=PresidioEngineConfig(
+       # spacy and stanza nlp engines are supported
+       # For more info refer
+       # https://microsoft.github.io/presidio/analyzer/developing_recognizers/#utilize-spacy-or-stanza
+       nlp_engine_name="spacy",
+       # Update desired spacy model and language
+       models=[PresidioModelConfig(model_name="en_core_web_lg", lang_code="en")]
+   )
+)
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/dummy.png" width="20" height="20"><b>Dummy Analyzer</b></summary><hr>
+
+Dummy Analyzer: Does nothing. Its simply used for transforming the input (TextPayload) to output (TextPayload) and adding the user supplied dummy data.
+
+```python
+from obsei.analyzer.dummy_analyzer import DummyAnalyzer, DummyAnalyzerConfig
+
+# initialize dummy analyzer's configuration settings
+analyzer_config = DummyAnalyzerConfig()
+
+# initialize dummy analyzer
+analyzer = DummyAnalyzer()
+```
+
+</details>
+</td>
+</tr>
+</tbody>
+</table>
+
+</details>
+
+<details><summary><b>Step 3: Configure Sink/Informer</b></summary>
+
+<table ><tbody ><tr></tr><tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/slack.svg" width="25" height="25"><b>Slack</b></summary><hr>
+
+```python
+from obsei.sink.slack_sink import SlackSink, SlackSinkConfig
+
+# initialize slack sink config
+sink_config = SlackSinkConfig(
+   # Provide slack bot/app token
+   # For more detail refer https://slack.com/intl/en-de/help/articles/215770388-Create-and-regenerate-API-tokens
+   slack_token="<Slack_app_token>",
+   # To get channel id refer https://stackoverflow.com/questions/40940327/what-is-the-simplest-way-to-find-a-slack-team-id-and-a-channel-id
+   channel_id="C01LRS6CT9Q"
+)
+
+# initialize slack sink
+sink = SlackSink()
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/zendesk.png" width="20" height="20"><b>Zendesk</b></summary><hr>
+
+```python
+from obsei.sink.zendesk_sink import ZendeskSink, ZendeskSinkConfig, ZendeskCredInfo
+
+# initialize zendesk sink config
+sink_config = ZendeskSinkConfig(
+   # provide zendesk domain
+   domain="zendesk.com",
+   # provide subdomain if you have one
+   subdomain=None,
+   # Enter zendesk user details
+   cred_info=ZendeskCredInfo(
+       email="<zendesk_user_email>",
+       password="<zendesk_password>"
+   )
+)
+
+# initialize zendesk sink
+sink = ZendeskSink()
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/jira.png" width="20" height="20"><b>Jira</b></summary><hr>
+
+```python
+from obsei.sink.jira_sink import JiraSink, JiraSinkConfig
+
+# For testing purpose you can start jira server locally
+# Refer https://developer.atlassian.com/server/framework/atlassian-sdk/atlas-run-standalone/
+
+# initialize Jira sink config
+sink_config = JiraSinkConfig(
+   url="http://localhost:2990/jira", # Jira server url
+    # Jira username & password for user who have permission to create issue
+   username="<username>",
+   password="<password>",
+   # Which type of issue to be created
+   # For more information refer https://support.atlassian.com/jira-cloud-administration/docs/what-are-issue-types/
+   issue_type={"name": "Task"},
+   # Under which project issue to be created
+   # For more information refer https://support.atlassian.com/jira-software-cloud/docs/what-is-a-jira-software-project/
+   project={"key": "CUS"},
+)
+
+# initialize Jira sink
+sink = JiraSink()
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/elastic.png" width="20" height="20"><b>ElasticSearch</b></summary><hr>
+
+```python
+from obsei.sink.elasticsearch_sink import ElasticSearchSink, ElasticSearchSinkConfig
+
+# For testing purpose you can start Elasticsearch server locally via docker
+# `docker run -d --name elasticsearch -p 9200:9200 -e "discovery.type=single-node" elasticsearch:8.5.0`
+
+# initialize Elasticsearch sink config
+sink_config = ElasticSearchSinkConfig(
+   # Elasticsearch server
+   hosts="http://localhost:9200",
+   # Index name, it will create if not exist
+   index_name="test",
+)
+
+# initialize Elasticsearch sink
+sink = ElasticSearchSink()
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/http_api.png" width="20" height="20"><b>Http</b></summary><hr>
+
+```python
+from obsei.sink.http_sink import HttpSink, HttpSinkConfig
+
+# For testing purpose you can create mock http server via postman
+# For more details refer https://learning.postman.com/docs/designing-and-developing-your-api/mocking-data/setting-up-mock/
+
+# initialize http sink config (Currently only POST call is supported)
+sink_config = HttpSinkConfig(
+   # provide http server url
+   url="https://localhost:8080/api/path",
+   # Here you can add headers you would like to pass with request
+   headers={
+       "Content-type": "application/json"
+   }
+)
+
+# To modify or converting the payload, create convertor class
+# Refer obsei.sink.dailyget_sink.PayloadConvertor for example
+
+# initialize http sink
+sink = HttpSink()
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/pandas.svg" width="20" height="20"><b>Pandas DataFrame</b></summary><hr>
+
+```python
+from pandas import DataFrame
+from obsei.sink.pandas_sink import PandasSink, PandasSinkConfig
+
+# initialize pandas sink config
+sink_config = PandasSinkConfig(
+   dataframe=DataFrame()
+)
+
+# initialize pandas sink
+sink = PandasSink()
+```
+
+</details>
+</td>
+</tr>
+<tr>
+<td><details ><summary><img style="vertical-align:middle;margin:2px 10px" src="https://raw.githubusercontent.com/obsei/obsei-resources/master/logos/logger.png" width="20" height="20"><b>Logger</b></summary><hr>
+
+This is useful for testing and dry running the pipeline.
+
+```python
+from obsei.sink.logger_sink import LoggerSink, LoggerSinkConfig
+import logging
+import sys
+
+logger = logging.getLogger("Obsei")
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+# initialize logger sink config
+sink_config = LoggerSinkConfig(
+   logger=logger,
+   level=logging.INFO
+)
+
+# initialize logger sink
+sink = LoggerSink()
+```
+
+</details>
+</td>
+</tr>
+</tbody>
+</table>
+
+</details>
+
+<details><summary><b>Step 4: Join and create workflow</b></summary>
+
+`source` will fetch data from the selected source, then feed it to the `analyzer` for processing, whose output we feed into a `sink` to get notified at that sink.
+
+```python
+# Uncomment if you want logger
+# import logging
+# import sys
+# logger = logging.getLogger(__name__)
+# logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+# This will fetch information from configured source ie twitter, app store etc
+source_response_list = source.lookup(source_config)
+
+# Uncomment if you want to log source response
+# for idx, source_response in enumerate(source_response_list):
+#     logger.info(f"source_response#'{idx}'='{source_response.__dict__}'")
+
+# This will execute analyzer (Sentiment, classification etc) on source data with provided analyzer_config
+analyzer_response_list = text_analyzer.analyze_input(
+    source_response_list=source_response_list,
+    analyzer_config=analyzer_config
+)
+
+# Uncomment if you want to log analyzer response
+# for idx, an_response in enumerate(analyzer_response_list):
+#    logger.info(f"analyzer_response#'{idx}'='{an_response.__dict__}'")
+
+# Analyzer output added to segmented_data
+# Uncomment to log it
+# for idx, an_response in enumerate(analyzer_response_list):
+#    logger.info(f"analyzed_data#'{idx}'='{an_response.segmented_data.__dict__}'")
+
+# This will send analyzed output to configure sink ie Slack, Zendesk etc
+sink_response_list = sink.send_data(analyzer_response_list, sink_config)
+
+# Uncomment if you want to log sink response
+# for sink_response in sink_response_list:
+#     if sink_response is not None:
+#         logger.info(f"sink_response='{sink_response}'")
+```
+
+</details>
+
+<details><summary><b>Step 5: Execute workflow</b></summary>
+Copy the code snippets from <b>Steps 1 to 4</b> into a python file, for example <code>example.py</code> and execute the following command -
+
+```shell
+python example.py
+```
+
+</details>
+
+## Demo
+
+We have a minimal [streamlit](https://streamlit.io/) based UI that you can use to test Obsei.
+
+![Screenshot](https://raw.githubusercontent.com/obsei/obsei-resources/master/images/obsei-ui-demo.png)
+
+### Watch UI demo video
+
+[![Introductory and demo video](https://img.youtube.com/vi/GTF-Hy96gvY/2.jpg)](https://www.youtube.com/watch?v=GTF-Hy96gvY)
+
+Check demo at [![](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-blue)](https://huggingface.co/spaces/obsei/obsei-demo)
+
+(**Note**: Sometimes the Streamlit demo might not work due to rate limiting, use the docker image (locally) in such cases.)
+
+To test locally, just run
+
+```
+docker run -d --name obesi-ui -p 8501:8501 obsei/obsei-ui-demo
+
+# You can find the UI at http://localhost:8501
+```
+
+**To run Obsei workflow easily using GitHub Actions (no sign ups and cloud hosting required), refer to this [repo](https://github.com/obsei/demo-workflow-action)**.
+
+## Companies/Projects using Obsei
+
+Here are some companies/projects (alphabetical order) using Obsei. To add your company/project to the list, please raise a PR or contact us via [email](contact@obsei.com).
+
+- [Oraika](https://www.oraika.com): Contextually understand customer feedback
+- [1Page](https://www.get1page.com/): Giving a better context in meetings and calls
+- [Spacepulse](http://spacepulse.in/): The operating system for spaces
+- [Superblog](https://superblog.ai/): A blazing fast alternative to WordPress and Medium
+- [Zolve](https://zolve.com/): Creating a financial world beyond borders
+- [Utilize](https://www.utilize.app/): No-code app builder for businesses with a deskless workforce
+
+## Articles
+
+<table>
+<thead>
+<tr class="header">
+<th>Sr. No.</th>
+<th>Title</th>
+<th>Author</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>1</td>
+<td>
+    <a href="https://reenabapna.medium.com/ai-based-comparative-customer-feedback-analysis-using-deep-learning-models-def0dc77aaee">AI based Comparative Customer Feedback Analysis Using Obsei</a>
+</td>
+<td>
+    <a href="linkedin.com/in/reena-bapna-66a8691a">Reena Bapna</a>
+</td>
+</tr>
+<tr>
+<td>2</td>
+<td>
+    <a href="https://medium.com/mlearning-ai/linkedin-app-user-feedback-analysis-9c9f98464daa">LinkedIn App - User Feedback Analysis</a>
+</td>
+<td>
+    <a href="http://www.linkedin.com/in/himanshusharmads">Himanshu Sharma</a>
+</td>
+</tr>
+</tbody>
+</table>
+
+## Tutorials
+
+<table>
+<thead>
+<tr class="header">
+<th>Sr. No.</th>
+<th>Workflow</th>
+<th>Colab</th>
+<th>Binder</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td rowspan="2">1</td>
+<td colspan="3">Observe app reviews from Google play store, Analyze them by performing text classification and then Inform them on console via logger</td>
+</tr>
+<tr>
+<td>PlayStore Reviews → Classification → Logger</td>
+<td>
+    <a href="https://colab.research.google.com/github/obsei/obsei/blob/master/tutorials/01_PlayStore_Classification_Logger.ipynb">
+        <img alt="Colab" src="https://colab.research.google.com/assets/colab-badge.svg">
+    </a>
+</td>
+<td>
+    <a href="https://mybinder.org/v2/gh/obsei/obsei/HEAD?filepath=tutorials%2F01_PlayStore_Classification_Logger.ipynb">
+        <img alt="Colab" src="https://mybinder.org/badge_logo.svg">
+    </a>
+</td>
+</tr>
+<tr>
+<td rowspan="2">2</td>
+<td colspan="3">Observe app reviews from Google play store, PreProcess text via various text cleaning functions, Analyze them by performing text classification, Inform them to Pandas DataFrame and store resultant CSV to Google Drive</td>
+</tr>
+<tr>
+<td>PlayStore Reviews → PreProcessing → Classification → Pandas DataFrame → CSV in Google Drive</td>
+<td>
+    <a href="https://colab.research.google.com/github/obsei/obsei/blob/master/tutorials/02_PlayStore_PreProc_Classification_Pandas.ipynb">
+        <img alt="Colab" src="https://colab.research.google.com/assets/colab-badge.svg">
+    </a>
+</td>
+<td>
+    <a href="https://mybinder.org/v2/gh/obsei/obsei/HEAD?filepath=tutorials%2F02_PlayStore_PreProc_Classification_Pandas.ipynb">
+        <img alt="Colab" src="https://mybinder.org/badge_logo.svg">
+    </a>
+</td>
+</tr>
+<tr>
+<td rowspan="2">3</td>
+<td colspan="3">Observe app reviews from Apple app store, PreProcess text via various text cleaning function, Analyze them by performing text classification, Inform them to Pandas DataFrame and store resultant CSV to Google Drive</td>
+</tr>
+<tr>
+<td>AppStore Reviews → PreProcessing → Classification → Pandas DataFrame → CSV in Google Drive</td>
+<td>
+    <a href="https://colab.research.google.com/github/obsei/obsei/blob/master/tutorials/03_AppStore_PreProc_Classification_Pandas.ipynb">
+        <img alt="Colab" src="https://colab.research.google.com/assets/colab-badge.svg">
+    </a>
+</td>
+<td>
+    <a href="https://mybinder.org/v2/gh/obsei/obsei/HEAD?filepath=tutorials%2F03_AppStore_PreProc_Classification_Pandas.ipynb">
+        <img alt="Colab" src="https://mybinder.org/badge_logo.svg">
+    </a>
+</td>
+</tr>
+<tr>
+<td rowspan="2">4</td>
+<td colspan="3">Observe news article from Google news, PreProcess text via various text cleaning function, Analyze them via performing text classification while splitting text in small chunks and later computing final inference using given formula</td>
+</tr>
+<tr>
+<td>Google News → Text Cleaner → Text Splitter → Classification → Inference Aggregator</td>
+<td>
+    <a href="https://colab.research.google.com/github/obsei/obsei/blob/master/tutorials/04_GoogleNews_Cleaner_Splitter_Classification_Aggregator.ipynb">
+        <img alt="Colab" src="https://colab.research.google.com/assets/colab-badge.svg">
+    </a>
+</td>
+<td>
+    <a href="https://mybinder.org/v2/gh/obsei/obsei/HEAD?filepath=tutorials%2F04_GoogleNews_Cleaner_Splitter_Classification_Aggregator.ipynb">
+        <img alt="Colab" src="https://mybinder.org/badge_logo.svg">
+    </a>
+</td>
+</tr>
+</tbody>
+</table>
+
+<details><summary><b>💡Tips: Handle large text classification via Obsei</b></summary>
+
+![](https://raw.githubusercontent.com/obsei/obsei-resources/master/gifs/Long_Text_Classification.gif)
+
+</details>
+
+## Documentation
+
+For detailed installation instructions, usages and examples, refer to our [documentation](https://obsei.github.io/obsei/).
+
+## Support and Release Matrix
+
+<table>
+<thead>
+<tr class="header">
+<th></th>
+<th>Linux</th>
+<th>Mac</th>
+<th>Windows</th>
+<th>Remark</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>Tests</td>
+<td style="text-align:center">✅</td>
+<td style="text-align:center">✅</td>
+<td style="text-align:center">✅</td>
+<td>Low Coverage as difficult to test 3rd party libs</td>
+</tr>
+<tr>
+<td>PIP</td>
+<td style="text-align:center">✅</td>
+<td style="text-align:center">✅</td>
+<td style="text-align:center">✅</td>
+<td>Fully Supported</td>
+</tr>
+<tr>
+<td>Conda</td>
+<td style="text-align:center">❌</td>
+<td style="text-align:center">❌</td>
+<td style="text-align:center">❌</td>
+<td>Not Supported</td>
+</tr>
+</tbody>
+</table>
+
+## Discussion forum
+
+Discussion about _Obsei_ can be done at [community forum](https://github.com/obsei/obsei/discussions)
+
+## Changelogs
+
+Refer [releases](https://github.com/obsei/obsei/releases) for changelogs
+
+## Security Issue
+
+For any security issue please contact us via [email](mailto:contact@oraika.com)
+
+## Stargazers over time
+
+[![Stargazers over time](https://starchart.cc/obsei/obsei.svg)](https://starchart.cc/obsei/obsei)
+
+## Maintainers
+
+This project is being maintained by [Oraika Technologies](https://www.oraika.com). [Lalit Pagaria](https://github.com/lalitpagaria) and [Girish Patel](https://github.com/GirishPatel) are maintainers of this project.
+
+## License
+
+- Copyright holder: [Oraika Technologies](https://www.oraika.com)
+- Overall Apache 2.0 and you can read [License](https://github.com/obsei/obsei/blob/master/LICENSE) file.
+- Multiple other secondary permissive or weak copyleft licenses (LGPL, MIT, BSD etc.) for third-party components refer [Attribution](https://github.com/obsei/obsei/blob/master/ATTRIBUTION.md).
+- To make project more commercial friendly, we void third party components which have strong copyleft licenses (GPL, AGPL etc.) into the project.
+
+## Attribution
+
+This could not have been possible without these [open source softwares](https://github.com/obsei/obsei/blob/master/ATTRIBUTION.md).
+
+## Contribution
+
+First off, thank you for even considering contributing to this package, every contribution big or small is greatly appreciated.
+Please refer our [Contribution Guideline](https://github.com/obsei/obsei/blob/master/CONTRIBUTING.md) and [Code of Conduct](https://github.com/obsei/obsei/blob/master/CODE_OF_CONDUCT.md).
+
+Thanks so much to all our contributors
+
+<a href="https://github.com/obsei/obsei/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=obsei/obsei" />
+</a>

obsei_module/SECURITY.md

@@ -0,0 +1,5 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+For any security issue please report it via [email](mailto:contact@oraika.com).

obsei_module/_config.yml

@@ -0,0 +1,9 @@
+theme: jekyll-theme-primer
+markdown: CommonMarkGhPages
+commonmark:
+  options: ["UNSAFE", "SMART", "FOOTNOTES"]
+  extensions: ["strikethrough", "autolink", "table", "tagfilter"]
+title: "Obsei: An open-source low-code AI powered automation tool"
+description: "Obsei is an open-source low-code AI powered automation tool"
+
+google_analytics: G-0E2FTKBK4T

obsei_module/_includes/head-custom-google-analytics.html

@@ -0,0 +1,9 @@
+<!-- Global site tag (gtag.js) - Google Analytics -->
+<script async src="https://www.googletagmanager.com/gtag/js?id=G-0E2FTKBK4T"></script>
+<script>
+  window.dataLayer = window.dataLayer || [];
+  function gtag(){dataLayer.push(arguments);}
+  gtag('js', new Date());
+
+  gtag('config', 'G-0E2FTKBK4T');
+</script>

obsei_module/binder/requirements.txt

@@ -0,0 +1,2 @@
+git+https://github.com/obsei/obsei@master#egg=obsei[all]
+trafilatura

obsei_module/example/app_store_scrapper_example.py

@@ -0,0 +1,41 @@
+import logging
+import sys
+from datetime import datetime, timedelta
+
+import pytz
+
+from obsei.analyzer.classification_analyzer import ClassificationAnalyzerConfig, ZeroShotClassificationAnalyzer
+from obsei.misc.utils import DATETIME_STRING_PATTERN
+from obsei.source.appstore_scrapper import (
+    AppStoreScrapperConfig,
+    AppStoreScrapperSource,
+)
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+since_time = datetime.utcnow().astimezone(pytz.utc) + timedelta(days=-5)
+source_config = AppStoreScrapperConfig(
+    app_url='https://apps.apple.com/us/app/gmail-email-by-google/id422689480',
+    lookup_period=since_time.strftime(DATETIME_STRING_PATTERN),
+    max_count=10,
+)
+
+source = AppStoreScrapperSource()
+
+text_analyzer = ZeroShotClassificationAnalyzer(
+    model_name_or_path="typeform/mobilebert-uncased-mnli", device="auto"
+)
+
+source_response_list = source.lookup(source_config)
+for idx, source_response in enumerate(source_response_list):
+    logger.info(f"source_response#'{idx}'='{source_response.__dict__}'")
+
+analyzer_response_list = text_analyzer.analyze_input(
+    source_response_list=source_response_list,
+    analyzer_config=ClassificationAnalyzerConfig(
+        labels=["interface", "slow", "battery"],
+    ),
+)
+for idx, an_response in enumerate(analyzer_response_list):
+    logger.info(f"analyzer_response#'{idx}'='{an_response.__dict__}'")

obsei_module/example/daily_get_example.py

@@ -0,0 +1,77 @@
+import logging
+import os
+import sys
+from pathlib import Path
+
+from obsei.sink.dailyget_sink import DailyGetSink, DailyGetSinkConfig
+from obsei.source.twitter_source import TwitterSource, TwitterSourceConfig
+from obsei.analyzer.classification_analyzer import (
+    ClassificationAnalyzerConfig,
+    ZeroShotClassificationAnalyzer,
+)
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+sink_config = DailyGetSinkConfig(
+    url=os.environ["DAILYGET_URL"],
+    partner_id=os.environ["DAILYGET_PARTNER_ID"],
+    consumer_phone_number=os.environ["DAILYGET_CONSUMER_NUMBER"],
+    source_information="Twitter " + os.environ["DAILYGET_QUERY"],
+    base_payload={
+        "partnerId": os.environ["DAILYGET_PARTNER_ID"],
+        "consumerPhoneNumber": os.environ["DAILYGET_CONSUMER_NUMBER"],
+    },
+)
+
+dir_path = Path(__file__).resolve().parent.parent
+source_config = TwitterSourceConfig(
+    keywords=[os.environ["DAILYGET_QUERY"]],
+    lookup_period=os.environ["DAILYGET_LOOKUP_PERIOD"],
+    tweet_fields=[
+        "author_id",
+        "conversation_id",
+        "created_at",
+        "id",
+        "public_metrics",
+        "text",
+    ],
+    user_fields=["id", "name", "public_metrics", "username", "verified"],
+    expansions=["author_id"],
+    place_fields=None,
+    max_tweets=10,
+)
+
+source = TwitterSource()
+sink = DailyGetSink()
+text_analyzer = ZeroShotClassificationAnalyzer(
+    model_name_or_path="joeddav/bart-large-mnli-yahoo-answers",
+    #   model_name_or_path="joeddav/xlm-roberta-large-xnli",
+)
+
+source_response_list = source.lookup(source_config)
+for idx, source_response in enumerate(source_response_list):
+    logger.info(f"source_response#'{idx}'='{source_response.__dict__}'")
+
+analyzer_response_list = text_analyzer.analyze_input(
+    source_response_list=source_response_list,
+    analyzer_config=ClassificationAnalyzerConfig(
+        labels=[
+            "service",
+            "delay",
+            "tracking",
+            "no response",
+            "missing items",
+            "delivery",
+            "mask",
+        ],
+    ),
+)
+for idx, an_response in enumerate(analyzer_response_list):
+    logger.info(f"analyzer_response#'{idx}'='{an_response.__dict__}'")
+
+# HTTP Sink
+sink_response_list = sink.send_data(analyzer_response_list, sink_config)
+for sink_response in sink_response_list:
+    if sink_response is not None:
+        logger.info(f"sink_response='{sink_response.__dict__}'")

obsei_module/example/elasticsearch_example.py

@@ -0,0 +1,69 @@
+import logging
+import sys
+from pathlib import Path
+
+from obsei.sink.elasticsearch_sink import ElasticSearchSink, ElasticSearchSinkConfig
+from obsei.source.twitter_source import TwitterSource, TwitterSourceConfig
+from obsei.analyzer.classification_analyzer import (
+    ClassificationAnalyzerConfig,
+    ZeroShotClassificationAnalyzer,
+)
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+dir_path = Path(__file__).resolve().parent.parent
+source_config = TwitterSourceConfig(
+    keywords="@Handle",
+    lookup_period="1h",  # 1 Hour
+    tweet_fields=[
+        "author_id",
+        "conversation_id",
+        "created_at",
+        "id",
+        "public_metrics",
+        "text",
+    ],
+    user_fields=["id", "name", "public_metrics", "username", "verified"],
+    expansions=["author_id"],
+    place_fields=None,
+    max_tweets=10,
+)
+
+source = TwitterSource()
+text_analyzer = ZeroShotClassificationAnalyzer(
+    model_name_or_path="joeddav/bart-large-mnli-yahoo-answers",
+)
+
+# Start Elasticsearch server locally
+# `docker run -d --name elasticsearch -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.9.2`
+sink_config = ElasticSearchSinkConfig(
+    host="localhost",
+    port=9200,
+    index_name="test",
+)
+
+source_response_list = source.lookup(source_config)
+for idx, source_response in enumerate(source_response_list):
+    logger.info(f"source_response#'{idx}'='{source_response.__dict__}'")
+
+analyzer_response_list = text_analyzer.analyze_input(
+    source_response_list=source_response_list,
+    analyzer_config=ClassificationAnalyzerConfig(
+        labels=[
+            "service",
+            "delay",
+            "tracking",
+            "no response",
+            "missing items",
+            "delivery",
+            "mask",
+        ],
+    ),
+)
+for idx, an_response in enumerate(analyzer_response_list):
+    logger.info(f"analyzer_response#'{idx}'='{an_response.__dict__}'")
+
+sink = ElasticSearchSink()
+sink_response = sink.send_data(analyzer_response_list, sink_config)
+logger.info(f"sink_response='{sink_response}'")

obsei_module/example/email_source_example.py

@@ -0,0 +1,36 @@
+import logging
+import os
+import sys
+from datetime import datetime, timedelta
+
+import pytz
+
+from obsei.misc.utils import DATETIME_STRING_PATTERN
+from obsei.source.email_source import EmailConfig, EmailCredInfo, EmailSource
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+since_time = datetime.utcnow().astimezone(pytz.utc) + timedelta(hours=-10)
+
+# List of IMAP servers for most commonly used email providers
+# https://www.systoolsgroup.com/imap/
+# Also, if you're using a Gmail account then make sure you allow less secure apps on your account -
+# https://myaccount.google.com/lesssecureapps?pli=1
+# Also enable IMAP access -
+# https://mail.google.com/mail/u/0/#settings/fwdandpop
+source_config = EmailConfig(
+    imap_server="imap.gmail.com",
+    cred_info=EmailCredInfo(
+        # It will fetch username and password from environment variable
+        username=os.environ.get("email_username"),
+        password=os.environ.get("email_password"),
+    ),
+    lookup_period=since_time.strftime(DATETIME_STRING_PATTERN),
+)
+
+source = EmailSource()
+source_response_list = source.lookup(source_config)
+
+for source_response in source_response_list:
+    logger.info(source_response.__dict__)

obsei_module/example/facebook_example.py

@@ -0,0 +1,19 @@
+import logging
+import sys
+
+from obsei.source.facebook_source import FacebookSource, FacebookSourceConfig
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+source_config = FacebookSourceConfig(page_id="110844591144719", lookup_period="2M")
+source = FacebookSource()
+source_response_list = source.lookup(source_config)
+
+logger.info("DETAILS:")
+for source_response in source_response_list:
+    logger.info(source_response)
+
+logger.info("TEXT:")
+for source_response in source_response_list:
+    logger.info(source_response.processed_text)

obsei_module/example/google_news_example.py

@@ -0,0 +1,58 @@
+from obsei.analyzer.classification_analyzer import (
+    ClassificationAnalyzerConfig,
+    ZeroShotClassificationAnalyzer,
+)
+from obsei.source.google_news_source import GoogleNewsConfig, GoogleNewsSource
+
+# Only fetch title and highlight
+source_config_without_full_text = GoogleNewsConfig(
+    query="ai",
+    max_results=150,
+    after_date='2023-12-01',
+    before_date='2023-12-31',
+)
+
+# Fetch full news article
+source_config_with_full_text = GoogleNewsConfig(
+    query="ai",
+    max_results=5,
+    fetch_article=True,
+    lookup_period="1d",
+    # proxy="http://127.0.0.1:8080"
+)
+
+source = GoogleNewsSource()
+
+analyzer_config = ClassificationAnalyzerConfig(
+    labels=["buy", "sell", "going up", "going down"],
+)
+
+text_analyzer = ZeroShotClassificationAnalyzer(
+    model_name_or_path="typeform/mobilebert-uncased-mnli", device="auto"
+)
+
+news_articles_without_full_text = source.lookup(source_config_without_full_text)
+
+news_articles_with_full_text = source.lookup(source_config_with_full_text)
+
+
+analyzer_responses_without_full_text = text_analyzer.analyze_input(
+    source_response_list=news_articles_without_full_text,
+    analyzer_config=analyzer_config,
+)
+
+analyzer_responses_with_full_text = text_analyzer.analyze_input(
+    source_response_list=news_articles_with_full_text, analyzer_config=analyzer_config
+)
+
+for article in news_articles_without_full_text:
+    print(article.__dict__)
+
+for response in analyzer_responses_without_full_text:
+    print(response.__dict__)
+
+for article in news_articles_with_full_text:
+    print(article.__dict__)
+
+for response in analyzer_responses_with_full_text:
+    print(response.__dict__)

obsei_module/example/jira_example.py

@@ -0,0 +1,77 @@
+# Jira Sink
+import logging
+import os
+import sys
+from pathlib import Path
+
+from pydantic import SecretStr
+
+from obsei.sink.jira_sink import JiraSink, JiraSinkConfig
+from obsei.source.twitter_source import (
+    TwitterCredentials,
+    TwitterSource,
+    TwitterSourceConfig,
+)
+from obsei.analyzer.classification_analyzer import (
+    ClassificationAnalyzerConfig,
+    ZeroShotClassificationAnalyzer,
+)
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+dir_path = Path(__file__).resolve().parent.parent
+source_config = TwitterSourceConfig(
+    keywords=["facing issue"],
+    lookup_period="1h",
+    tweet_fields=[
+        "author_id",
+        "conversation_id",
+        "created_at",
+        "id",
+        "public_metrics",
+        "text",
+    ],
+    user_fields=["id", "name", "public_metrics", "username", "verified"],
+    expansions=["author_id"],
+    place_fields=None,
+    max_tweets=10,
+    cred_info=TwitterCredentials(
+        consumer_key=SecretStr(os.environ["twitter_consumer_key"]),
+        consumer_secret=SecretStr(os.environ["twitter_consumer_secret"]),
+    ),
+)
+
+source = TwitterSource()
+
+# To start jira server locally `atlas-run-standalone --product jira`
+jira_sink_config = JiraSinkConfig(
+    url="http://localhost:2990/jira",
+    username=SecretStr("admin"),
+    password=SecretStr("admin"),
+    issue_type={"name": "Task"},
+    project={"key": "CUS"},
+)
+jira_sink = JiraSink()
+
+text_analyzer = ZeroShotClassificationAnalyzer(
+    model_name_or_path="joeddav/bart-large-mnli-yahoo-answers"
+)
+
+source_response_list = source.lookup(source_config)
+for idx, source_response in enumerate(source_response_list):
+    logger.info(f"source_response#'{idx}'='{source_response.__dict__}'")
+
+analyzer_response_list = text_analyzer.analyze_input(
+    source_response_list=source_response_list,
+    analyzer_config=ClassificationAnalyzerConfig(
+        labels=["service", "delay", "performance"],
+    ),
+)
+for idx, an_response in enumerate(analyzer_response_list):
+    logger.info(f"analyzer_response#'{idx}'='{an_response.__dict__}'")
+
+sink_response_list = jira_sink.send_data(analyzer_response_list, jira_sink_config)
+for sink_response in sink_response_list:
+    if sink_response is not None:
+        logger.info(f"sink_response='{sink_response}'")

obsei_module/example/maps_review_scrapper_example.py

@@ -0,0 +1,22 @@
+import logging
+import sys
+
+from obsei.source.google_maps_reviews import (OSGoogleMapsReviewsConfig,
+                                              OSGoogleMapsReviewsSource)
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+source_config = OSGoogleMapsReviewsConfig(
+    api_key="<Enter Your API Key>",  # Get API key from https://outscraper.com/
+    queries=[
+        "https://www.google.co.in/maps/place/Taj+Mahal/@27.1751496,78.0399535,17z/data=!4m5!3m4!1s0x39747121d702ff6d:0xdd2ae4803f767dde!8m2!3d27.1751448!4d78.0421422"
+    ],
+    number_of_reviews=3,
+)
+
+source = OSGoogleMapsReviewsSource()
+
+source_response_list = source.lookup(source_config)
+for source_response in source_response_list:
+    logger.info(source_response.__dict__)

obsei_module/example/pandas_sink_example.py

@@ -0,0 +1,49 @@
+import logging
+import sys
+
+from pandas import DataFrame
+
+from obsei.analyzer.classification_analyzer import (
+    ClassificationAnalyzerConfig,
+    ZeroShotClassificationAnalyzer,
+)
+from obsei.sink.pandas_sink import PandasSink, PandasSinkConfig
+from obsei.source.playstore_scrapper import (
+    PlayStoreScrapperConfig,
+    PlayStoreScrapperSource,
+)
+
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+source_config = PlayStoreScrapperConfig(
+    countries=["us"], package_name="com.apcoaconnect", max_count=3
+)
+
+source = PlayStoreScrapperSource()
+
+text_analyzer = ZeroShotClassificationAnalyzer(
+    model_name_or_path="typeform/mobilebert-uncased-mnli", device="auto"
+)
+
+# initialize pandas sink config
+sink_config = PandasSinkConfig(dataframe=DataFrame())
+
+# initialize pandas sink
+sink = PandasSink()
+
+source_response_list = source.lookup(source_config)
+
+analyzer_response_list = text_analyzer.analyze_input(
+    source_response_list=source_response_list,
+    analyzer_config=ClassificationAnalyzerConfig(
+        labels=["no parking", "registration issue", "app issue", "payment issue"],
+    ),
+)
+
+dataframe = sink.send_data(
+    analyzer_responses=analyzer_response_list, config=sink_config
+)
+
+print(dataframe.to_csv())

obsei_module/example/pandas_source_example.py

@@ -0,0 +1,27 @@
+import pandas as pd
+
+from obsei.source.pandas_source import (
+    PandasSourceConfig,
+    PandasSource,
+)
+import logging
+import sys
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+# Initialize your Pandas DataFrame from your sources like csv, excel, sql etc
+# In following example we are reading csv which have two columns title and text
+csv_file = "https://raw.githubusercontent.com/deepset-ai/haystack/master/tutorials/small_generator_dataset.csv"
+dataframe = pd.read_csv(csv_file)
+
+source_config = PandasSourceConfig(
+    dataframe=dataframe,
+    include_columns=["title"],
+    text_columns=["text"],
+)
+source = PandasSource()
+
+source_response_list = source.lookup(source_config)
+for idx, source_response in enumerate(source_response_list):
+    logger.info(f"source_response#'{idx}'='{source_response.__dict__}'")

obsei_module/example/pii_analyzer_example.py

@@ -0,0 +1,33 @@
+import logging
+import sys
+
+from obsei.payload import TextPayload
+from obsei.analyzer.pii_analyzer import (
+    PresidioEngineConfig,
+    PresidioModelConfig,
+    PresidioPIIAnalyzer,
+    PresidioPIIAnalyzerConfig,
+)
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+analyzer_config = PresidioPIIAnalyzerConfig(
+    analyze_only=False, return_decision_process=True
+)
+analyzer = PresidioPIIAnalyzer(
+    engine_config=PresidioEngineConfig(
+        nlp_engine_name="spacy",
+        models=[PresidioModelConfig(model_name="en_core_web_lg", lang_code="en")],
+    )
+)
+
+text_to_anonymize = "His name is Mr. Jones and his phone number is 212-555-5555"
+
+analyzer_results = analyzer.analyze_input(
+    source_response_list=[TextPayload(processed_text=text_to_anonymize)],
+    analyzer_config=analyzer_config,
+)
+
+for analyzer_result in analyzer_results:
+    logging.info(analyzer_result.to_dict())

obsei_module/example/play_store_reviews_example.py

@@ -0,0 +1,4 @@
+# TDB
+
+# Need proper service account file to test the changes :(
+print("TBD")

obsei_module/example/playstore_scrapper_example.py

@@ -0,0 +1,40 @@
+import logging
+import sys
+
+from obsei.analyzer.classification_analyzer import (
+    ClassificationAnalyzerConfig,
+    ZeroShotClassificationAnalyzer,
+)
+
+from obsei.source.playstore_scrapper import (
+    PlayStoreScrapperConfig,
+    PlayStoreScrapperSource,
+)
+
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+source_config = PlayStoreScrapperConfig(
+    app_url='https://play.google.com/store/apps/details?id=com.google.android.gm&hl=en_IN&gl=US',
+    max_count=3
+)
+
+source = PlayStoreScrapperSource()
+
+text_analyzer = ZeroShotClassificationAnalyzer(
+    model_name_or_path="typeform/mobilebert-uncased-mnli", device="auto"
+)
+
+source_response_list = source.lookup(source_config)
+for idx, source_response in enumerate(source_response_list):
+    logger.info(f"source_response#'{idx}'='{source_response.__dict__}'")
+
+analyzer_response_list = text_analyzer.analyze_input(
+    source_response_list=source_response_list,
+    analyzer_config=ClassificationAnalyzerConfig(
+        labels=["interface", "slow", "battery"],
+    ),
+)
+for idx, an_response in enumerate(analyzer_response_list):
+    logger.info(f"analyzer_response#'{idx}'='{an_response.__dict__}'")

obsei_module/example/playstore_scrapper_translator_example.py

@@ -0,0 +1,86 @@
+import json
+import logging
+import sys
+from datetime import datetime, timedelta
+
+import pytz
+
+from obsei.payload import TextPayload
+from obsei.analyzer.classification_analyzer import (
+    ClassificationAnalyzerConfig,
+    ZeroShotClassificationAnalyzer,
+)
+from obsei.analyzer.translation_analyzer import TranslationAnalyzer
+from obsei.misc.utils import DATETIME_STRING_PATTERN
+from obsei.source.playstore_scrapper import (
+    PlayStoreScrapperConfig,
+    PlayStoreScrapperSource,
+)
+
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+source = PlayStoreScrapperSource()
+
+
+def source_fetch():
+    since_time = datetime.utcnow().astimezone(pytz.utc) + timedelta(days=-1)
+    source_config = PlayStoreScrapperConfig(
+        countries=["us"],
+        package_name="com.color.apps.hindikeyboard.hindi.language",
+        lookup_period=since_time.strftime(
+            DATETIME_STRING_PATTERN
+        ),  # todo should be optional
+        max_count=5,
+    )
+    return source.lookup(source_config)
+
+
+def translate_text(text_list):
+    translate_analyzer = TranslationAnalyzer(
+        model_name_or_path="Helsinki-NLP/opus-mt-hi-en", device="auto"
+    )
+    source_responses = [
+        TextPayload(processed_text=text.processed_text, source_name="sample")
+        for text in text_list
+    ]
+    analyzer_responses = translate_analyzer.analyze_input(
+        source_response_list=source_responses
+    )
+    return [
+        TextPayload(
+            processed_text=response.segmented_data["translated_text"],
+            source_name="translator",
+        )
+        for response in analyzer_responses
+    ]
+
+
+def classify_text(text_list):
+    text_analyzer = ZeroShotClassificationAnalyzer(
+        model_name_or_path="joeddav/bart-large-mnli-yahoo-answers", device="cpu"
+    )
+
+    return text_analyzer.analyze_input(
+        source_response_list=text_list,
+        analyzer_config=ClassificationAnalyzerConfig(
+            labels=["no parking", "registration issue", "app issue", "payment issue"],
+        ),
+    )
+
+
+def print_list(text_name, text_list):
+    for idx, text in enumerate(text_list):
+        json_response = json.dumps(text.__dict__, indent=4, sort_keys=True, default=str)
+        logger.info(f"\n{text_name}#'{idx}'='{json_response}'")
+
+
+logger.info("Started...")
+
+source_responses_list = source_fetch()
+translated_text_list = translate_text(source_responses_list)
+analyzer_response_list = classify_text(translated_text_list)
+
+print_list("source_response", source_responses_list)
+print_list("translator_response", translated_text_list)
+print_list("classifier_response", analyzer_response_list)

obsei_module/example/reddit_example.py

@@ -0,0 +1,50 @@
+import logging
+import sys
+import time
+from datetime import datetime, timedelta
+
+import pytz
+
+from obsei.misc.utils import DATETIME_STRING_PATTERN
+from obsei.source.reddit_source import RedditConfig, RedditSource
+from obsei.workflow.store import WorkflowStore
+from obsei.workflow.workflow import Workflow, WorkflowConfig
+
+
+def print_state(id: str):
+    logger.info(f"Source State: {source.store.get_source_state(id)}")
+
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+since_time = datetime.utcnow().astimezone(pytz.utc) + timedelta(hours=-2)
+# Credentials will be fetched from env variable named reddit_client_id and reddit_client_secret
+source_config = RedditConfig(
+    subreddits=["wallstreetbets"],
+    lookup_period=since_time.strftime(DATETIME_STRING_PATTERN),
+)
+
+source = RedditSource(store=WorkflowStore())
+
+workflow = Workflow(
+    config=WorkflowConfig(
+        source_config=source_config,
+    ),
+)
+source.store.add_workflow(workflow)
+
+
+for i in range(1, 4):
+    print_state(workflow.id)
+    source_response_list = source.lookup(source_config, id=workflow.id)
+
+    if source_response_list is None or len(source_response_list) == 0:
+        break
+
+    for source_response in source_response_list:
+        logger.info(source_response.__dict__)
+
+    time.sleep(10)
+
+print_state(workflow.id)

obsei_module/example/reddit_scrapper_example.py

@@ -0,0 +1,30 @@
+import logging
+import sys
+from datetime import datetime, timedelta
+
+import pytz
+
+from obsei.misc.utils import DATETIME_STRING_PATTERN
+from obsei.source.reddit_scrapper import RedditScrapperConfig, RedditScrapperSource
+
+
+def print_state(id: str):
+    logger.info(f"Source State: {source.store.get_source_state(id)}")
+
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+since_time = datetime.utcnow().astimezone(pytz.utc) + timedelta(days=-1)
+
+source_config = RedditScrapperConfig(
+    url="https://www.reddit.com/r/wallstreetbets/comments/.rss?sort=new",
+    user_agent="testscript by u/FitStatistician7378",
+    lookup_period=since_time.strftime(DATETIME_STRING_PATTERN),
+)
+
+source = RedditScrapperSource()
+
+source_response_list = source.lookup(source_config)
+for source_response in source_response_list:
+    logger.info(source_response.__dict__)

obsei_module/example/sdk.yaml

@@ -0,0 +1,97 @@
+twitter_source:
+  _target_: obsei.source.twitter_source.TwitterSourceConfig
+  keywords:
+    - "@sample"
+  lookup_period: "1d"
+  tweet_fields:
+    - "author_id"
+    - "conversation_id"
+    - "created_at"
+    - "id"
+    - "public_metrics"
+    - "text"
+  user_fields:
+    - "id"
+    - "name"
+    - "public_metrics"
+    - "username"
+    - "verified"
+  expansions:
+    - "author_id"
+  place_fields: []
+  max_tweets: 10
+  credential:
+    _target_: obsei.source.twitter_source.TwitterCredentials
+    bearer_token: "bearer_token"
+
+play_store_source:
+  _target_: obsei.source.playstore_reviews.PlayStoreConfig
+  package_name: "com.company.package"
+  max_results: 10
+  num_retries: 2
+  cred_info:
+    _target_: obsei.source.playstore_reviews.GoogleCredInfo
+    service_cred_file: "foo/credential.json"
+    developer_key: "test_key"
+
+daily_get_sink:
+  _target_: obsei.sink.dailyget_sink.DailyGetSinkConfig
+  url: "http://localhost:8080/sample"
+  partner_id: "123456"
+  consumer_phone_number: "1234567890"
+  source_information: "Twitter @sample"
+  base_payload:
+    partnerId: daily_get_sink.partner_id
+    consumerPhoneNumber: daily_get_sink.consumer_phone_number
+
+http_sink:
+  _target_: obsei.sink.http_sink.HttpSinkConfig
+  url: "http://localhost:8080/sample"
+
+elasticsearch_sink:
+  _target_: obsei.sink.elasticsearch_sink.ElasticSearchSinkConfig
+  host: "localhost"
+  port: 9200
+  index_name: "test"
+
+jira_sink:
+  _target_: obsei.sink.jira_sink.JiraSinkConfig
+  url: "http://localhost:2990/jira"
+  username: "user"
+  password: "pass"
+  issue_type:
+    name: "Task"
+  project:
+    key: "CUS"
+
+analyzer_config:
+  _target_: obsei.analyzer.classification_analyzer.ClassificationAnalyzerConfig
+  labels:
+    - "service"
+    - "delay"
+    - "tracking"
+    - "no response"
+  add_positive_negative_labels: false
+
+analyzer:
+  _target_: obsei.analyzer.classification_analyzer.ZeroShotClassificationAnalyzer
+  model_name_or_path: "typeform/mobilebert-uncased-mnli"
+  device: "auto"
+
+slack_sink:
+  _target_: obsei.sink.SlackSink
+
+slack_sink_config:
+  _target_: obsei.sink.SlackSinkConfig
+  slack_token: 'Enter token'
+  channel_id: 'slack channel id'
+  jinja_template: |
+    ```
+           {%- for key, value in payload.items() recursive%}
+               {%- if value is mapping -%}
+      {{loop(value.items())}}
+               {%- else %}
+      {{key}}: {{value}}
+               {%- endif %}
+           {%- endfor%}
+    ```

obsei_module/example/slack_example.py

@@ -0,0 +1,66 @@
+import logging
+import os
+import sys
+
+from obsei.analyzer.dummy_analyzer import DummyAnalyzer, DummyAnalyzerConfig
+from obsei.processor import Processor
+from obsei.sink.slack_sink import SlackSink, SlackSinkConfig
+from obsei.source.playstore_scrapper import (PlayStoreScrapperConfig,
+                                             PlayStoreScrapperSource)
+from obsei.workflow.store import WorkflowStore
+from obsei.workflow.workflow import Workflow, WorkflowConfig
+
+
+def print_state(identifier: str):
+    logger.info(f"Source State: {source.store.get_source_state(identifier)}")
+
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+
+workflow_store = WorkflowStore()
+
+source_config = PlayStoreScrapperConfig(
+    app_url='https://play.google.com/store/apps/details?id=com.google.android.gm&hl=en_IN&gl=US',
+    max_count=3
+)
+
+source = PlayStoreScrapperSource(store=workflow_store)
+
+sink_config = SlackSinkConfig(
+    slack_token=os.environ["SLACK_TOKEN"],
+    channel_id="C01TUPZ23NZ",
+    jinja_template="""
+```
+     {%- for key, value in payload.items() recursive%}
+         {%- if value is mapping -%}
+{{loop(value.items())}}
+         {%- else %}
+{{key}}: {{value}}
+         {%- endif %}
+     {%- endfor%}
+```
+   """
+)
+sink = SlackSink(store=workflow_store)
+
+analyzer_config = DummyAnalyzerConfig()
+analyzer = DummyAnalyzer()
+
+workflow = Workflow(
+    config=WorkflowConfig(
+        source_config=source_config,
+        sink_config=sink_config,
+        analyzer_config=analyzer_config,
+    ),
+)
+workflow_store.add_workflow(workflow)
+
+processor = Processor(
+    analyzer=analyzer, sink=sink, source=source, analyzer_config=analyzer_config
+)
+
+processor.process(workflow=workflow)
+
+print_state(workflow.id)

obsei_module/example/twitter_source_example.py

@@ -0,0 +1,98 @@
+import logging
+import sys
+
+from obsei.analyzer.classification_analyzer import ZeroShotClassificationAnalyzer, ClassificationAnalyzerConfig
+from obsei.sink.slack_sink import SlackSinkConfig, SlackSink
+from obsei.source.twitter_source import TwitterSourceConfig, TwitterSource, TwitterCredentials
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+twitter_cred_info = None
+
+# Enter your twitter credentials
+# Get it from https://developer.twitter.com/en/apply-for-access
+# Currently it will fetch from environment variables: twitter_bearer_token, twitter_consumer_key, twitter_consumer_secret
+# Uncomment below lines if you like to pass credentials directly instead of env variables
+
+# twitter_cred_info = TwitterCredentials(
+#     bearer_token='<Enter bearer_token>',
+#     consumer_key="<Enter consumer_key>",
+#     consumer_secret="<Enter consumer_secret>"
+# )
+
+source_config = TwitterSourceConfig(
+    query="bitcoin",
+    lookup_period="1h",
+    tweet_fields=[
+        "author_id",
+        "conversation_id",
+        "created_at",
+        "id",
+        "public_metrics",
+        "text",
+    ],
+    user_fields=["id", "name", "public_metrics", "username", "verified"],
+    expansions=["author_id"],
+    place_fields=None,
+    max_tweets=10,
+    cred_info=twitter_cred_info or None
+)
+
+source = TwitterSource()
+
+
+sink_config = SlackSinkConfig(
+    # Uncomment below lines if you like to pass credentials directly instead of env variables
+    #    slack_token="SLACK_TOKEN",
+    #    channel_id="CHANNEL_ID",
+    jinja_template="""
+:bell: Hi there!, a new `<{{payload['meta']['tweet_url']}}|tweet>` of interest is found by *Obsei*
+>📝 Content: 
+```{{payload['meta']['text']}}```
+>ℹ️Information:
+```
+User Name: {{payload['meta']['author_info']['name']}} ({{payload['meta']['author_info']['user_url']}})
+Tweet Metrics: Retweets={{payload['meta']['public_metrics']['retweet_count']}}, Likes={{payload['meta']['public_metrics']['like_count']}}
+Author Metrics: Verified={{payload['meta']['author_info']['verified']}}, Followers={{payload['meta']['author_info']['public_metrics']['followers_count']}}
+```
+>🧠 AI Engine Data:
+```
+     {%- for key, value in payload['segmented_data']['classifier_data'].items() recursive%}
+         {%- if value is mapping -%}
+{{loop(value.items())}}
+         {%- else %}
+{{key}}: {{value}}
+         {%- endif %}
+     {%- endfor%}
+```
+   """
+)
+sink = SlackSink()
+
+text_analyzer = ZeroShotClassificationAnalyzer(
+    model_name_or_path="typeform/mobilebert-uncased-mnli", device="auto"
+)
+
+analyzer_config = ClassificationAnalyzerConfig(
+    labels=["going up", "going down"],
+    add_positive_negative_labels=False,
+)
+
+source_response_list = source.lookup(source_config)
+for idx, source_response in enumerate(source_response_list):
+    logger.info(f"source_response#'{idx}'='{source_response.__dict__}'")
+
+analyzer_response_list = text_analyzer.analyze_input(
+    source_response_list=source_response_list,
+    analyzer_config=analyzer_config,
+)
+
+for idx, an_response in enumerate(analyzer_response_list):
+    logger.info(f"analyzer_response#'{idx}'='{an_response.__dict__}'")
+
+sink_response_list = sink.send_data(
+    analyzer_responses=analyzer_response_list, config=sink_config, id=id
+)
+for idx, sink_response in enumerate(sink_response_list):
+    logger.info(f"source_response#'{idx}'='{sink_response.__dict__}'")

obsei_module/example/web_crawler_example.py

@@ -0,0 +1,43 @@
+# Fetch full news article
+from obsei.source.website_crawler_source import (
+    TrafilaturaCrawlerConfig,
+    TrafilaturaCrawlerSource,
+)
+
+
+def print_list(response_list):
+    for response in response_list:
+        print(response.__dict__)
+
+
+# Single URL
+source_config = TrafilaturaCrawlerConfig(urls=["https://obsei.github.io/obsei/"])
+
+source = TrafilaturaCrawlerSource()
+
+source_response_list = source.lookup(source_config)
+print_list(source_response_list)
+
+
+# RSS feed (Note it will take lot of time)
+source_config = TrafilaturaCrawlerConfig(
+    urls=["https://news.google.com/rss/search?q=bitcoin&hl=en&gl=US&ceid=US:en"],
+    is_feed=True,
+)
+
+source = TrafilaturaCrawlerSource()
+
+source_response_list = source.lookup(source_config)
+print_list(source_response_list)
+
+
+# Full website (Note it will take lot of time)
+source_config = TrafilaturaCrawlerConfig(
+    urls=["https://haystack.deepset.ai/"],
+    is_sitemap=True,
+)
+
+source = TrafilaturaCrawlerSource()
+
+source_response_list = source.lookup(source_config)
+print_list(source_response_list)

obsei_module/example/with_sdk_config_file.py

@@ -0,0 +1,28 @@
+import logging
+import sys
+
+from obsei.configuration import ObseiConfiguration
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+obsei_configuration = ObseiConfiguration(
+    config_path="../example",
+    config_filename="sdk.yaml",
+)
+
+text_analyzer = obsei_configuration.initialize_instance("analyzer")
+analyzer_config = obsei_configuration.initialize_instance("analyzer_config")
+slack_source_config = obsei_configuration.initialize_instance("slack_sink_config")
+slack_sink = obsei_configuration.initialize_instance("slack_sink")
+
+play_store_source_config = obsei_configuration.initialize_instance("play_store_source")
+twitter_source_config = obsei_configuration.initialize_instance("twitter_source")
+http_sink_config = obsei_configuration.initialize_instance("http_sink")
+daily_get_sink_config = obsei_configuration.initialize_instance("daily_get_sink")
+# docker run -d --name elasticsearch -p 9200:9200 -e "discovery.type=single-node" elasticsearch:7.9.2
+elasticsearch_sink_config = obsei_configuration.initialize_instance(
+    "elasticsearch_sink"
+)
+# Start jira server locally `atlas-run-standalone --product jira`
+jira_sink_config = obsei_configuration.initialize_instance("jira_sink")