openapi: 3.0.0
# Added by API Auto Mocking Plugin
servers:
  - description: DataHen API
    url: https://app.datahen.com/api/v1
security:
- httpBearer: []
info:
  description: | 
    API documentation on how to manage scrapers, jobs, variables, exports, etc on DataHen.
    
    # Authentication Token
    
    To make a proper request to the API endpoints, you need to be authenticated via the Auth Token. 
    There are two ways to supply your token to the API. One is via the Query Parameter, and another one is via the Request Header.
    To get an Auth Token, you can login to DataHen and visit the `Developers > Auth Tokens` section to find or create your Auth Token.
    
    ## Query Parameter based Authentication
    To make an authenticated request based on query parameter, you need to add `token` parameter to the URL query. 
    
    An example request would be the following. 
    
    `https://app.datahen.com/api/v1/scrapers?token=yourtokenhere`
    
    ## Request Header based Authentication
    To make an authenticated request based on request header, you can need to add `Authorization` header on your request.
    
    An example of a curl command that uses a header based authentication.
    
    ```
    curl --request GET 'https://app.datahen.com/api/v1/scrapers' --header 'Authorization: Bearer <yourtokenhere>
    ```
    # Pagination
    The API Endpoints support the ability to specify the page number and the number of records per page.
    
    ## Per Page
    The Per Page parameter specifies how many records per page the API endpoint will return.  
    By default is set to 100 records.
    
    To specify a different number of records per page, use the `pp` parameter when you query the API endpoint.
    
    The example request below sets the `per page` parameter to 30.
    
    `https://app.datahen.com/api/v1/scrapers?pp=30`
    
    ## Page Number
    
    The Page Number parameter specifies which page you are in when you are retreiving the records.
    By default, when not specifically set, the page number is set to 1. When you've reached the end of the page, you will receive the response body as `null`.
    
    To specify a different page number, use the `p` parameter when you query the API endpoint.
    
    The example request below sets the `page` parameter to 2.
    
    `https://app.datahen.com/api/v1/scrapers?p=2`
    
    ## Pagination metadata
    When an API endpoints returns a paginated response, it will return also the response headers that would tell you more about the the number of potential records, and pages.
    
    The response Headers will contain the following `X-Pagination-*` headers:
    
    ```
    X-Pagination-Current-Page: 1
    X-Pagination-First-Page: 1
    X-Pagination-Last-Page: 1
    X-Pagination-Per-Page: 100
    X-Pagination-Total-Pages: 1
    X-Pagination-Total-Records: 1
    ```

  version: "1.0.2"
  title: DataHen API Reference
  contact:
    email: services@datahen.com
  license:
    name: Apache 2.0
    url: 'http://www.apache.org/licenses/LICENSE-2.0.html'
tags:
  - name: Scrapers
    description: Manage Scrapers

  - name: Jobs
    description: Manage Scrape Jobs
  - name: Current Job
    description: Manage Current Job of a Scraper
  
  - name: Job Stats
    description: Get the Job's Stats
  - name: Current Job Stats
    description: Get the Current Job's Stats

  - name: Job Output
    description: Manage Job Outputs.
  - name: Current Job Output
    description: Manage Current Job Outputs.

  - name: Job Collection
    description: Manage Job Collections.
  - name: Current Job Collection
    description: Manage Current Job Collections.


  - name: Scraper Export
    description: Manage Scraper Exports
  - name: Job Export
    description: Manage Job Exports
  - name: Scraper Exporter
    description: Manage Scraper Exporters


  - name: Global Pages
    description: View Pages that has been scraped or being scraped by the account.
  - name: Job Pages
    description: Manage Job Pages.
  - name: Current Job Pages
    description: Manage Current Job Pages.


  - name: Scraper Variables
    description: Manage Input Variables on a Scraper
  - name: Current Job Variables
    description: Manage Input Variables on a Current Job of a Scraper
  - name: Account Variables
    description: Manage Environment Variables on the Account

  - name: Auth Token
    description: "Manage Auth Tokens on the Account. Restricted to users with `admin` or `owner` role."
  - name: Deploy Key
    description: "Manage the Deploy Key on the Account. Restricted to users with `admin` or `owner` role."
  

  - name: Job Logs
    description: View Log entries on a Job.
  - name: Page Logs
    description: View Log entries on a Page on a Job.

  - name: Deployment
    description: Manage Scraper Deployments

  
  - name: Job Seeding
    description: Manage Job Seeding
  - name: Job Finisher
    description: Manage Job Finisher
  - name: Current Job Finisher
    description: Manage Current Job Finisher

  - name: Internal Job Seeding
    description: "`internal` Endpoints used by scraper agents to execute the seeding script on the job container."
  - name: Internal Job Page Parsing
    description: "`internal` Endpoints used by scraper agents to execute parser scripts on the job container."
  
  - name: Internal Job Finisher
    description: "`internal` Endpoints used by scraper agents to execute the finisher scripton the job container."
  

paths:
  /scrapers:
    post:
      tags:
      - Scrapers
      summary: Create a Scraper
      security:
        - bearerAuth: []
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateUpdateScraper'
      responses:
        200:
          $ref: '#/components/responses/ScraperSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    get:
      tags:
      - Scrapers
      summary: Get a list of scrapers
      security:
        - bearerAuth: []
      parameters:
        - in: query
          name: only_schema
          description: When specified, will return only Scrapers that has schema specified in the Scraper Config.
          required: false
          schema:
            type: boolean
        - in: query
          name: only_exporters
          description: When specified, will return only Scrapers that has any exporter specified in the Scraper Config.
          required: false
          schema:
            type: boolean
        - $ref: '#/components/parameters/PageNum'
        - $ref: '#/components/parameters/PerPage'
      responses:
        200:
          $ref: '#/components/responses/ScrapersSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}:
    get:
      tags:
      - Scrapers
      summary: Show a Scraper
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
      responses:
        200:
          $ref: '#/components/responses/ScraperSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    put:
      tags:
      - Scrapers
      summary: Update a Scraper
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateUpdateScraper'
      responses:
        200:
          $ref: '#/components/responses/ScraperSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    delete:
      tags:
      - Scrapers
      summary: Delete a Scraper
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
      responses:
        200:
          $ref: '#/components/responses/DeleteSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs:
    get:
      tags:
      - Jobs
      summary: Get a list of Jobs
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/PageNum'
        - $ref: '#/components/parameters/PerPage'
      responses:
        200:
          $ref: '#/components/responses/JobsSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}:
    get:
      tags:
      - Jobs
      summary: Show a Scrape Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
      responses:
        200:
          $ref: '#/components/responses/ScraperSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/jobs: 
    post: 
      tags:
      - Jobs
      summary: Create a Scrape Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateScrapeJob'
      responses:
        200:
          $ref: '#/components/responses/JobSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    get:
      tags:
      - Jobs
      summary: Get a list of Jobs on a Scraper
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/PageNum'
        - $ref: '#/components/parameters/PerPage'
      responses:
        200:
          $ref: '#/components/responses/JobSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job:
    get:
      tags:
      - Current Job
      summary: Show the Current Job on a Scraper
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
      responses:
        200:
          $ref: '#/components/responses/ScraperSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    put:
      tags:
      - Current Job
      summary: Update the Current Scrape Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
      requestBody:
        description: Request Body
        required: false
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateScrapeJob'
      responses:
        200:
          $ref: '#/components/responses/JobSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/pause:
    put:
      tags:
      - Current Job
      summary: Pause the Current Scrape Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
      responses:
        200:
          $ref: '#/components/responses/JobSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/resume:
    put:
      tags:
      - Current Job
      summary: Resume the Current Scrape Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
      responses:
        200:
          $ref: '#/components/responses/JobSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/cancel:
    put:
      tags:
      - Current Job
      summary: Cancel the Current Scrape Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
      responses:
        200:
          $ref: '#/components/responses/JobSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/stats/current:
    get:
      tags:
      - Current Job Stats
      summary: Show the Stats on a Current Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
      responses:
        200:
          $ref: '#/components/responses/JobStatSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/stats/history:
    get:
      tags:
      - Current Job Stats
      summary: Show the Stats History on a Current Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/MinTimestamp'
        - $ref: '#/components/parameters/MaxTimestamp'
      responses:
        200:
          $ref: '#/components/responses/JobStatHistorySuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/stats/current:
    get:
      tags:
      - Job Stats
      summary: Show the Stats on a Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
      responses:
        200:
          $ref: '#/components/responses/JobStatSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/stats/history:
    get:
      tags:
      - Job Stats
      summary: Show the Stats History on a Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
        - $ref: '#/components/parameters/MinTimestamp'
        - $ref: '#/components/parameters/MaxTimestamp'
      responses:
        200:
          $ref: '#/components/responses/JobStatHistorySuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'        
  /scrapers/{name}/vars:
    get:
      tags:
      - Scraper Variables
      summary: Get a list of Input Variables on a Scraper
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/PageNum'
        - $ref: '#/components/parameters/PerPage'
      responses:
        200:
          $ref: '#/components/responses/VarsSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    put:
      tags:
      - Scraper Variables
      summary: Sets a Scraper's multiple Input Variables at once.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              type: array
              items:
                $ref: '#/components/schemas/SetUnsetVar'
      responses:
        200:
          $ref: '#/components/responses/VarsSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/vars/{var_name}:
    get:
      tags:
      - Scraper Variables
      summary: Show one Input Variable on a Scraper.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/VarName'
      responses:
        200:
          $ref: '#/components/responses/VarSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    put:
      tags:
      - Scraper Variables
      summary: Sets one Input Variable on a Scraper.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/VarName'
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/SetUnsetVar'
            example:
              secret: false
              value: macbook air
      responses:
        200:
          $ref: '#/components/responses/VarsSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    delete:
      tags:
      - Scraper Variables
      summary: Unsets a Scraper's Input Variable. One at a time.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/VarName'
      responses:
        200:
          $ref: '#/components/responses/DeleteSuccess'  
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/vars:
    get:
      tags:
      - Current Job Variables
      summary: Get a list of Input Variables on a Scrape Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/PageNum'
        - $ref: '#/components/parameters/PerPage'
      responses:
        200:
          $ref: '#/components/responses/VarsSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    put:
      tags:
      - Current Job Variables
      summary: Sets Current Job's multiple Input Variables at once
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              type: array
              items:
                $ref: '#/components/schemas/SetUnsetVar'
      responses:
        200:
          $ref: '#/components/responses/VarsSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/vars/{var_name}:
    get:
      tags:
      - Current Job Variables
      summary: Show one Input Variable on a Current Job.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/VarName'
      responses:
        200:
          $ref: '#/components/responses/VarSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    put:
      tags:
      - Current Job Variables
      summary: Sets one Input Variable on a Current Job.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/VarName'
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/SetUnsetVar'
            example:
              secret: false
              value: macbook air
      responses:
        200:
          $ref: '#/components/responses/VarSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    delete:
      tags:
      - Current Job Variables
      summary: Unsets a Scrape Job's Input Variable. One at a time.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/VarName'
      responses:
        200:
          $ref: '#/components/responses/DeleteSuccess'  
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /env_vars:
    get:
      tags:
      - Account Variables
      summary: Get a list of Environment Variables on the Account
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/PageNum'
        - $ref: '#/components/parameters/PerPage'
      responses:
        200:
          $ref: '#/components/responses/VarsSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    put:
      tags:
      - Account Variables
      summary: Sets the Account's multiple Environment Variables at once.
      security:
        - bearerAuth: []
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              type: array
              items:
                $ref: '#/components/schemas/SetUnsetVar'
      responses:
        200:
          $ref: '#/components/responses/VarsSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /env_vars/{var_name}:
    get:
      tags:
      - Account Variables
      summary: Show one Environment Variable on the Account.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/VarName'
      responses:
        200:
          $ref: '#/components/responses/VarSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    put:
      tags:
      - Account Variables
      summary: Sets one Environment Variable on the Account.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/VarName'
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/SetUnsetVar'
            example:
              secret: false
              value: macbook air
      responses:
        200:
          $ref: '#/components/responses/VarsSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    delete:
      tags:
      - Account Variables
      summary: Unsets Account's Environment Variable. One at a time.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/VarName'
      responses:
        200:
          $ref: '#/components/responses/DeleteSuccess'  
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/exports:
    get:
      tags:
      - Scraper Export
      summary: Get a list of all Exports
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/PageNum'
        - $ref: '#/components/parameters/PerPage'
      responses:
        200:
          $ref: '#/components/responses/ExportsSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/exports/{export_id}:
    get:
      tags:
      - Scraper Export
      summary: Show an Export
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ExportID'
      responses:
        200:
          $ref: '#/components/responses/ExportSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/exports/{export_id}/download:
    get:
      tags:
      - Scraper Export
      summary: Get a Signed URL to Download an Export
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ExportID'
      responses:
        200:
          $ref: '#/components/responses/DownloadExportSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/exports:
    get:
      tags:
      - Scraper Export
      summary: Get a list of all Exports on a Scraper
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/PageNum'
        - $ref: '#/components/parameters/PerPage'
      responses:
        200:
          $ref: '#/components/responses/ExportsSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/exports/{exporter_name}:
    post:
      tags:
      - Scraper Export
      summary: Start an Export on a Scraper
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/ExporterName'
      responses:
        200:
          $ref: '#/components/responses/ExportSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/exporters:
    get:
      tags:
      - Scraper Exporter
      summary: Get a list of Exporters on a Scraper
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/ExporterType'
        - $ref: '#/components/parameters/ExporterWriteMode'
        - $ref: '#/components/parameters/QueryCollection'
      responses:
        200:
          $ref: '#/components/responses/ExporterSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/exporters/{exporter_name}:
    get:
      tags:
      - Scraper Exporter
      summary: Show an Exporter on a Scraper
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/ExporterName'
      responses:
        200:
          $ref: '#/components/responses/ExporterSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/exports/{exporter_name}:
    post:
      tags:
      - Job Export
      summary: Start an Export on a Scrape Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
        - $ref: '#/components/parameters/ExporterName'
      responses:
        200:
          $ref: '#/components/responses/ExportSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /global_pages/{gid}:
    get:
      tags:
      - Global Pages
      summary: Show a Global Page
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/GID'
      responses:
        200:
          $ref: '#/components/responses/GlobalPageSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /global_pages/{gid}/content:
    get:
      tags:
      - Global Pages
      summary: Show a Content of a Global Page
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/GID'
      responses:
        200:
          $ref: '#/components/responses/PageContentSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/pages:
    post:
      tags:
      - Job Pages
      summary: Add a Page in a Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateJobPage'
      responses:
        200:
          $ref: '#/components/responses/JobPageSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    get:
      tags:
      - Job Pages
      summary: Get a list of Pages in a Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
        - $ref: '#/components/parameters/FetchFail'
        - $ref: '#/components/parameters/ParseFail'
        - $ref: '#/components/parameters/PageType'
        - $ref: '#/components/parameters/JobPageStatus'
        - $ref: '#/components/parameters/PageNum'
        - $ref: '#/components/parameters/PerPage'
      responses:
        200:
          $ref: '#/components/responses/JobPagesSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/pages/refetch:
    put:
      tags:
      - Job Pages
      summary: Refetch Pages in a Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
        - $ref: '#/components/parameters/FetchFail'
        - $ref: '#/components/parameters/ParseFail'
        - $ref: '#/components/parameters/JobPageStatus'
        - in: query
          name: gid
          required: false
          schema:
            type: string
          description: GID of a page you need to refetch. If not specified, will refetch everything in the Job.
      responses:
        200:
          $ref: '#/components/responses/JobPageRefetchSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/pages/reparse:
    put:
      tags:
      - Job Pages
      summary: Reparse Pages in a Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
        - $ref: '#/components/parameters/ParseFail'
        - $ref: '#/components/parameters/JobPageStatus'
        - in: query
          name: gid
          required: false
          schema:
            type: string
          description: GID of a page you need to refetch. If not specified, will refetch everything in the Job.
      responses:
        200:
          $ref: '#/components/responses/JobPageReparseSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/pages/{gid}:
    get:
      tags:
      - Job Pages
      summary: Show a Page in a Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
        - $ref: '#/components/parameters/GID'
      responses:
        200:
          $ref: '#/components/responses/JobPageSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    put:
      tags:
      - Job Pages
      summary: Update a Page in a Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
        - $ref: '#/components/parameters/GID'
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateJobPage'
      responses:
        200:
          $ref: '#/components/responses/JobPageSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/pages/{gid}/content:
    get:
      tags:
      - Job Pages
      summary: Show a Content of a Job Page
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
        - $ref: '#/components/parameters/GID'
      responses:
        200:
          $ref: '#/components/responses/PageContentSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/pages/{gid}/failed_content:
    get:
      tags:
      - Job Pages
      summary: Show a Content of a failed Job Page
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
        - $ref: '#/components/parameters/GID'
      responses:
        200:
          $ref: '#/components/responses/PageContentSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/output/collections/{collection}/records:
    get:
      tags:
      - Job Output
      summary: Get a list of Output Records from a Collection in a Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
        - $ref: '#/components/parameters/Collection'
        - $ref: '#/components/parameters/JobOutputQuery'
        - $ref: '#/components/parameters/PageNum'
        - $ref: '#/components/parameters/PerPage'
      responses:
        200:
          $ref: '#/components/responses/JobOutputsSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/output/collections/{collection}/records/{record_id}:
    get:
      tags:
      - Job Output
      summary: Show one Output Record from a Collection in a Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
        - $ref: '#/components/parameters/Collection'
        - $ref: '#/components/parameters/JobOutputRecordID'
      responses:
        200:
          $ref: '#/components/responses/JobOutputSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/output/collections:
    get:
      tags:
      - Job Collection
      summary: Get a list of Job Output Collections
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
        - $ref: '#/components/parameters/JobOutputCollectionFromSchema'
      responses:
        200:
          $ref: '#/components/responses/JobOutputCollectionsSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/output/collections/{collection}/schema:
    get:
      tags:
      - Job Collection
      summary: Show the Schema that is used to validate the records in the Job Collection
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
        - $ref: '#/components/parameters/Collection'
        - $ref: '#/components/parameters/SchemaType'
        - $ref: '#/components/parameters/ConnectorType'
      responses:
        200:
          $ref: '#/components/responses/SchemaSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/output/collections:
    get:
      tags:
      - Current Job Collection
      summary: Get a list of Current Job Output Collections
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/JobOutputCollectionFromSchema'
      responses:
        200:
          $ref: '#/components/responses/JobOutputCollectionsSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/output/collections/{collection}/schema:
    get:
      tags:
      - Current Job Collection
      summary: Show the Schema that is used to validate the records in the Current Job Collection
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/Collection'
        - $ref: '#/components/parameters/SchemaType'
        - $ref: '#/components/parameters/ConnectorType'
      responses:
        200:
          $ref: '#/components/responses/SchemaSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/output/collections/{collection}/records:
    get:
      tags:
      - Current Job Output
      summary: Get a list of Output Records from a Collection in a Current Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/Collection'
        - $ref: '#/components/parameters/JobOutputQuery'
        - $ref: '#/components/parameters/PageNum'
        - $ref: '#/components/parameters/PerPage'
      responses:
        200:
          $ref: '#/components/responses/JobOutputsSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/output/collections/{collection}/records/{record_id}:
    get:
      tags:
      - Current Job Output
      summary: Show one Output Record from a Collection in a Current Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/Collection'
        - $ref: '#/components/parameters/JobOutputRecordID'
      responses:
        200:
          $ref: '#/components/responses/JobOutputSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/pages:
    post:
      tags:
      - Current Job Pages
      summary: Add a Page in a Current Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/CreateJobPage'
      responses:
        200:
          $ref: '#/components/responses/JobPageSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    get:
      tags:
      - Current Job Pages
      summary: Get a list of Pages in a Current Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/FetchFail'
        - $ref: '#/components/parameters/ParseFail'
        - $ref: '#/components/parameters/PageType'
        - $ref: '#/components/parameters/JobPageStatus'
        - $ref: '#/components/parameters/PageNum' 
        - $ref: '#/components/parameters/PerPage'
      responses:
        200:
          $ref: '#/components/responses/JobPagesSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/pages/refetch:
    put:
      tags:
      - Current Job Pages
      summary: Refetch Pages in a Current Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/FetchFail'
        - $ref: '#/components/parameters/ParseFail'
        - $ref: '#/components/parameters/JobPageStatus'
        - in: query
          name: gid
          required: false
          schema:
            type: string
          description: GID of a page you need to refetch. If not specified, will refetch everything in the Job.
      responses:
        200:
          $ref: '#/components/responses/JobPageRefetchSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/pages/reparse:
    put:
      tags:
      - Current Job Pages
      summary: Reparse Pages in a Current Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/ParseFail'
        - $ref: '#/components/parameters/JobPageStatus'
        - in: query
          name: gid
          required: false
          schema:
            type: string
          description: GID of a page you need to refetch. If not specified, will refetch everything in the Job.
      responses:
        200:
          $ref: '#/components/responses/JobPageReparseSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/pages/{gid}:
    get:
      tags:
      - Current Job Pages
      summary: Show a Page in a Current Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/GID'
      responses:
        200:
          $ref: '#/components/responses/JobPageSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    put:
      tags:
      - Current Job Pages
      summary: Update a Page in a Current Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/GID'
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/UpdateJobPage'
      responses:
        200:
          $ref: '#/components/responses/JobPageSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/pages/{gid}/content:
    get:
      tags:
      - Current Job Pages
      summary: Show a Content of a Job Page
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/GID'
      responses:
        200:
          $ref: '#/components/responses/PageContentSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/pages/{gid}/failed_content:
    get:
      tags:
      - Current Job Pages
      summary: Show a Content of a failed Job Page
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/GID'
      responses:
        200:
          $ref: '#/components/responses/PageContentSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/pages/parse_dequeue:
    put:
      tags:
      - Internal Job Page Parsing
      summary: Dequeue a list of pages to get parsed by the scraper agent on the Job.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/PageParseDequeue'
      responses:
        200:
          $ref: '#/components/responses/JobPagesToParseSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/pages/{gid}/parsing_update:
    put:
      tags:
      - Internal Job Page Parsing
      summary: After parsing is done by the scraper agent, a request is sent to this endpoint to update the status of the page.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
        - $ref: '#/components/parameters/GID'
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/ParsedJobPage'
      responses:
        200:
          $ref: '#/components/responses/SuccessHeader'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/log:
    get:
      tags:
      - Job Logs
      summary: Show the Logs on a Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
        - $ref: '#/components/parameters/LogPageToken'
        - $ref: '#/components/parameters/LogJobType'
        - $ref: '#/components/parameters/LogOrder'
      responses:
        200:
          $ref: '#/components/responses/JobLogSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/log:
    get:
      tags:
      - Job Logs
      summary: Show the Logs on a Current Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/LogPageToken'
        - $ref: '#/components/parameters/LogJobType'
        - $ref: '#/components/parameters/LogOrder'
      responses:
        200:
          $ref: '#/components/responses/JobLogSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/pages/{gid}/log:
    get:
      tags:
      - Page Logs
      summary: Show the Logs on a Page in a Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
        - $ref: '#/components/parameters/GID'
        - $ref: '#/components/parameters/LogPageToken'
        - $ref: '#/components/parameters/LogJobType'
        - $ref: '#/components/parameters/LogOrder'
      responses:
        200:
          $ref: '#/components/responses/JobLogSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/pages/{gid}/log:
    get:
      tags:
      - Page Logs
      summary: Show the Logs on Page in a Current Job
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
        - $ref: '#/components/parameters/GID'
        - $ref: '#/components/parameters/LogPageToken'
        - $ref: '#/components/parameters/LogJobType'
        - $ref: '#/components/parameters/LogOrder'
      responses:
        200:
          $ref: '#/components/responses/JobLogSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/clear_seeder:
    put:
      tags:
      - Job Seeding
      summary: Clear interrupted Seeder on a Job so that seeding gets restarted.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
      responses:
        200:
          $ref: '#/components/responses/SuccessHeader'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/seeding_update:
    put:
      tags:
      - Internal Job Seeding
      summary: After Seeding is done by the scraper agent, a request is sent to this endpoint to update the status of the seeding.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/SeededJob'
      responses:
        200:
          $ref: '#/components/responses/SuccessHeader'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/confirm_seeding:
    put:
      tags:
      - Internal Job Seeding
      summary: Does atomic confirm, to ensure that only one seeder can run at any given time on a Scraper Agent.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
      responses:
        200:
          $ref: '#/components/responses/SuccessHeader'
        406:
          description: Your Seeding confirmation is not accepted. Likely because another Seeder is currently running, or the Seeder already ran before.
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/clear_finisher:
    put:
      tags:
      - Job Finisher
      summary: Clear interrupted Finisher on a Job so that the Finisher process gets restarted.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
      responses:
        200:
          $ref: '#/components/responses/SuccessHeader'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/missing_finisher:
    put:
      tags:
      - Internal Job Finisher
      summary: Mark the job to have a non-existent(missing) finisher script. This prevents execution of finisher at the end of the job.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
      responses:
        200:
          $ref: '#/components/responses/SuccessHeader'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/finisher_update:
    put:
      tags:
      - Internal Job Finisher
      summary: After Finisher process is done by the scraper agent, a request is sent to this endpoint to update the status of the finisher.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/FinisherJob'
      responses:
        200:
          $ref: '#/components/responses/SuccessHeader'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/confirm_finisher:
    put:
      tags:
      - Internal Job Finisher
      summary: Does atomic confirm to ensure that only one finisher can run at any given time on a Scraper Agent.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
      responses:
        200:
          $ref: '#/components/responses/SuccessHeader'
        406:
          description: Your Finisher confirmation is not accepted. Likely because another Seeder is currently running, or the Seeder already ran before.
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/current_job/finisher/reset:
    put:
      tags:
      - Current Job Finisher
      summary: Reset Finisher on a Current Job so that the Finisher process gets restarted.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
      responses:
        200:
          $ref: '#/components/responses/SuccessHeader'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /scrapers/{name}/deployments:
    post:
      tags:
      - Deployment
      summary: Deploy codes from a GIT repository onto a Scraper.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
      responses:
        200:
          $ref: '#/components/responses/ScraperDeploymentSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    get:
      tags:
      - Deployment
      summary: Get a list of Scraper Deployments.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/ScraperName'
      responses:
        200:
          $ref: '#/components/responses/ScraperDeploymentsSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /jobs/{id}/successful_deployment:
    get:
      tags:
      - Deployment
      summary: Get the latest successful Git Deployment on a Job. Used by scraper agent to deploy the latest Deployment into the Scrape Job container.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/JobID'
      responses:
        200:
          $ref: '#/components/responses/ScraperDeploymentSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /auth_tokens:
    post:
      tags:
      - Auth Token
      summary: Create an Auth Token on the account.
      security:
        - bearerAuth: []
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              type: object
              required:
              - role
              properties:
                role:
                  type: string
                  example: guest
                  enum:
                  - owner
                  - account_admin
                  - basic
                  - guest
                description:
                  type: string
                  example: John Doe
      responses:
        200:
          $ref: '#/components/responses/AuthTokenSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    get:
      tags:
      - Auth Token
      summary: Get a list of Auth Tokens on the account.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/PageNum'
        - $ref: '#/components/parameters/PerPage'
      responses:
        200:
          $ref: '#/components/responses/AuthTokensSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /auth_tokens/{token}:
    get:
      tags:
      - Auth Token
      summary: Show one Auth Token on the account.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/Token'
      responses:
        200:
          $ref: '#/components/responses/AuthTokenSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    put:
      tags:
      - Auth Token
      summary: Update an Auth Token on the account.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/Token'
      requestBody:
        description: Request Body
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/AuthToken'
      responses:
        200:
          $ref: '#/components/responses/AuthTokenSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    delete:
      tags:
      - Auth Token
      summary: Delete an Auth Token on an Account.
      security:
        - bearerAuth: []
      parameters:
        - $ref: '#/components/parameters/Token'
      responses:
        200:
          $ref: '#/components/responses/DeleteTokenSuccess'  
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /validate_token:
    get:
      tags:
      - Auth Token
      summary: Validate the current token.
      security:
        - bearerAuth: []
      responses:
        200:
          $ref: '#/components/responses/AuthTokenSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
  /deploy_key:
    post:
      tags:
      - Deploy Key
      summary: Generate the Deploy Key on the account. Sending another request will re-generate the Deploy Key.
      security:
        - bearerAuth: []
      responses:
        200:
          $ref: '#/components/responses/AuthTokenSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    get:
      tags:
      - Deploy Key
      summary: Show the Deploy Key on the account.
      security:
        - bearerAuth: []
      responses:
        200:
          $ref: '#/components/responses/AuthTokensSuccess'
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
    delete:
      tags:
      - Deploy Key
      summary: Delete the Deploy Key on the account.
      security:
        - bearerAuth: []
      responses:
        200:
          description: Successful Response
          content:
            application/json:
              schema:
                type: object
                properties:
                  Deleted:
                    type: boolean
                    example: true
                example:
                  Deleted: true
        400:
          $ref: '#/components/responses/AuthTokenMissing'
        401:
          $ref: '#/components/responses/UnauthorizedError'
        422:
          $ref: '#/components/responses/UnprocessableEntity'
components:
  securitySchemes:
    bearerAuth:
      type: http
      scheme: bearer
  responses:
    SuccessHeader:
      description: Successful Response
    AuthTokensSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            type: array
            items:
              $ref: '#/components/schemas/AuthToken'
    AuthTokenSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/AuthToken'
    ExporterSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            type: object
            oneOf:
            - $ref: '#/components/schemas/JSONExporterConfig'
            - $ref: '#/components/schemas/CSVExporterConfig'
            - $ref: '#/components/schemas/ContentExporterConfig'
            example:
              exporter_name: products_last10_json
              exporter_type: json
              write_mode: pretty_array
              collection: products
              order: desc
              limit: 10
              offset: 0
    ConnectorDataSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            type: array
            title: ConnectorExportData
            items:
              oneOf:
                - $ref: '#/components/schemas/GDSConnectorExportData'
            example:
            - values:
                - '-'
                - '2019-10-21 15:06:11 +0000'
                - '6'
                - ml
            - values:
                - '-'
                - '2019-10-21 14:57:33 +0000'
                - '1'
                - '-'
    JobPageRefetchSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            type: object
            properties:
              refetched_pages:
                type: integer
            example:
              refetched_pages: 7
    JobPageReparseSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            type: object
            properties:
              reparsed_pages:
                type: integer
            example:
              reparsed_pages: 7
    JobPagesToParseSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            type: array
            items:
              $ref: '#/components/schemas/JobPageToParse'
    JobLogSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/JobLog'
    GlobalPageSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/GlobalPage'
    ScraperDeploymentSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ScraperDeployment'
    ScraperDeploymentsSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            type: array
            items:
              $ref: '#/components/schemas/ScraperDeployment'
    JobPageSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/JobPage'
    JobPagesSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            type: array
            items:
              $ref: '#/components/schemas/JobPage'
    JobOutputSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/JobOutput'
    JobOutputsSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            type: array
            items:
              $ref: '#/components/schemas/JobOutput'
    JobOutputCollectionsSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            type: array
            items:
              $ref: '#/components/schemas/JobOutputCollection'
    PageContentSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            type: object
            properties:
              gid:
                type: string
                title: GID
                example: www.ebay.com-fda1505c93d29c8ff97d885710d3e075
              available:
                type: boolean
                title: Available
              signed_url:
                type: string
                title: Signed URL
                format: uri
                description: The Signed URL to download the content.
                example: https://f002.backblazeb2.com/file/content-fetch-answersengine/www.ebay.com-fda1505c93d29c8ff97d885710d3e075?Authorization=3_20200304173320_8f7994afe3082e44ff43ec13_14fb550cbb8cd4a1f46f0dfb5bcee1e102b37cc8_002_20200304183320_0000_dnl
              preview_url:
                type: string
                title: Preview URL
                description: The URL to the preview of the content.
                format: uri
                example: https://app.datahen.com/public/global_pages/preview/5h89NZUEKRjIfeFAnjgKeBs_OuaxrhM7YYN0P0yTuKZh_fsqiFs8O6ssarbRmD0PyBQsan6vZ3LuZDnIboaseZJj2XPaMtJhQvvf7LjwRYMvB0f4yGuTOqVbAsADTeRmRdmUYzMoMT5bIHVJ6y8dnZPA1tyyLKxiiQHuHWKrAKyN4Eez1jRCv7ooitYms3Lq-kY6efmMTS_G6QdPWeMyPp5X8vP7ikT9wdTfLnIdNWmG5kG5VMJYtLZotibWiFUGGi_qJfMqxc6jvMCMg32sB08sag8ljJNw4NQBls_2imf3LwR6WX3UXJzXVluZkqXD3YH8OzCua4IR
    SchemaSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            oneOf:
            - $ref: '#/components/schemas/TabularSchema'
            - $ref: '#/components/schemas/JSONSchema'
          example:
            $id: 'https://example.com/person.schema.json'
            $schema: 'http://json-schema.org/draft-07/schema#'
            title: Person
            type: object
            properties:
              firstName:
                type: string
                description: The person's first name.
              lastName:
                type: string
                description: The person's last name.
              age:
                description: Age in years which must be equal to or greater than zero.
                type: integer
                minimum: 0
    JobSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/ScrapeJob'
    JobsSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            type: array
            items:
              $ref: '#/components/schemas/ScrapeJob'
    JobStatSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/JobStat'
    JobStatHistorySuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            type: array
            items:
              $ref: '#/components/schemas/JobStat'
    ScraperSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Scraper'
    ScrapersSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            type: array
            items:
              $ref: '#/components/schemas/Scraper'
    VarSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/SetUnsetVar'
    VarsSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            type: array
            items:
              $ref: '#/components/schemas/SetUnsetVar'
    ExportSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            $ref: '#/components/schemas/Export'
    ExportsSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            type: array
            items:
              $ref: '#/components/schemas/Export'
    DownloadExportSuccess:
      description: Successful Response
      content:
        application/json:
          schema:
            type: object
            properties:
              signed_url: 
                type: string
                format: uri
                title: Signed URL
                description: The Signed URL to download the export.
                example: 'https://f002.backblazeb2.com/file/exports-fetch-answersengine/76ef855b13dd41a6a1353eb0c5767fe0.tgz?Authorization=3_20200302120858_7b649d805498a58312bebfda_f665e0bca5285f8279cba79cbb7af9e18b7f9af2_002_20200302130858_0000_dnld'
              
    DeleteTokenSuccess:
      description: Successful Delete
      content:
          application/json:
            schema:
              type: object
              properties:
                Token: 
                  type: string
                  example: abcdefghijklmno1234567890
                Deleted: 
                  type: boolean
                  example: true
    DeleteSuccess:
      description: Successful Delete
      content:
          application/json:
            schema:
              type: object
              properties:
                Name: 
                  type: string
                  example: foo
                Deleted: 
                  type: boolean
                  example: true
    PaginatedResponse:
      description: Paginated response headers
      headers:
        x-pagination-current-page: 
          description: "Current page number"
          schema:
            type: integer
        x-pagination-first-page: 
          description: "First page number"
          schema:
            type: integer
        x-pagination-last-page:  
          description: "Last page number"
          schema:
            type: integer
        x-pagination-per-page: 
          description: "Number of records per page"
          schema:
            type: integer
        x-pagination-total-pages: 
          description: "Total number of pages"
          schema:
            type: integer
        x-pagination-total-records:
          description: "Total records found"
          schema:
            type: integer
    UnauthorizedError:
      description: Unauthorized
    AuthTokenMissing:
      description: Access token is missing or invalid
    UnprocessableEntity:
      description: Unprocessable Entity
    NotAcceptable:
      description: Not Acceptable
  parameters:
    PageNum:
      in: query
      name: p
      required: false
      schema:
        type: integer
        minimum: 1
        default: 1
      description: Page number
    PerPage:
      in: query
      name: pp
      required: false
      schema:
        type: integer
        minimum: 1
        maximum: 500
        default: 30
      description: Number of records per page. Maximum 500 per page.
    MaxTimestamp:
      in: query
      name: max-timestamp
      required: false
      schema:
        type: string
        format: date-time
        example: '2020-03-04T13:13:59.341449Z'
      description: Filter Records with Maximum timestamp (Must be in RFC3339 Format).
    MinTimestamp:
      in: query
      name: min-timestamp
      required: false
      schema:
        type: string
        format: date-time
        example: '2020-02-04T13:13:59.341449Z'
      description: Filter Records with Minimum timestamp (Must be in RFC3339 Format).
    VarName:
      in: path
      name: var_name
      required: true
      schema:
        type: string
        pattern: '^[a-zA-Z0-9_]+$'
      description: Variable name.
    Collection:
      in: path
      name: collection
      required: true
      example: default
      schema:
        type: string
      description: Name of the collection.
    QueryCollection:
      in: query
      name: collection
      schema:
        type: string
      description: Name of the collection.
    ExporterType:
      in: query
      name: exporter_type
      schema:
        type: string
        enum:
        - json
        - csv
        - content
      description: Type of exporter.
    ExporterWriteMode:
      in: query
      name: write_mode
      schema:
        type: string
        enum:
        - line
        - pretty
        - array
        - pretty_array
      description: Write Mode of Exporter. Must be used together with Exporter Type parameter.
    SchemaType:
      in: query
      name: schema_type
      required: false
      schema:
        type: string
        enum:
        - tabular
      description: Type of Schema.
    ConnectorType:
      in: query
      name: connector_type
      required: false
      schema:
        type: string
        enum:
        - gds
      description: Type of Connector. For example `gds` for Google Data Studio.
    JobOutputQuery:
      in: query
      name: q
      required: false
      schema:
        type: string
      description: Query must be in JSON format.
      example:
        _id: 764c645759804254bf9f5c1a18091f76
    JobOutputRecordID:
      in: path
      name: record_id
      required: true
      schema:
        type: string
      description: Record ID of the Job Output
    ExportID:
      in: path
      name: export_id
      required: true
      schema:
        type: string
      description: Export ID.
    GID:
      in: path
      name: gid
      required: true
      schema:
        type: string
      description: Global ID of pages.
    ScraperName:
      in: path
      name: name
      required: true
      schema:
        type: string
        pattern: '^[a-zA-Z0-9_-]+$'
      description: Name of the scraper.
    ExporterName:
      in: path
      name: exporter_name
      required: true
      schema:
        type: string
        pattern: '^[a-zA-Z0-9_-]+$'
      description: Name of the Exporter.
    ExportFields:
      in: query
      name: fields
      style: form
      explode: false
      schema:
        type: array
        items:
          type: string
      description: Fields on the export data.
    LogJobType:
      in: query
      name: job_type
      schema:
        type: string
        enum:
        - parsing
        - seeding
      description: Job Type. Can be `parsing` or `seeding` or leave blank, for stdout from the Job container.
    LogPageToken:
      in: query
      name: page_token
      schema:
        type: string
      description: Page Token of the current page of the log entries. 
    LogOrder:
      in: query
      name: order
      schema:
        type: string
        enum:
        - head
        - tail
      description: Order of the log entries. Defaults to `tail`.
    JobID:
      in: path
      name: id
      required: true
      schema:
        type: integer
        format: int64
      description: ID of the Scrape Job.
    Token:
      in: path
      name: token
      required: true
      schema:
        type: string
      description: Auth Token
    FetchFail:
      in: query
      name: fetchfail
      required: false
      schema:
        type: boolean
      description: Filter by Fetch Fails
    ParseFail:
      in: query
      name: parsefail
      required: false
      schema:
        type: boolean
      description: Filter by Parse Fails
    PageType:
      in: query
      name: page_type
      required: false
      schema:
        type: string
      description: Filter by Page Type
    JobPageStatus:
      in: query
      name: status
      required: false
      schema:
        type: string
        enum: 
          - to_fetch
          - fetching
          - fetching_failed
          - fetching_dequeue_failed
          - to_parse
          - parsing_started
          - parsing
          - parsed
          - parsing_failed
          - parsing_dequeue_failed
          - limbo
      description: Filter by Job Page Status
    JobOutputCollectionFromSchema:
      in: query
      name: from_schema
      required: false
      schema:
        type: boolean
      description: If set to true, will return Collections from the Schema.
  schemas:
    GlobalPage:
      type: object
      title: Global Page
      description: a unique Page that was or is being scraped by all scrapers on the account.
      required:
        - gid
        - fetch_type
        - hostname
        - method
        - ua_type
        - status
      properties:
        gid:
          type: string
          title: GID (Global ID)
          description: The Globally unique ID of a page request.
          example: www.ebay.com-fda1505c93d29c8ff97d885710d3e075
        fetch_type:
          type: string
          title: Fetch Type
          description: The type of fetcher to use (standard, or browser, or fullbrowser).
          default: 'standard'
          enum:
          - standard
          - browser
          - fullbrowser
          example: standard
        hostname:
          type: string
          title: Hostname
          description: Hostname of the URL.
          example: www.ebay.com
        method:
          type: string
          title: HTTP Method
          description: HTTP Method of the request
          default: 'GET'
          enum:
            - GET
            - PUT
            - POST
            - DELETE
            - PATCH
            - HEAD
            - OPTIONS
        url:
          type: string
          title: URL
          description: URL of the request.
          default: ''
          example: 'https://www.ebay.com/itm/Apple-Macbook-Pro-13-Touchbar-ID-MUHN2LL-A-i5-8GB-128GB-Space-Gray-2019-Model'
        effective_url:
          type: string
          title: Effective URL
          description: The final URL after the request is being redirected by the target server.
          example: 'https://www.ebay.com/itm/Apple-Macbook-Pro-13-Touchbar-ID-MUHN2LL-A-i5-8GB-128GB-Space-Gray-2019-Model/303469768584?epid=14032813598&hash=item46a8352f88%3Ag%3AeNwAAOSwkXdeNfdd'
        headers:
          title: Headers
          description: Request Headers
          default: null
          type: object
          example: 
            Header1: foo
            Header2: bar
        cookie:
          title: Cookie
          type: string
          description: The Cookie that can be set and read from.
          example: yummy_cookie=choco; tasty_cookie=strawberry
        body:
          title: Body
          type: string
          description: Request Body.
          default: null
          example: foo=foo1&bar=bar1
        created_at:
          type: string
          title: Created At
          description: Creation Date of this Global Page
          format: date-time
          example: '2020-02-29T07:56:35.307162Z'
        no_redirect:
          type: boolean
          title: No Redirect
          description: If set to true, the fetcher does not do redirection at all on this request.
          default: false
        no_url_encode:
          type: boolean
          title: No URL Encode
          description: If set to true, will use raw URL for the page request.
          default: false
        http2:
          type: boolean
          title: HTTP2
          description: If set to true, will use HTTP2 on standard fetchers.
          default: false
        ua_type:
          type: string
          title: UA Type
          description: User Agent Type. What kind of randomized user agents to use (either desktop or mobile).
          default: 'desktop'
          enum:
          - desktop
          - mobile
        fetching_at:
          type: string
          title: Fetching At
          description: When the Fetching was attempted.
          format: date-time
          example: '2001-01-01T00:00:00Z'
        fetched_at:
          type: string
          title: Fetched At
          description: When the Fetching was done.
          format: date-time
          example: '2020-03-02T11:33:09.531705Z'
        to_fetch:
          type: string
          title: To Fetch
          description: When the Fetching of this page should be available to be started.
          default: ''
          example: '2020-02-29T07:56:35.307162Z'
        total_successes:
          type: integer
          title: Total Successes
          description: Total Successes of this page while being fetched.
          default: 0
        total_requests:
          type: integer
          title: Total Requests
          description: Total Requests of this page while being fetched.
          default: 0
        total_failures:
          type: integer
          title: Total Failures
          description: Total Failures of this page while being fetched.
          default: 0
        response_checksum:
          type: string
          title: Response Checksum
          description: The Checksum of the content of the HTTP response.
          default: ''
          example: 3bbcdb5a6c666e48b7a5c49d6f19af42
        response_status:
          type: string
          title: Response Status
          description: HTTP Status of the response.
          default: ''
          example: 200 OK
        response_status_code:
          type: integer
          title: Response Status Code
          description: The HTTP Status Code of the response.
          example: 200
        response_headers:
          type: object
          title: Response Headers
          description: HTTP Headers of the response.
          default: {}
          example:
            X-Frame-Options:
              - SAMEORIGIN
            X-Sw-No-Cache:
              - 'true'
            Content-Encoding:
              - gzip
            Content-Language:
              - en
            Connection:
              - keep-alive
              - Transfer-Encoding
            Date:
              - 'Mon, 02 Mar 2020 11:33:06 GMT'
        response_cookie:
          type: string
          title: Response Cookie
          description: The Cookie that is set by the target server as part of the HTTP response.
          default: ''
          example: yummy_cookie=choco; tasty_cookie=strawberry
        response_proto:
          type: string
          title: Response Proto
          description: The Protocol of the http response.
          default: ''
          example: HTTP/1.1
        content_type:
          type: string
          title: Content Type
          description: HTTP Content Type of the response.
          default: ''
          example: text/html;charset=UTF-8
        failed_response_checksum:
          title: Failed Response Checksum
          type: string
          description: The Checksum of the content of the failed HTTP response.
          default: ''
          example: 3bbcdb5a6c666e48b7a5c49d6f19af42
        failed_response_status:
          type: string
          title: Failed Response Status
          description: HTTP Status of the failed HTTP response.
          default: ''
          example: 500 Internal Server Error
        failed_response_status_code:
          title: Failed Response Status
          type: integer
          description: HTTP Status of the failed HTTP response.
          example: 500
        failed_response_headers:
          type: object
          title: Failed Response Headers
          description: HTTP Headers of the failed HTTP response.
          default: {}
          example:
            X-Frame-Options:
              - SAMEORIGIN
            X-Sw-No-Cache:
              - 'true'
            Content-Encoding:
              - gzip
            Content-Language:
              - en
            Connection:
              - keep-alive
              - Transfer-Encoding
            Date:
              - 'Mon, 02 Mar 2020 11:33:06 GMT'
        failed_response_cookie:
          type: string
          title: Failed Response Cookie
          description: The Cookie that is set by the target server as part of the failed HTTP response.
          default: ''
          example: yummy_cookie=choco; tasty_cookie=strawberry
        failed_response_proto:
          type: string
          title: FAILED Response Proto
          description: The Protocol of the failed HTTP response.
          default: ''
          example: HTTP/1.1
        failed_cid:
          title: Failed CID (Content ID)
          description: The Content ID (Hash) of the failed HTTP response.
          default: null
          example: www.ebay.com-3bbcdb5a6c666e48b7a5c49d6f19af42
        failed_content_type:
          type: string
          title: Failed Content Type
          description: HTTP Content Type of the failed response.
          default: ''
          example: text/html;charset=UTF-8
        failed_effective_url:
          type: string
          title: Failed Effective URL
          description: The final URL after the failed request is being redirected by the target server.
          example: 'https://www.ebay.com/itm/Apple-Macbook-Pro-13-Touchbar-ID-MUHN2LL-A-i5-8GB-128GB-Space-Gray-2019-Model/303469768584?epid=14032813598&hash=item46a8352f88%3Ag%3AeNwAAOSwkXdeNfdd'
        failed_at:
          type: string
          title: Failed At
          description: When the fetching failed.
          format: date-time
          example: '2020-03-02T11:33:09.531705Z'
        content_size:
          type: integer
          title: Content Size
          description: The Size of the content of the HTTP response.
          default: 0
          example: 65650
        failed_content_size:
          type: integer
          title: Failed Content Size
          description: The Size of the content of the failed HTTP response.
          default: 0
          example: 65650
        status:
          type: string
          title: Status
          description: The Status of the Global Page
          enum: 
          - to_fetch
          - fetching
          - fetched
          example:
            - fetched
    JobPage:
      type: object
      title: Job Page
      description: a Page on a Job
      required:
        - gid
        - url
        - job_id
        - job_status
        - status
        - fetch_type
        - page_type
        - hostname
        - method
        - ua_type
      properties:
        gid:
          type: string
          title: GID (Global ID)
          description: The Globally unique ID of a page request.
          example: www.ebay.com-fda1505c93d29c8ff97d885710d3e075
        job_id:
          type: integer
          title: Job ID
          example: 123
        job_status:
          type: string
          title: Job Status
          example: active
          enum:
          - active
          - cancelled
          - done
          - paused
          - cancelling
          - finishing
        status:
          type: string
          title: Status
          description: The Status of the Job Page
          enum: 
          - to_fetch
          - fetching
          - fetching_failed
          - fetching_dequeue_failed
          - to_parse
          - parsing_started
          - parsing
          - parsed
          - parsing_failed
          - parsing_dequeue_failed
          - limbo
          example:
            - to_fetch
        page_type:
          type: string
          title: Page Type
          description: The type of Page. Grouping multiple pages into certain page types are a useful way to organize pages that can be parsed by their types.
        priority:
          type: integer
          title: Priority
          description: Priority of this Page. Defaults to 0. Higher numbers means will get fetched sooner
          default: 0
        fetch_type:
          type: string
          title: Fetch Type
          description: The type of fetcher to use (standard, or browser, or fullbrowser).
          default: 'standard'
          enum:
          - standard
          - browser
          - fullbrowser
          example: standard
        hostname:
          type: string
          title: Hostname
          description: Hostname of the URL.
          example: www.ebay.com
        method:
          type: string
          title: HTTP Method
          description: HTTP Method of the request
          default: 'GET'
          enum:
            - GET
            - PUT
            - POST
            - DELETE
            - PATCH
            - HEAD
            - OPTIONS
        url:
          type: string
          title: URL
          description: URL of the request.
          default: ''
          example: 'https://www.ebay.com/itm/Apple-Macbook-Pro-13-Touchbar-ID-MUHN2LL-A-i5-8GB-128GB-Space-Gray-2019-Model'
        effective_url:
          type: string
          title: Effective URL
          description: The final URL after the request is being redirected by the target server.
          example: 'https://www.ebay.com/itm/Apple-Macbook-Pro-13-Touchbar-ID-MUHN2LL-A-i5-8GB-128GB-Space-Gray-2019-Model/303469768584?epid=14032813598&hash=item46a8352f88%3Ag%3AeNwAAOSwkXdeNfdd'
        headers:
          title: Headers
          description: Request Headers
          default: null
          type: object
          example: 
            Header1: foo
            Header2: bar
        cookie:
          title: Cookie
          type: string
          description: The Cookie that can be set and read from.
          example: yummy_cookie=choco; tasty_cookie=strawberry
        body:
          title: Body
          type: string
          description: Request Body.
          default: null
          example: foo=foo1&bar=bar1
        created_at:
          type: string
          title: Created At
          description: Creation Date of this Global Page
          format: date-time
          example: '2020-02-29T07:56:35.307162Z'
        no_redirect:
          type: boolean
          title: No Redirect
          description: If set to true, the fetcher does not do redirection at all on this request.
          default: false
        no_url_encode:
          type: boolean
          title: No URL Encode
          description: If set to true, will use raw URL for the page request.
          default: false
        http2:
          type: boolean
          title: HTTP2
          description: If set to true, will use HTTP2 on standard fetchers.
          default: false
        ua_type:
          type: string
          title: UA Type
          description: User Agent Type. What kind of randomized user agents to use (either desktop or mobile).
          default: 'desktop'
          enum:
          - desktop
          - mobile
        freshness:
          type: string
          format: date-time
          title: Freshness
          description: Freshness criteria. When is this page considered fresh.
        fresh:
          type: boolean
          title: Fresh
          description: Is this page Fresh.
        fetching_at:
          type: string
          title: Fetching At
          description: When the Fetching was attempted.
          format: date-time
          example: '2001-01-01T00:00:00Z'
        fetched_at:
          type: string
          title: Fetched At
          description: When the Fetching was done.
          format: date-time
          example: '2020-03-02T11:33:09.531705Z'
        to_fetch:
          type: string
          title: To Fetch
          description: When the Fetching of this page should be available to be started.
          default: ''
          example: '2020-02-29T07:56:35.307162Z'
        total_successes:
          type: integer
          title: Total Successes
          description: Total Successes of this page when being fetched.
          default: 0
        total_requests:
          type: integer
          title: Total Requests
          description: Total Requests of this page when being fetched.
          default: 0
        total_failures:
          type: integer
          title: Total Failures
          description: Total Failures of this page when being fetched.
          default: 0
        response_checksum:
          type: string
          title: Response Checksum
          description: The Checksum of the content of the HTTP response.
          default: ''
          example: 3bbcdb5a6c666e48b7a5c49d6f19af42
        response_status:
          type: string
          title: Response Status
          description: HTTP Status of the response.
          default: ''
          example: 200 OK
        response_status_code:
          type: integer
          title: Response Status Code
          description: The HTTP Status Code of the response.
          example: 200
        response_headers:
          type: object
          title: Response Headers
          description: HTTP Headers of the response.
          default: {}
          example:
            X-Frame-Options:
              - SAMEORIGIN
            X-Sw-No-Cache:
              - 'true'
            Content-Encoding:
              - gzip
            Content-Language:
              - en
            Connection:
              - keep-alive
              - Transfer-Encoding
            Date:
              - 'Mon, 02 Mar 2020 11:33:06 GMT'
        response_cookie:
          type: string
          title: Response Cookie
          description: The Cookie that is set by the target server as part of the HTTP response.
          default: ''
          example: yummy_cookie=choco; tasty_cookie=strawberry
        response_proto:
          type: string
          title: Response Proto
          description: The Protocol of the http response.
          default: ''
          example: HTTP/1.1
        content_type:
          type: string
          title: Content Type
          description: HTTP Content Type of the response.
          default: ''
          example: text/html;charset=UTF-8
        failed_response_checksum:
          title: Failed Response Checksum
          type: string
          description: The Checksum of the content of the failed HTTP response.
          default: ''
          example: 3bbcdb5a6c666e48b7a5c49d6f19af42
        failed_response_status:
          type: string
          title: Failed Response Status
          description: HTTP Status of the failed HTTP response.
          default: ''
          example: 500 Internal Server Error
        failed_response_status_code:
          title: Failed Response Status
          type: integer
          description: HTTP Status of the failed HTTP response.
          example: 500
        failed_response_headers:
          type: object
          title: Failed Response Headers
          description: HTTP Headers of the failed HTTP response.
          default: {}
          example:
            X-Frame-Options:
              - SAMEORIGIN
            X-Sw-No-Cache:
              - 'true'
            Content-Encoding:
              - gzip
            Content-Language:
              - en
            Connection:
              - keep-alive
              - Transfer-Encoding
            Date:
              - 'Mon, 02 Mar 2020 11:33:06 GMT'
        failed_response_cookie:
          type: string
          title: Failed Response Cookie
          description: The Cookie that is set by the target server as part of the failed HTTP response.
          default: ''
          example: yummy_cookie=choco; tasty_cookie=strawberry
        failed_response_proto:
          type: string
          title: FAILED Response Proto
          description: The Protocol of the failed HTTP response.
          default: ''
          example: HTTP/1.1
        failed_content_type:
          type: string
          title: Failed Content Type
          description: HTTP Content Type of the failed response.
          default: ''
          example: text/html;charset=UTF-8
        failed_effective_url:
          type: string
          title: Failed Effective URL
          description: The final URL after the failed request is being redirected by the target server.
          example: 'https://www.ebay.com/itm/Apple-Macbook-Pro-13-Touchbar-ID-MUHN2LL-A-i5-8GB-128GB-Space-Gray-2019-Model/303469768584?epid=14032813598&hash=item46a8352f88%3Ag%3AeNwAAOSwkXdeNfdd'
        failed_at:
          type: string
          title: Failed At
          description: When the fetching failed.
          format: date-time
          example: '2020-03-02T11:33:09.531705Z'
        content_size:
          type: integer
          title: Content Size
          description: The Size of the content of the HTTP response.
          default: 0
          example: 65650
        failed_content_size:
          type: integer
          title: Failed Content Size
          description: The Size of the content of the failed HTTP response.
          default: 0
          example: 65650
        fetched_from:
          type: string
          enum:
          - web
          - cache
        reset:
          type: boolean
          default: false
        vars:
          type: object
          title: Vars
          description: Page Variables field allow you to add arbitrary variables to a page which is accessable later.
          default: {}
          example:
           foo: foo1
           bar: bar1
        driver:
          type: object
          title: Driver
          description: Driver store driver options to interact with browser fetchers via Puppeteer JS
          properties:
            name:
              type: string
              description: The name of the driver.
            code:
              type: string
              description: The actual Puppeteer JS code
            goto_options:
              type: object
              description: Goto Options of Puppeter JS
            stealth:
              type: boolean
              description: When set to true, will use Puppeteer Stealth plugin.
        display:
          type: object
          title: Display
          description: Size of the browser display to use.
          properties:
            width:
              type: integer
            height:
              type: integer
        screenshot:
          type: object
          title: Screenshot
          properties:
            take_screenshot:
              type: boolean
            options:
              type: object
        driver_log:
          type: string
          title: Driver Log
          description: The log of the fetching process of the browser interaction.
        force_fetch:
          type: boolean
          title: Force Fetch
          description: Forces the fetcher to fetch this page, if page in the cache is not fresh.
          deprecated: true
    CreateJobPage:
      type: object
      title: Job Page
      description: Add a Page on a Job
      example:
        url: https://www.ebay.com
        page_type: listings
        vars:
          foo: bar
      required:
        - url
      properties:
        url:
          type: string
          title: URL
          description: URL of the request.
          example: 'https://www.ebay.com/itm/Apple-Macbook-Pro-13-Touchbar-ID-MUHN2LL-A-i5-8GB-128GB-Space-Gray-2019-Model'
        page_type:
          type: string
          title: Page Type
          description: The type of Page. Grouping multiple pages into certain page types are a useful way to organize pages that can be parsed by their types.
        priority:
          type: integer
          title: Priority
          description: Priority of this Page. Defaults to 0. Higher numbers means will get fetched sooner
          default: 0
        fetch_type:
          type: string
          title: Fetch Type
          description: The type of fetcher to use (standard, or browser, or fullbrowser).
          default: 'standard'
          enum:
          - standard
          - browser
          - fullbrowser
          example: standard
        method:
          type: string
          title: HTTP Method
          description: HTTP Method of the request
          default: 'GET'
          enum:
            - GET
            - PUT
            - POST
            - DELETE
            - PATCH
            - HEAD
            - OPTIONS
        
        headers:
          title: Headers
          description: Request Headers
          default: null
          type: object
          example: 
            Header1: foo
            Header2: bar
        cookie:
          title: Cookie
          type: string
          description: The Cookie that can be set and read from.
          example: yummy_cookie=choco; tasty_cookie=strawberry
        body:
          title: Body
          type: string
          description: Request Body.
          default: null
          example: foo=foo1&bar=bar1
        no_redirect:
          type: boolean
          title: No Redirect
          description: If set to true, the fetcher does not do redirection at all on this request.
          default: false
        no_url_encode:
          type: boolean
          title: No URL Encode
          description: If set to true, will use raw URL for the page request.
          default: false
        http2:
          type: boolean
          title: HTTP2
          description: If set to true, will use HTTP2 on standard fetchers.
          default: false
        ua_type:
          type: string
          title: UA Type
          description: User Agent Type. What kind of randomized user agents to use (either desktop or mobile).
          default: 'desktop'
          enum:
          - desktop
          - mobile
        freshness:
          type: string
          format: date-time
          title: Freshness
          description: Freshness criteria. When is this page considered fresh.
        reset:
          type: boolean
          default: false
        vars:
          type: object
          title: Vars
          description: Page Variables field allow you to add arbitrary variables to a page which is accessable later.
          default: {}
          example:
           foo: foo1
           bar: bar1
        driver:
          type: object
          title: Driver
          description: Driver store driver options to interact with browser fetchers via Puppeteer JS
          properties:
            name:
              type: string
              description: The name of the driver.
            code:
              type: string
              description: The actual Puppeteer JS code
            goto_options:
              type: object
              description: Goto Options of Puppeter JS
            stealth:
              type: boolean
              description: When set to true, will use Puppeteer Stealth plugin.
              default: false
        display:
          type: object
          title: Display
          description: Size of the browser display to use.
          properties:
            width:
              type: integer
            height:
              type: integer
        screenshot:
          type: object
          title: Screenshot
          properties:
            take_screenshot:
              type: boolean
            options:
              type: object
        driver_log:
          type: string
          title: Driver Log
          description: The log of the fetching process of the browser interaction.
        force_fetch:
          type: boolean
          title: Force Fetch
          description: Forces the fetcher to fetch this page, if page in the cache is not fresh.
          deprecated: true
    UpdateJobPage:
      type: object
      title: Job Page
      description: Update a Page on a Job
      example:
        page_type: listings
        priority: 1
        vars:
          foo: foo1
          bar: bar1
      properties:
        page_type:
          type: string
          title: Page Type
          description: The type of Page. Grouping multiple pages into certain page types are a useful way to organize pages that can be parsed by their types.
          default: default
        priority:
          type: integer
          title: Priority
          description: Priority of this Page. Defaults to 0. Higher numbers means will get fetched sooner
          default: 0
        vars:
          type: object
          title: Vars
          description: Page Variables field allow you to add arbitrary variables to a page which is accessable later.
          default: {}
          example:
           foo: foo1
           bar: bar1
    Scraper:
      required:
      - name
      - freshness_type
      - git_repository
      properties:
        name:
          type: string
          title: Name
          description: Scraper Name.
          pattern: '^[a-zA-Z0-9_-]+$'
          example: ebay-scraper
        id:
          type: integer
          title: ID
          description: Scraper ID.
          example: 123
        account_id:
          type: integer
          title: Account ID
          description: Account ID that the scraper belongs to.
          example: 123
        force_fetch:
          type: boolean
          title: Force Fetch
          description: Forces the fetcher to fetch pages, if page in the cache is not fresh.
          default: false
        freshness_type:
          type: string
          title: Freshness type
          description: "How fresh should the cached page be used. Default: any."
          enum: [day,week,month,year,any]
          default: any
        proxy_type:
          type: string
          title: Proxy Type
          description: Uses certain types of proxies that are provided by DataHen.
          example: standard
        created_at:
          type: string
          format: date-time
          title: Created At
          description: Scraper Creation Date & Time.
          example: '2018-11-26T22:00:43.007755Z'
        git_repository:
          type: string
          format: uri
          title: Git Repository
          description: URL to the Git repository that the scraper uses.
          example: https://github.com/datahenofficial/ebay-scraper.git
        git_branch:
          type: string
          title: Git Branch
          description: Branch of the Git Repository.
          default: master
          example: master
        deployed_git_repository:
          type: string
          format: uri
          title: Deployed Git Repository
          description: The deployed URL to the Git repository that the scraper uses.
          example: https://github.com/datahenofficial/ebay-scraper.git
        deployed_git_branch:
          type: string
          title: Deployed Git Branch
          description: The deployed Branch of the Git repository.
          default: master
          example: master
        deployed_commit_hash:
          type: string
          title: Deployed Commit Hash
          description: The Deployed Commit Hash of the Git Repository.
          example: a0f4999ec6b01e868c0a75516d52d18ee62631b7
        deployed_at:
          type: string
          format: date-time
          title: Deployed At
          description: When did the deployment occured.
          example: '2020-02-29T06:50:29.005823Z'
        standard_worker_count:
          type: integer
          title: Standard Worker Count
          description: How many Standard workers are allocated to this scraper.
          example: 1
        browser_worker_count:
          type: integer
          title: Browser Worker Count
          description: How many Browser workers are allocated ot this scarper.
          example: 0
        config:
          $ref: "#/components/schemas/ScraperConfig"
        disable_scheduler:
          type: boolean
          title: Disable Scheduler
          description: Specify whether the CRON scheduler should be disabled or not.
          default: false
        cancel_current_job:
          type: boolean
          title: Cancel Current Job
          description: Specify if the CRON scheduler should cancel the currently active scraper job before creating a new one.
          default: false
        schedule:
          type: string
          title: Schedule
          description: Specify that the scraper should run on a regular schedule. Using CRON expression.
          example: 0 8 * * * *
        timezone:
          type: string
          title: Timezone
          description: Timezone in which the scraper scheduler should run at. Must be in the IANA standard.
          example: America/Toronto
        next_run:
          type: string
          title: Next Run
          description: When would the next time the scraper would run (a new scrape job is created by the sheduler).
          format: date-time
          example: '2020-02-29T13:00:00Z'
        run_attempted_at:
          type: string
          title: Run Attempted At
          description: Last time the scheduler attempts to run a scraper.
          format: date-time
          example: '2020-02-29T13:30:26.542679Z'
        run_attempts:
          type: integer
          title: Run Attempts
          description: How many times a scheduler attempts to run a scraper on that exact scheduled run. A number greater than 1 means that the scheduler failed to run a scraper on that exact schedule.
          example: 3
        last_run:
          type: string
          title: Last Run
          description: When was the last time the scraper ran.
          format: date-time
          example: '2020-02-28T13:05:38.327667Z'
        current_job:
          $ref: '#/components/schemas/ScrapeJob'
    CreateUpdateScraper:
      required:
      - name
      - git_repository
      properties:
        name:
          type: string
          title: Name
          description: Scraper Name.
          pattern: '^[a-zA-Z0-9_-]+$'
          example: ebay-scraper
        force_fetch:
          type: boolean
          title: Force Fetch
          description: Forces the fetcher to fetch pages, if page in the cache is not fresh.
          default: false
        freshness_type:
          type: string
          title: Freshness type
          description: "How fresh should the cached page be used. Default: any."
          enum: [day,week,month,year,any]
          default: any
        proxy_type:
          type: string
          title: Proxy Type
          description: Uses certain types of proxies that are provided by DataHen.
          example: standard
        git_repository:
          type: string
          format: uri
          title: Git Repository
          description: URL to the Git repository that the scraper uses.
          example: https://github.com/datahenofficial/ebay-scraper.git
        git_branch:
          type: string
          title: Git Branch
          description: Branch of the Git Repository.
          default: master
          example: master
        standard_worker_count:
          type: integer
          title: Standard Worker Count
          description: How many Standard workers are allocated to this scraper.
          example: 1
          default: 1
        browser_worker_count:
          type: integer
          title: Browser Worker Count
          description: How many Browser workers are allocated ot this scarper.
          example: 0
          default: 0
        disable_scheduler:
          type: boolean
          title: Disable Scheduler
          description: Specify whether the CRON scheduler should be disabled or not.
          default: false
        cancel_current_job:
          type: boolean
          title: Cancel Current Job
          description: Specify if the CRON scheduler should cancel the currently active scraper job before creating a new one.
          default: false
        schedule:
          type: string
          title: Schedule
          description: Specify that the scraper should run on a regular schedule. Using CRON expression.
          example: 0 8 * * * *
        timezone:
          type: string
          title: Timezone
          description: Timezone in which the scraper scheduler should run at. Must be in the IANA standard.
          example: America/Toronto
    ScraperConfig:
      type: object
      title: ScraperConfig
      description: Scraper Configuration
      example:
        exporters:
        - exporter_type: json
          write_mode: pretty_array
          collection: products
          limit: 10
          offset: 0
          order: desc
          exporter_name: products_last10_json
        - start_on_job_done: true
          limit: 
          offset: 
          exporter_name: products_json
          exporter_type: json
          collection: products
        - collection: products
          start_on_job_done: true
          limit: 
          offset: 
          exporter_name: products_csv
          fields:
          - path: _gid
            header: gid
          - header: created_at
            path: _created_at
          - path: title
            header: title
          - header: price
            path: price
          - header: feedback
            path: feedback
          - header: seller
            path: seller
          exporter_type: csv
        finisher:
          disabled: true
          file: ''
        parsers:
        - page_type: listings
          file: "./parsers/listings.rb"
        - page_type: details
          file: "./parsers/details.rb"
        input_vars:
        - type: string
          default: macbook
          title: Search
          description: 'Search for something. Example: macbook'
          required: true
          name: search
        schemas:
        - schema: |-
            {
              "definitions": {},
              "$schema": "http://json-schema.org/draft-07/schema#",
              "$id": "http://example.com/root.json",
              "type": "object",
              "title": "The Root Schema",
              "required": [
                "feedback",
                "price",
                "seller",
                "title"
              ],
              "properties": {
                "feedback": {
                  "$id": "#/properties/feedback",
                  "type": "string",
                  "title": "The Feedback Schema",
                  "default": "",
                  "examples": [
                    "100% Positive feedback"
                  ],
                  "pattern": "^(.*)$"
                },
                "price": {
                  "$id": "#/properties/price",
                  "type": "number",
                  "title": "The Price Schema",
                  "description": "The description of the price schema",
                  "default": 0.0,
                  "examples": [
                    450.99
                  ]
                },
                "seller": {
                  "$id": "#/properties/seller",
                  "type": "string",
                  "title": "The Seller Schema",
                  "default": "",
                  "examples": [
                    "tlkennedy02012"
                  ],
                  "pattern": "^(.*)$"
                },
                "title": {
                  "$id": "#/properties/title",
                  "type": "string",
                  "title": "The Title Schema",
                  "default": "",
                  "examples": [
                    "Apple Iphone X 256 GB Unlocked (verizon)"
                  ],
                  "pattern": "^(.*)$"
                }
              }
            }
          collections: products
        seeder:
          file: "./seeder/seeder.rb"
      properties:
        exporters:
          type: array
          items:
            oneOf:
            - $ref: '#/components/schemas/JSONExporterConfig'
            - $ref: '#/components/schemas/CSVExporterConfig'
            - $ref: '#/components/schemas/ContentExporterConfig'
        finisher:
          type: object
          title: Finisher
          description: Finisher specifies which file needs to be run when the Scrape Job is finishing.
          example:
            disabled: true
            file: './finisher/finisher.rb'
          required:
          - file
          properties:
            disabled:
              type: boolean
              title: Disabled
              default: false
              example: true
            file:
              type: string
              title: File
              description: Specifies which file to be executed
              example: './finisher/finisher.rb'
        input_vars:
          type: array
          title: Input Variables
          description: Input Variables specifies a variables that are accessable by the Scraper or its Job.
          items:
            $ref: '#/components/schemas/InputVar'
        parsers:
          type: array
          title: Parsers
          description: Specifies parser files that is to be executed by the scarper.
          items:
            type: object
            example:
            - page_type: listings
              file: "./parsers/listings.rb"
            - page_type: details
              file: "./parsers/details.rb"
            required:
            - file
            - page_type
            properties:
              file:
                type: string
                title: File
                description: The path of the File to be executed.
                example: "./parsers/listings.rb"
              page_type:
                type: string
                title: Page Type
                description: Page Type allows for pages to be grouped into the different types, based on your needs. For example, if you are scraping e-commerce websites, you may have two different page types, such as, listings and product
                example: listings
        schemas:
          type: array
          title: Schemas
          description: Specifies the Schema to be used to validate job output records that are being stored in the collection. This uses JSON-Schema format to validates output records.
          items:
            type: object
            example:
            - schema: |-
                {
                  "definitions": {},
                  "$schema": "http://json-schema.org/draft-07/schema#",
                  "$id": "http://example.com/root.json",
                  "type": "object",
                  "title": "The Root Schema",
                  "required": [
                    "feedback",
                    "price",
                    "seller",
                    "title"
                  ],
                  "properties": {
                    "feedback": {
                      "$id": "#/properties/feedback",
                      "type": "string",
                      "title": "The Feedback Schema",
                      "default": "",
                      "examples": [
                        "100% Positive feedback"
                      ],
                      "pattern": "^(.*)$"
                    },
                    "price": {
                      "$id": "#/properties/price",
                      "type": "number",
                      "title": "The Price Schema",
                      "description": "The description of the price schema",
                      "default": 0.0,
                      "examples": [
                        450.99
                      ]
                    },
                    "seller": {
                      "$id": "#/properties/seller",
                      "type": "string",
                      "title": "The Seller Schema",
                      "default": "",
                      "examples": [
                        "tlkennedy02012"
                      ],
                      "pattern": "^(.*)$"
                    },
                    "title": {
                      "$id": "#/properties/title",
                      "type": "string",
                      "title": "The Title Schema",
                      "default": "",
                      "examples": [
                        "Apple Iphone X 256 GB Unlocked (verizon)"
                      ],
                      "pattern": "^(.*)$"
                    }
                  }
                }
              collections: products
            required:
            - collections
            - schema
            properties:
              collections:
                type: string
                title: Collections
                description: Specifies which collections the schema should validate. To validate more than one collections, use comma delimitation.
                example: products,listings
              schema:
                type: string
                title: Schema
                description: The actual JSON-Schema script, that is used to validate the output records.
                example:
                - |-
                  {
                    "definitions": {},
                    "$schema": "http://json-schema.org/draft-07/schema#",
                    "$id": "http://example.com/root.json",
                    "type": "object",
                    "title": "The Root Schema",
                    "required": [
                      "feedback",
                      "price",
                      "seller",
                      "title"
                    ],
                    "properties": {
                      "feedback": {
                        "$id": "#/properties/feedback",
                        "type": "string",
                        "title": "The Feedback Schema",
                        "default": "",
                        "examples": [
                          "100% Positive feedback"
                        ],
                        "pattern": "^(.*)$"
                      },
                      "price": {
                        "$id": "#/properties/price",
                        "type": "number",
                        "title": "The Price Schema",
                        "description": "The description of the price schema",
                        "default": 0.0,
                        "examples": [
                          450.99
                        ]
                      },
                      "seller": {
                        "$id": "#/properties/seller",
                        "type": "string",
                        "title": "The Seller Schema",
                        "default": "",
                        "examples": [
                          "tlkennedy02012"
                        ],
                        "pattern": "^(.*)$"
                      },
                      "title": {
                        "$id": "#/properties/title",
                        "type": "string",
                        "title": "The Title Schema",
                        "default": "",
                        "examples": [
                          "Apple Iphone X 256 GB Unlocked (verizon)"
                        ],
                        "pattern": "^(.*)$"
                      }
                    }
                  }
        seeder:
          type: object
          title: Seeder
          description: Specifies which Seeder file to execute.
          default: {}
          example:
            file: "./seeder/seeder.rb"
          required:
          - file
          properties:
            file:
              type: string
              title: File
              description: The path of the File to be executed.
              default: ''
              example: "./seeder/seeder.rb"
    ScrapeJob:
      type: object
      title: Scrape Job
      description: A Scrape Job is the job in which a scraper project is being run.
      example:
        id: 20103
        scraper_name: ebay
        scraper_id: 20
        created_at: '2020-02-29T07:52:48.397401Z'
        freshness: 
        force_fetch: true
        proxy_type: standard
        status: paused
        seeding_at: '2020-02-29T07:52:56.841901Z'
        seeding_failed_at: 
        seeded_at: '2020-02-29T07:52:58.156202Z'
        seeding_status: done
        seeding_try_count: 1
        seeding_fail_count: 0
        standard_worker_count: 4
        browser_worker_count: 0
        finisher_executing_at: 
        finisher_failed_at: 
        finisher_executed_at: 
        finisher_status: missing
        finisher_try_count: 0
        finisher_fail_count: 0
        failed_stats:
          fetching_failed: 0
          fetching_dequeue_failed: 0
          parsing_failed: 1
          parsing_dequeue_failed: 0
          limbo: 0
        is_current_job: true

      required:
      - status
      properties:
        job_id:
          type: integer
          title: Job ID
          example: 123
        status:
          type: string
          title: Status
          description: The status of the Job
          example: active
          enum:
          - active
          - paused
          - cancelled
          - cancelling
          - finishing
          - done
        seeding_status:
          type: string
          title: Seeding Status
          description: Status of the Seeder file execution.
          example: done
          enum:
          - starting
          - seeding
          - seeding_try
          - failed
          - done
        finisher_status:
          type: string
          title: Finisher Status
          description: Status of the Finisher file execution
          example: done
          enum:
          - missing
          - starting
          - executing
          - executing_try
          - failed
          - done
        fetching_failed:
          type: integer
          title: Fetching Failed Count
          description: How many pages have failed the Fetching process.
          default: 0
        fetching_dequeue_failed:
          type: integer
          title: Fetching Dequeue Failed Count
          description: How many pages have failed the Fetching Dequeue process.
          default: 0
        parsing_failed:
          type: integer
          title: Parsing Failed Count
          description: How many pages have failed the Parsing process
          default: 0
        parsing_dequeue_failed:
          type: integer
          title: Parsing Dequeue Failed Count
          description: How many pages have failed the Parsing Dequeue process
          default: 0
        limbo:
          type: integer
          title: Limbo Count
          description: How many pages are in Limbo, meaning, there is a problem somewhere during scraping process, but the system could not determine where the failure occurs. This is rarely used.
          default: 0
        is_current_job:
          type: boolean
          title: Is Current Job
          description: Whether or not this is the current job of the scraper. A Scraper can only have one active job at a time.
          default: false
        standard_worker_count:
          type: integer
          title: Standard Worker Count
          description: How many Standard workers are allocated to this scraper.
          example: 1
        browser_worker_count:
          type: integer
          title: Browser Worker Count
          description: How many Browser workers are allocated ot this scarper.
          example: 0
    CreateScrapeJob:
      type: object
      title: Create a Scrape Job
      description: Request body to send when creating a Scrape Job.
      example:
        force_fetch: true
        proxy_type: standard
        status: active
        standard_worker_count: 1
        browser_worker_count: 0
        vars:
        - name: foo
          secret: false
          value: bar
      properties:
        status:
          type: string
          title: Status
          description: The status of the Job
          example: active
          enum:
          - active
          - paused
          - cancelled
        standard_worker_count:
          type: integer
          title: Standard Worker Count
          description: How many Standard workers are allocated to this scraper.
          example: 1
        browser_worker_count:
          type: integer
          title: Browser Worker Count
          description: How many Browser workers are allocated ot this scarper.
          example: 0
        vars:
          type: array
          example:
            - name: foo
              secret: false
              value: bar
          items:
            $ref: '#/components/schemas/SetUnsetVar'
          
    InputVar:
      type: object
      title: Input Variable
      description: Input Variable specifies a variable that is accessable by the Scraper or its Job. When specified, the DataHen's Scraper UI will display a form and input variable fields that users can enter, without the need to hard-code any variables.
      example:
        description: 'Search for something. Example: macbook'
        name: search
        required: true
        type: string
        default: macbook
        title: Search
      required:
      - name
      properties:
        default:
          type: string
          title: Default Value
          description: Default value of this input variable, if user does not specify one
          example: macbook
        description:
          type: string
          title: Description
          description: Describes what this Input Variable is.
          example: 'Search for something. Example: macbook'
        name:
          type: string
          title: Name
          description: The Name of this Input Variable. This specifies the name of the HTML input field. This is also the name that is accessable by the script during execution.
          example: search
        required:
          type: boolean
          title: Required
          description: When Required is set to true. The DataHen's scraper UI form will make this field be required.
          default: false
          example: true
        title:
          type: string
          title: Title
          description: The title of the HTML field of this input variable.
          example: Search
        type:
          type: string
          title: Type
          description: What kind of HTML form field to display
          default: string
          enum:
          - string
          - text
          - secret
          - date
          - datetime
    SetUnsetVar:
      type: object
      title: Variable
      example:
        name: search
        secret: false
        value: macbook air
      required:
      - name
      properties:
        name:
          type: string
          title: Name
          pattern: '^[a-zA-Z0-9_]+$'
          description: The Name of this Input Variable. This specifies the name of the HTML input field. This is also the name that is accessable by the script during execution.
          example: search
        secret:
          type: boolean
          title: Secret
          description: When it is set to true, will encrypt this variable's value.
          default: false
          example: true
        value:
          type: string
          title: Value
          description: The value of the Variable
          example: Search
    Export:
      type: object
      title: Export
      example:
        id: 76ef855b13dd41a6a1353eb0c5767fe0
        job_id: 19951
        scraper_id: 1172
        scraper_name: indeed
        exporter_name: jobs_csv
        exporter_type: csv
        config:
          collection: jobs
          exporter_name: jobs_csv
          exporter_type: csv
          fields:
            - header: company
              path: company
            - header: job_description
              path: job_description
            - header: location
              path: location
          limit: null
          offset: null
          start_on_job_done: true
        status: done
        created_at: '2020-02-24T08:20:22.219419Z'
        progress: 100
        exported_records: 213
        to_export: 213
        exporting_at: '2020-02-24T08:20:25.898222Z'
        exported_at: '2020-02-24T08:20:27.326275Z'
        file_name: 76ef855b13dd41a6a1353eb0c5767fe0.tgz
      properties:
        id: 
          type: string
          title: ID
          example: 76ef855b13dd41a6a1353eb0c5767fe0
        job_id: 
          type: integer
          title: Job ID
          example: 19951
        scraper_id: 
          type: integer
          title: Scraper ID
          example: 1172
        scraper_name: 
          type: string
          title: Scraper Name
          example: ebay
        exporter_name: 
          type: string
          title: Exporter Name
          pattern: '^[a-zA-Z0-9_]+$'
          example: jobs_csv
        exporter_type: 
          type: string
          title: Exporter Type
          enum:
          - json
          - csv
          - content
          example: csv
        config:
          oneOf:
          - $ref: '#/components/schemas/JSONExporterConfig'
          - $ref: '#/components/schemas/CSVExporterConfig'
          - $ref: '#/components/schemas/ContentExporterConfig'
        status: 
          type: string
          title: Status
          enum:
          - enqueued
          - active
          - finishing
          - done
          - failed
          - cancelled
          example: done
        created_at: 
          type: string
          format: date-time
          title: Created At
          example: '2020-02-24T08:20:22.219419Z'
        progress: 
          type: integer
          title: Progress
          minimum: 0
          maximum: 100
          example: 100
        exported_records: 
          type: integer
          title: Exported Records
          minimum: 0
          example: 213
        to_export: 
          type: integer
          title: To Export
          minimum: 0
          example: 213
        exporting_at: 
          type: string
          format: date-time
          title: Exporting At
          example: '2020-02-24T08:20:25.898222Z'
        exported_at: 
          type: string
          format: date-time
          title: Exported At
          example: '2020-02-24T08:20:27.326275Z'
        file_name: 
          type: string
          title: File name
          description: Name of the downloaded file.
          example: 76ef855b13dd41a6a1353eb0c5767fe0.tgz
    JSONExporterConfig:
      type: object
      title: JSON Exporter Config
      description: JSON Exporters handle the JSON export of outputs.
      required:
      - exporter_name
      - exporter_type
      - collection
      properties:
        collection:
          type: string
          title: Collection
          description: Collections are a containers for output records that the scraper stores data in. Similar concept to MongoDB collections, or Table in SQL.
          example: products
        exporter_name:
          type: string
          title: Exporter Name
          description: The name of the Exporter must unique, and can only be alphanumeric, or underscore or dash.
          pattern: '^[a-zA-Z0-9_-]+$'
          example: products_last10_json
        exporter_type:
          type: string
          title: Exporter Type
          description: What kind of exporter this is. DataHen supports exports of outputs to JSON and CSV, as well as exports of contents such as HTML, PDF, images, etc.
          enum:
            - json
            - csv
            - content
        limit:
          type: integer
          title: Limit
          description: Specifies the limit to how many exported records. Sometimes you don't want to export everything. 
        offset:
          type: integer
          title: Offset
          description: Specifies the records should be exported from which offset. For example, if there are total of 100 records, and you specify an Offset of 10, the remaining 90 records will be exported.
        order:
          type: string
          title: Order
          description: Specifies the order in which the record will be exported.
          enum:
          - asc
          - desc
          default: 'asc'
          example: desc
        write_mode:
          type: string
          title: Write Mode
          description: Specifies what kind of write the exporter will perform, depending on the exporter type. JSON exporter supports line, pretty, pretty_array, or array.
          default: pretty_array
          enum:
          - line
          - pretty
          - pretty_array
          - array
        start_on_job_done:
          type: boolean
          title: Start On Job Done
          description: Specifies if this exporter should start when a scrape job is done.
          default: false
        disabled:
          type: boolean
          title: Disabled
          default: false
          example: true
    CSVExporterConfig:
      type: object
      title: CSV Exporter Config
      description: CSV Exporters handle the CSV export of scrape jobs.
      required:
      - exporter_name
      - exporter_type
      - collection
      properties:
        collection:
          type: string
          title: Collection
          description: Collections are a containers for output records that the scraper stores data in. Similar concept to MongoDB collections, or Table in SQL.
          example: products
        exporter_name:
          type: string
          title: Exporter Name
          description: The name of the Exporter must unique, and can only be alphanumeric, or underscore or dash.
          pattern: '^[a-zA-Z0-9_-]+$'
          example: products_last10_json
        exporter_type:
          type: string
          title: Exporter Type
          description: What kind of exporter this is. DataHen supports exports of outputs to JSON and CSV, as well as exports of contents such as HTML, PDF, images, etc. 
          enum:
            - json
            - csv
            - content
        limit:
          type: integer
          title: Limit
          description: Specifies the limit to how many exported records. Sometimes you don't want to export everything. 
        offset:
          type: integer
          title: Offset
          description: Specifies the records should be exported from which offset. For example, if there are total of 100 records, and you specify an Offset of 10, the remaining 90 records will be exported.
        order:
          type: string
          title: Order
          description: Specifies the order in which the record will be exported.
          enum:
          - asc
          - desc
          default: 'asc'
          example: desc
        write_mode:
          type: string
          title: Write Mode
          description: Specifies what kind of write the exporter will perform, depending on the exporter type. JSON exporter supports line, pretty, pretty_array, or array.
          default: pretty_array
          enum:
          - line
          - pretty
          - pretty_array
          - array
        start_on_job_done:
          type: boolean
          title: Start On Job Done
          description: Specifies if this exporter should start when a scrape job is done.
          default: false
        fields:
          type: array
          title: Fields
          description: When an exporter type is CSV, we can specify what fields to export to the CSV.
          items:
            type: object
            default: {}
            title: CSV Field
            description: Specifies the field in the CSV
            required: 
            - path
            properties:
              header:
                type: string
                title: Header
                description: Specifies the header/title of the CSV. This is the first row of a CSV.
                example: Price
              path:
                type: string
                title: Path
                description: Specifies the path to the JSON output of the job the data needs to come from. Nested JSON objects must have `.` dot delimiter. 
                example: products.price
        disabled:
          type: boolean
          title: Disabled
          default: false
          example: true
    ContentExporterConfig:
      type: object
      title: Exporter Config
      description: Content Exporters handle the export of contents on jobs.
      required:
      - exporter_name
      - exporter_type
      - page_type
      properties:
        collection:
          type: string
          title: Collection
          description: Collections are a containers for output records that the scraper stores data in. Similar concept to MongoDB collections, or Table in SQL.
          example: products
        exporter_name:
          type: string
          title: Exporter Name
          description: The name of the Exporter must unique, and can only be alphanumeric, or underscore or dash.
          pattern: '^[a-zA-Z0-9_-]+$'
          example: products_last10_json
        exporter_type:
          type: string
          title: Exporter Type
          description: What kind of exporter this is. DataHen supports exports of outputs to JSON and CSV, as well as exports of contents such as HTML, PDF, images, etc. 
          enum:
            - json
            - csv
            - content
        page_type:
          type: string
          title: Page Type
          description: This tells the content exporter to export contents that are in a particular Page Type.
          example: productis
        filename_var:
          type: string
          title: Filename Var
          description: Specify the name of the Page Variable that will be used to name this content as it is saved.
        ignore_extension:
          type: boolean
          title: Ignore Extension
          default: false
          description: If set to true, contents will be saved without file extensions.
        include_failed_contents:
          type: boolean
          title: Include Failed Contents
          description: If set to true, exporter will export all contents, including those that failed fetching.
          default: false
        limit:
          type: integer
          title: Limit
          description: Specifies the limit to how many exported records. Sometimes you don't want to export everything. 
        offset:
          type: integer
          title: Offset
          description: Specifies the records should be exported from which offset. For example, if there are total of 100 records, and you specify an Offset of 10, the remaining 90 records will be exported.
        order:
          type: string
          title: Order
          description: Specifies the order in which the record will be exported.
          enum:
          - asc
          - desc
          default: 'asc'
          example: desc
        start_on_job_done:
          type: boolean
          title: Start On Job Done
          description: Specifies if this exporter should start when a scrape job is done.
          default: false
        disabled:
          type: boolean
          title: Disabled
          default: false
          example: true
    JobStat:
      type: object
      example:
        scraper_name: ebay
        job_id: 20145
        job_status: paused
        seeding_status: done
        finisher_status: missing
        pages: 64
        to_fetch: 0
        fetching: 0
        fetching_failed: 0
        fetching_dequeue_failed: 0
        refetch_failed: 0
        to_parse: 0
        parsing_started: 0
        parsing: 0
        parsed: 60
        parsing_failed: 4
        parsing_dequeue_failed: 0
        limbo: 0
        fetched: 64
        fetched_from_web: 64
        fetched_from_cache: 0
        outputs: 122
        output_collections: 2
        standard_workers: 1
        browser_workers: 0
        time_stamp: '2020-03-02T12:19:12.558102Z'
      properties:
        scraper_name: 
          type: string
          title: Scraper Name
          example: ebay
        job_id: 
          type: integer
          title: Job ID
          example: 20145
        job_status: 
          type: string
          title: Status
          description: The status of the Job
          example: active
          enum:
          - active
          - paused
          - cancelled
          - cancelling
          - finishing
          - done
        seeding_status:
          type: string
          title: Seeding Status
          description: Status of the Seeder file execution.
          example: done
          enum:
          - starting
          - seeding
          - seeding_try
          - failed
          - done
        finisher_status:
          type: string
          title: Finisher Status
          description: Status of the Finisher file execution
          example: done
          enum:
          - missing
          - starting
          - executing
          - executing_try
          - failed
          - done
        pages: 
          type: integer
          title: Pages
          minimum: 0
          example: 64
        to_fetch: 
          type: integer
          title: To Fetch
          minimum: 0
          example: 0
        fetching: 
          type: integer
          title: Fetching
          minimum: 0
          example: 0
        fetching_failed: 
          type: integer
          title: Fetching Failed
          minimum: 0
          example: 0
        fetching_dequeue_failed: 
          type: integer
          title: Fetching Dequeue Failed
          minimum: 0
          example: 0
        to_parse: 
          type: integer
          title: To Parse
          minimum: 0
          example: 0
        parsing_started: 
          type: integer
          title: Parsing Started
          minimum: 0
          example: 0
        parsing: 
          type: integer
          title: Parsing
          minimum: 0
          example: 0
        parsed: 
          type: integer
          title: Parsed
          minimum: 0
          example: 0
        parsing_failed: 
          type: integer
          title: Parsing Failed
          minimum: 0
          example: 0
        parsing_dequeue_failed: 
          type: integer
          title: Parsing Dequeue Failed
          minimum: 0
          example: 0
        limbo: 
          type: integer
          title: Limbo
          minimum: 0
          example: 0
        fetched: 
          type: integer
          title: Fetched
          minimum: 0
          example: 0
        fetched_from_web:
          type: integer
          title: Fetched From Web 
          minimum: 0
          example: 0
        fetched_from_cache: 
          type: integer
          title: Fetched From Cache
          minimum: 0
          example: 0
        outputs:
          type: integer
          title: Outputs
          minimum: 0
          example: 0
        output_collections:
          type: integer
          title: Output Collections
          minimum: 0
          example: 0
        standard_workers: 
          type: integer
          title: Standard Workers
          minimum: 0
          example: 1
        browser_workers: 
          type: integer
          title: Browser Workers
          minimum: 0
          example: 0
        time_stamp: 
          type: string
          format: date-time
          example: '2020-03-02T12:19:12.558102Z'
    JobLog:
      type: object
      title: Job Log
      description: Logs on a Job.
      properties:
        entries:
          type: array
          items:
            properties:
              severity:
                type: string
                title: Severity
                example: Info
              payload:
                type: string
                title: Payload
                description: The actual log message.
              labels:
                type: object
                title: Labels
        more_token:
          type: string
          title: More Token
          description: This is the Page Token of the next page of the log entries.
      example:
        entries:
          - severity: Info
            timestamp: '2020-03-04T13:13:18.391669389Z'
            payload: |
              Standard Fetching GID: www.ebay.com-52e240cc6e97aafe34fe7343877b0e72
            labels: {}
          - severity: Info
            timestamp: '2020-03-04T13:13:19.48801801Z'
            payload: |
              Standard Fetching GID: www.ebay.com-d2e679ec3079d4b84b00a47f5fbe6983
            labels: {}
          - severity: Info
            timestamp: '2020-03-04T13:13:20.596184185Z'
            payload: |
              Successful Fetch GID: www.ebay.com-d3262b01c0f7fb3da3edfcebdb6e1eaa
            labels: {}
          - severity: Info
            timestamp: '2020-03-04T13:13:22.221224738Z'
            payload: |
              Found 1 page(s) to parse
            labels: {}
        more_token: EAE42f757d7TutmHAUrMByIfIg8KDTE2cWx6ZWtlZDlscG0qDAju0v7yBRCNzeG6AUqoBwqCB0r_BgDQLyAG_q5TDorT1Mi_t6g5eEg-
    PageParseDequeue:
      type: object
      title: Page Parse Dequeue
      properties:
        page_types:
          type: array
          items:
            type: string
        limit: 
          type: integer
          description: How many pages to dequeue.
        parse_failed_pages:
          type: boolean
          default: false
          description: If set to true, will parse non 200 http status pages as well.
      example:
        page_types:
          - default
        limit: 2
        parse_failed_pages: true
    ParsedJobPage:
      type: object
      title: Page Parsing Update
      properties:
        gid:
          type: string
          title: GID
          example: www.ebay.com-fda1505c93d29c8ff97d885710d3e075
        job_id: 
          type: integer
          title: Job ID
          example: 20145
        job_status:
          type: string
          title: Job Status
          example: active
          enum:
          - active
          - cancelled
          - done
          - paused
          - cancelling
          - finishing
        outputs:
          type: array
          title: Outputs
          description: Output records to save in the Job.
          items:
            $ref: '#/components/schemas/JobOutput'
        pages:
          type: array
          title: Pages
          description: Pages to enqueue in the Job.
          items:
            $ref: '#/components/schemas/JobPage'
        parsing_status:
          type: string
          enum:
          - starting
          - parsing
          - parsing_try
          - failed
          - done
        parsing_updated_at:
          type: string
          format: date-time
          example: '2020-02-29T07:56:35.307162Z'
        log_error:
          type: string
      example:
        parsing_status: parsing
        outputs:
          - _collection: listings
            hello: world1
            outter:
              inner:
                value1: 10
                value2: 22
              alsoInner:
                value1: 20
          - _collection: details
            _job_id: 8067
            hello: world2
            name: 123
            outter:
              inner:
                value1: 10
                value2: 22
              alsoInner:
                value1: 20
          - hello: world3 default
            outter:
              inner:
                value1: 10
                value2: 22
              alsoInner:
                value1: 20
        pages:
          - url: 'http://fetchtest.datahen.com/statuses/200?q=queuedFromParser-err'
            method: get
          - url: 'http://fetchtest.datahen.com/statuses/200?q=queuedFromParser'
          - url: 'http://fetchtest.datahen.com/statuses/200?q=queuedFromParser2'
          - url: 'http://fetchtest.datahen.com/statuses/200?q=queuedFromParser3'
    JobPageToParse:
      type: object
      properties:
        gid:
          type: string
          title: GID
          example: www.ebay.com-fda1505c93d29c8ff97d885710d3e075
        job_id: 
          type: integer
          title: Job ID
          example: 20145
        status:
          type: string
          enum: 
            - to_fetch
            - fetching
            - fetching_failed
            - fetching_dequeue_failed
            - to_parse
            - parsing_started
            - parsing
            - parsed
            - parsing_failed
            - parsing_dequeue_failed
            - limbo
        fetched_from:
          type: string
          enum:
          - web
          - cache
        page_type:
          type: string
          title: Page Type
        parsing_try_count:
          type: integer
          title: Parsing Try Count
          description: How many Parses have been attempted on this page.
          example: 1
        content_type:
          type: string
          title: Content Type
          description: HTTP Content Type of the HTTP response.
          example: text/html;charset=UTF-8
        content_size:
          type: integer
          title: Content Size
          description: The Size of the content of the HTTP response.
          default: 0
          example: 65650
        vars:
          type: object
          title: Vars
          description: Page Variables field.
          default: {}
          example:
           foo: foo1
           bar: bar1
    JobOutput:
      title: Job Output
      description: The Output record that will be saved on the Scrape Job. Any arbitrary JSON can be set. But it does have default fields, like _collection, _gid, etc.
      type: object
      properties:
        _collection: 
          type: string
          title: Collection
          default: default
          description: Collection Name
        _created_at: 
          type: string
          title: Created At
          format: date-time
          example: '2020-03-05T18:57:55.219222Z'
        _gid: 
          type: string
          title: GID
          description: GID where this output came from.
          example: www.ebay.com-41d7ff78950c00e6b6e9df143547fe76
        _id: 
          type: string
          title: ID
          description: ID can be specified manually, if not, a random hash will be generated automatically.
          example: 764c645759804254bf9f5c1a18091f76
        _job_id: 
          type: integer
          title: Job ID
          example: 20484
      example:
        _collection: default
        _created_at: '2020-03-05T18:57:55.219222Z'
        _gid: www.ebay.com-41d7ff78950c00e6b6e9df143547fe76
        _id: 764c645759804254bf9f5c1a18091f76
        _job_id: 20484
        foo: bar
        some_val:
          inner1:
            value1: 20
          inner2:
            value1: 10
            value2: 22
    JobOutputCollection:
      title: Job Output Collection
      description: The Job Collection that holds Output records
      type: object
      properties:
        job_id:
          type: integer
          title: Job ID
          example: 123
        collection:
          type: string
          title: Collection
          description: Collections are a containers for output records that the scraper stores data in. Similar concept to MongoDB collections, or Table in SQL.
          example: products
        outputs:
          type: integer
          description: Output record count.
    SeededJob:
      type: object
      title: Seeded Job
      properties:
        job_id: 
          type: integer
          title: Job ID
          example: 20145
        job_status:
          type: string
          title: Job Status
          example: active
          enum:
          - active
          - cancelled
          - done
          - paused
          - cancelling
          - finishing
        outputs:
          type: array
          title: Outputs
          description: Output records to save in the Job.
          items:
            $ref: '#/components/schemas/JobOutput'
        pages:
          type: array
          title: Pages
          description: Pages to enqueue in the Job.
          items:
            $ref: '#/components/schemas/JobPage'
        seeding_status:
          type: string
          enum:
          - starting
          - seeding
          - seeding_try
          - failed
          - done
        log_error:
          type: string
      example:
        seeding_status: seeding
        outputs:
          - _collection: listings
            hello: world1
            outter:
              inner:
                value1: 10
                value2: 22
              alsoInner:
                value1: 20
          - _collection: details
            _job_id: 8067
            hello: world2
            name: 123
            outter:
              inner:
                value1: 10
                value2: 22
              alsoInner:
                value1: 20
          - hello: world3 default
            outter:
              inner:
                value1: 10
                value2: 22
              alsoInner:
                value1: 20
        pages:
          - url: 'http://fetchtest.datahen.com/statuses/200?q=queuedFromSeeder-err'
            method: get
          - url: 'http://fetchtest.datahen.com/statuses/200?q=queuedFromSeeder'
          - url: 'http://fetchtest.datahen.com/statuses/200?q=queuedFromSeeder2'
          - url: 'http://fetchtest.datahen.com/statuses/200?q=queuedFromSeeder3'
    FinisherJob:
      type: object
      title: Finisher Job
      properties:
        job_id: 
          type: integer
          title: Job ID
          example: 20145
        job_status:
          type: string
          title: Job Status
          example: active
          enum:
          - active
          - cancelled
          - done
          - paused
          - cancelling
          - finishing
        outputs:
          type: array
          title: Outputs
          description: Output records to save in the Job.
          items:
            $ref: '#/components/schemas/JobOutput'
        pages:
          type: array
          title: Pages
          description: Pages to enqueue in the Job. Deprecated. Finisher does not enqueue new pages at all.
          items:
            $ref: '#/components/schemas/JobPage'
          deprecated: true
        finisher_status:
          type: string
          enum:
          - starting
          - executing
          - failed
          - done
          - missing
          - executing_try
        log_error:
          type: string
      example:
        finisher_status: executing
        outputs:
          - _collection: listings
            hello: world1
            outter:
              inner:
                value1: 10
                value2: 22
              alsoInner:
                value1: 20
          - _collection: details
            _job_id: 8067
            hello: world2
            name: 123
            outter:
              inner:
                value1: 10
                value2: 22
              alsoInner:
                value1: 20
          - hello: world3 default
            outter:
              inner:
                value1: 10
                value2: 22
              alsoInner:
                value1: 20
    ScraperDeployment:
      type: object
      properties:
        id:
          type: integer
          title: ID
        scraper_id: 
          type: integer
          title: Scraper ID
          example: 1172
        commit_hash:
          type: string
          title: Commit Hash
          description: The Commit Hash of the Git Repository.
          example: a0f4999ec6b01e868c0a75516d52d18ee62631b7
        git_repository:
          type: string
          format: uri
          title: Git Repository
          description: URL to the Git repository that the scraper uses.
          example: https://github.com/datahenofficial/ebay-scraper.git
        git_branch:
          type: string
          title: Git Branch
          description: Branch of the Git Repository.
          default: master
          example: master
        errors:
          type: string
          title: Errors
          description: Shows errors on deployment, if any.
          default: null
        success:
          type: boolean
          title: Success
          description: Was this deployment a success or not.
        created_at:
          type: string
          title: Created At
          description: Creation Date of this Deployment
          format: date-time
          example: '2020-02-29T07:56:35.307162Z'
        signed_url:
          type: string
          title: Signed URL
          format: uri
          description: The URL to download the deployed scraper project.
          example: file:///scraper_deployments/a0f4999ec6b01e868c0a75516d52d18ee62631b7.tgz
        config:
          $ref: '#/components/schemas/ScraperConfig'
      example:
        id: 15984
        scraper_id: 20
        commit_hash: a0f4999ec6b01e868c0a75516d52d18ee62631b7
        git_repository: 'https://github.com/answersengine/ebay-scraper.git'
        git_branch: demo_stg
        errors: null
        success: true
        created_at: '2020-02-29T06:50:29.005823Z'
        signed_url: 'file:///scraper_deployments/a0f4999ec6b01e868c0a75516d52d18ee62631b7.tgz'
        config:
          exporters:
            - collection: products
              exporter_name: products_last10_json
              exporter_type: json
              limit: 10
              offset: 0
              order: desc
              write_mode: pretty_array
            - collection: products
              exporter_name: products_json
              exporter_type: json
              limit: null
              offset: null
              start_on_job_done: true
            - collection: products
              exporter_name: products_csv
              exporter_type: csv
              fields:
                - header: gid
                  path: _gid
                - header: created_at
                  path: _created_at
                - header: title
                  path: title
                - header: price
                  path: price
                - header: feedback
                  path: feedback
                - header: seller
                  path: seller
              limit: null
              offset: null
              start_on_job_done: true
          finisher:
            disabled: true
            file: ''
          input_vars:
            - default: macbook
              description: 'Search for something. Example: macbook'
              name: search
              required: true
              title: Search
              type: string
          parsers:
            - file: ./parsers/listings.rb
              page_type: listings
            - file: ./parsers/details.rb
              page_type: details
          schemas:
            - collections: products
              schema: |-
                {
                  "definitions": {},
                  "$schema": "http://json-schema.org/draft-07/schema#",
                  "$id": "http://example.com/root.json",
                  "type": "object",
                  "title": "The Root Schema",
                  "required": [
                    "feedback",
                    "price",
                    "seller",
                    "title"
                  ],
                  "properties": {
                    "feedback": {
                      "$id": "#/properties/feedback",
                      "type": "string",
                      "title": "The Feedback Schema",
                      "default": "",
                      "examples": [
                        "100% Positive feedback"
                      ],
                      "pattern": "^(.*)$"
                    },
                    "price": {
                      "$id": "#/properties/price",
                      "type": "number",
                      "title": "The Price Schema",
                      "description": "The description of the price schema",
                      "default": 0.0,
                      "examples": [
                        450.99
                      ]
                    },
                    "seller": {
                      "$id": "#/properties/seller",
                      "type": "string",
                      "title": "The Seller Schema",
                      "default": "",
                      "examples": [
                        "tlkennedy02012"
                      ],
                      "pattern": "^(.*)$"
                    },
                    "title": {
                      "$id": "#/properties/title",
                      "type": "string",
                      "title": "The Title Schema",
                      "default": "",
                      "examples": [
                        "Apple Iphone X 256 GB Unlocked (verizon)"
                      ],
                      "pattern": "^(.*)$"
                    }
                  }
                }
          seeder:
            file: ./seeder/seeder.rb
    JSONSchema:
      type: object
      title: JSON Schema
      description: Schema based on json-schema.org
      example:
        $id: 'https://example.com/person.schema.json'
        $schema: 'http://json-schema.org/draft-07/schema#'
        title: Person
        type: object
        properties:
          firstName:
            type: string
            description: The person's first name.
          lastName:
            type: string
            description: The person's last name.
          age:
            description: Age in years which must be equal to or greater than zero.
            type: integer
            minimum: 0
    TabularSchema:
      type: array
      title: TabularSchema
      description: Used for Connecting to External Business Intelligence tools such as Google Data Studio
      items:
        type: object
        properties:
          name:
            type: string
          title:
            type: string
          type:
            type: string
          description:
            type: string
      example:
        - name: feedback
          title: The Feedback Schema
          type: TEXT
        - name: price
          title: The Price Schema
          description: The description of the price schema
          type: NUMBER
        - name: seller
          title: The Seller Schema
          type: TEXT
        - name: title
          title: The Title Schema
          type: TEXT
    GDSConnectorExportData:
      type: object
      properties:
        values:
          type: array
          items:
            type: string
    AuthToken:
      type: object
      properties:
        token:
          type: string
          example: abcdefghijklmno1234567890
        role:
          type: string
          title: Role
          enum:
          - basic
          - account_admin
          - owner
          - guest
        account_id:
          type: integer
          title: Account ID
          example: 123
        created_at:
          type: string
          title: Created At
          description: Creation Date of this Auth Token
          format: date-time
          example: '2020-02-29T07:56:35.307162Z'
      example:
        token: abcdefghijklmno1234567890
        role: guest
        account_id: 1
        description: John Doe
        created_at: '2018-09-28T03:27:30.598704Z'
    DeployKey:
      type: object
      title: Deploy Key
      description: Public Key to be used for Deployment
      properties:
        account_id:
          type: integer
          title: Account ID
          description: Account ID that the Deploy Key belongs to.
          example: 123
        public_key:
          type: string
          description: The Public Key string.
        created_at:
          type: string
          title: Created At
          description: Creation Date of this Deploy Key
          format: date-time
          example: '2020-02-29T07:56:35.307162Z'
      example:
        account_id: 113
        public_key: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQD2993xNG5H3QgOG/zyuGFk9brODZVOcbso7TxigFwiDmHqIvQfI6EtjKdJCVw8Kni2JBrz8PI6xC1ZoSurynhq3QlLFQwfOWsBw0jasqov7/ipQ+nErYiocTELH/3bH5c6lSQkczGA5vbh3TPWvMnrXzs5Ky48c7zrJDq7YxusF+qlKkARKJfIkXZron60sIjGFwprmCcGFOo9EdQVHIMcpVkB3iUzHyBh6BTPuFudXP/5foPIMN8JtKfhc1604nLuE3/WHDxoibJ87YdVPFf4lKtR8pLu5t0K5XqSc1JX8Ijrlh2qYFXkF/0ZLBio4DULQteSFT0kZ0kJXqs7h5XczLTERmsASh1YMpo5/j9N5Kr4P4sb6OVrBIDyaznXBWsE0S3lslWdWQDNVA5wwz9otsYvf4+4nku28cdD4hE8x5xtwRZS2DWh3fqGpRD0JV+oG/vMHe0ratc0ZfLrTeW2JUKkR0CyLMNZaaVJnQLRkc2q3bC55ZvJytqBITpMZlIDih+Jf00l6JX3Onq9mg5ZbkUNbz7z4Jix4+qnJ7g3ad3f7q8/AJ+drHweKhRRBSMsngz3M2klcNJBHXA5S4Gs8IMPpxeoZincINwG6MohJ5uxY1BwZVikD8vcrpTHKMVBxBIh+jzMr50z8NHqc7/G8fdSV3cbqIR5hL4oOn3iwQ==
        created_at: '2020-03-07T06:49:29.683549Z'
