diff --git a/.github/workflows/autoupdate.yaml b/.github/workflows/autoupdate.yaml new file mode 100644 index 00000000..640230ba --- /dev/null +++ b/.github/workflows/autoupdate.yaml @@ -0,0 +1,17 @@ +# autoupdate is a GitHub Action that auto-updates pull requests branches whenever changes land on their destination branch. +name: autoupdate +on: + push: + branches: + - main +jobs: + autoupdate: + name: autoupdate + runs-on: ubuntu-22.04 + steps: + - uses: docker://chinthakagodawita/autoupdate-action:v1 + env: + GITHUB_TOKEN: ${{ secrets.ORG_GH_TOKEN }} + # Only monitor PRs that are not currently in the draft state. + PR_READY_STATE: "ready_for_review" + MERGE_CONFLICT_ACTION: "ignore" # Possible option to prevent retrying failed merges diff --git a/.github/workflows/deploy-docs.yml b/.github/workflows/deploy-docs.yml index a3988f5b..7e892c90 100644 --- a/.github/workflows/deploy-docs.yml +++ b/.github/workflows/deploy-docs.yml @@ -1,19 +1,67 @@ name: Deploy Documentation on: - workflow_dispatch: # Manual trigger only + workflow_dispatch: inputs: subfolder: - description: 'Subfolder for documentation (e.g., docs3, docsv3, docs)' + description: 'Subfolder for documentation (leave empty for root deployment)' required: false - default: 'docs3' + default: '' + type: string + triggering_pr_number: + description: 'PR number that triggered this deployment' + required: false + default: '' + type: string + triggering_pr_title: + description: 'PR title that triggered this deployment' + required: false + default: '' + type: string + triggering_commit_sha: + description: 'Commit SHA that triggered this deployment' + required: false + default: '' + type: string + triggering_branch: + description: 'Branch that triggered this deployment' + required: false + default: '' + type: string + original_pr_number: + description: 'Original PR number (if different from triggering PR)' + required: false + default: '' type: string +# Ensure only latest deployment runs +concurrency: + group: docs-deployment + cancel-in-progress: true + jobs: merge-docs: runs-on: ubuntu-latest steps: + - name: Log deployment trigger information + run: | + echo "πŸš€ Starting documentation deployment" + echo "πŸ“‹ Trigger Information:" + echo " - Branch: ${{ inputs.triggering_branch || 'N/A' }}" + echo " - Commit: ${{ inputs.triggering_commit_sha || 'N/A' }}" + if [ -n "${{ inputs.triggering_pr_number }}" ]; then + echo " - PR: #${{ inputs.triggering_pr_number }} - ${{ inputs.triggering_pr_title }}" + if [ -n "${{ inputs.original_pr_number }}" ] && [ "${{ inputs.original_pr_number }}" != "${{ inputs.triggering_pr_number }}" ]; then + echo " - Original PR: #${{ inputs.original_pr_number }} (cherry-picked)" + fi + else + echo " - PR: N/A (direct push)" + fi + echo " - Timestamp: $(date)" + echo " - Workflow Run: ${{ github.run_id }}" + echo " - Subfolder: ${{ inputs.subfolder || '(root deployment)' }}" + - name: Checkout production branch uses: actions/checkout@v4 with: @@ -43,77 +91,69 @@ jobs: - name: Cleanup old version folders and assets run: | - echo "🧹 Cleaning up old version folders and assets..." - - # Get current version folders from config - current_folders=$(python3 -c "import json; config=json.load(open('branches-config.json')); folders=[v.get('folder','') for v in config.get('versions',[]) if v.get('folder','')]; print(' '.join(folders))") - - echo "πŸ“‹ Current version folders: $current_folders" - - # Define asset types that will be regenerated - asset_types=( - "style.css" - "images" - "img" - "logo" - "favicon.ico" - "favicon.png" - "snippets" + echo "🧹 Cleaning up all content except essential files..." + + # Define folders to keep (whitelist) + keep_folders=( + ".github" + ".git" + ".devcontainer" + "scripts" + ".vale" + ".probe" ) - # Remove old assets (they'll be regenerated from current versions) - echo "🎨 Cleaning up old assets..." - for asset in "${asset_types[@]}"; do - if [ -e "$asset" ]; then - echo "πŸ—‘οΈ Removing old asset: $asset" - rm -rf "$asset" - fi - done - - # Remove old docs.json (will be regenerated) - if [ -f "docs.json" ]; then - echo "πŸ—‘οΈ Removing old docs.json" - rm -f "docs.json" - fi + # Define files to keep (whitelist) + keep_files=( + "branches-config.json" + ".gitignore" + "README.md" + ".vale.ini" + ) - # Also clean up the subfolder if it exists (using input parameter) - SUBFOLDER="${{ inputs.subfolder || 'docs3' }}" - if [ -n "$SUBFOLDER" ] && [ -d "$SUBFOLDER" ]; then - echo "πŸ—‘οΈ Removing old $SUBFOLDER subfolder" - rm -rf "$SUBFOLDER" - fi + echo "πŸ“‹ Folders to keep: ${keep_folders[*]}" + echo "πŸ“‹ Files to keep: ${keep_files[*]}" - # Remove old version folders that aren't in the current config - # Look for directories that look like version folders (numeric or version-like names) + # Remove all directories except the ones we want to keep + echo "πŸ—‘οΈ Removing old directories..." for dir in */; do if [ -d "$dir" ]; then dir_name=${dir%/} # Remove trailing slash - - # Skip non-version directories - if [[ "$dir_name" =~ ^\..*$ ]] || [ "$dir_name" = ".github" ]; then - continue - fi - - # Check if this folder is in current config - is_current=false - for current in $current_folders; do - if [ "$dir_name" = "$current" ]; then - is_current=true + + should_keep=false + for keep_folder in "${keep_folders[@]}"; do + if [ "$dir_name" = "$keep_folder" ]; then + should_keep=true break fi done + + if [ "$should_keep" = false ]; then + echo "πŸ—‘οΈ Removing directory: $dir_name/" + rm -rf "$dir" + else + echo "πŸ“ Keeping essential directory: $dir_name/" + fi + fi + done - # If it's not in current config and looks like a version folder, remove it - if [ "$is_current" = false ]; then - # Only remove if it looks like a version folder (contains numbers/dots or common version patterns) - if [[ "$dir_name" =~ ^[0-9]+\.[0-9]+$ ]] || [[ "$dir_name" =~ ^v[0-9] ]] || [[ "$dir_name" =~ ^[0-9] ]]; then - echo "πŸ—‘οΈ Removing old version folder: $dir_name" - rm -rf "$dir" - else - echo "πŸ“ Keeping non-version directory: $dir_name" + # Remove all files except the ones we want to keep + echo "πŸ—‘οΈ Removing old files..." + for file in *; do + if [ -f "$file" ]; then + should_keep=false + for keep_file in "${keep_files[@]}"; do + if [ "$file" = "$keep_file" ]; then + should_keep=true + break fi + done + + if [ "$should_keep" = false ]; then + echo "πŸ—‘οΈ Removing file: $file" + rm -f "$file" else - echo "πŸ“ Keeping current version folder: $dir_name (will be refreshed)" + echo "πŸ“„ Keeping essential file: $file" fi fi done @@ -129,7 +169,7 @@ jobs: echo "πŸ”„ Starting branch cloning and organization..." # Read the branches config and extract branch information - branches=$(python3 -c "import json; config=json.load(open('branches-config.json')); [print(f\"{v.get('folder','')}:{v.get('branch','main')}\") for v in config.get('versions',[]) if v.get('folder','')]") + branches=$(python3 -c "import json; config=json.load(open('branches-config.json')); [print(f\"{v.get('sourceFolder','')}:{v.get('branch','main')}\") for v in config.get('versions',[]) if v.get('sourceFolder','')]") echo "πŸ“‹ Branches to process:" echo "$branches" @@ -160,8 +200,18 @@ jobs: mkdir -p "$folder" echo "πŸ“ Moving contents from $temp_dir to $folder..." - # Copy all files except .git directory - find "$temp_dir" -mindepth 1 -maxdepth 1 ! -name '.git' -exec cp -r {} "$folder/" \; + # Copy documentation content only, excluding development/build files + # Excluded: .git, scripts/, branches-config.json, .github/, README.md, .gitignore, .devcontainer/, .probe + find "$temp_dir" -mindepth 1 -maxdepth 1 \ + ! -name '.git' \ + ! -name 'scripts' \ + ! -name 'branches-config.json' \ + ! -name '.github' \ + ! -name 'README.md' \ + ! -name '.gitignore' \ + ! -name '.devcontainer' \ + ! -name '.probe' \ + -exec cp -r {} "$folder/" \; # Clean up temp directory rm -rf "$temp_dir" @@ -180,19 +230,21 @@ jobs: run: | echo "πŸ”„ Running documentation merger..." - # Get subfolder from input (with fallback) - SUBFOLDER="${{ inputs.subfolder || 'docs3' }}" - echo "πŸ“ Using subfolder: $SUBFOLDER" + # Get subfolder from input (no fallback - empty means root deployment) + SUBFOLDER="${{ inputs.subfolder }}" + echo "πŸ“ Using subfolder: '$SUBFOLDER'" # Run the merge script with branches config if [ -n "$SUBFOLDER" ]; then - python3 merge_docs_configs.py \ + echo "πŸ“ Deploying to subfolder: $SUBFOLDER" + python3 scripts/merge_docs_configs.py \ --branches-config branches-config.json \ --base-dir . \ --subfolder "$SUBFOLDER" \ --output docs.json else - python3 merge_docs_configs.py \ + echo "πŸ“ Deploying to root (no subfolder)" + python3 scripts/merge_docs_configs.py \ --branches-config branches-config.json \ --base-dir . \ --output docs.json @@ -205,7 +257,7 @@ jobs: echo "🧹 Cleaning up temporary cloned version folders..." # Get current version folders from config - current_folders=$(python3 -c "import json; config=json.load(open('branches-config.json')); folders=[v.get('folder','') for v in config.get('versions',[]) if v.get('folder','')]; print(' '.join(folders))") + current_folders=$(python3 -c "import json; config=json.load(open('branches-config.json')); folders=[v.get('sourceFolder','') for v in config.get('versions',[]) if v.get('sourceFolder','')]; print(' '.join(folders))") echo "πŸ“‹ Removing cloned folders: $current_folders" @@ -237,7 +289,38 @@ jobs: echo "πŸ“ Generated file structure:" find . -name "*.mdx" -o -name "*.md" -o -name "*.json" -o -name "*.css" -o -name "*.png" | head -20 + - name: Close previous deployment PRs + run: | + echo "πŸ” Finding and closing previous deployment PRs..." + + # Find PRs that match BOTH criteria: + # 1. Have the "auto-deployment" label + # 2. Branch starts with "docs-merge-" + DEPLOYMENT_PRS=$(gh pr list \ + --state open \ + --label "auto-deployment" \ + --json number,headRefName,labels \ + --jq '.[] | select(.headRefName | startswith("docs-merge-")) | .number') + + if [ -n "$DEPLOYMENT_PRS" ]; then + echo "πŸ“‹ Found deployment PRs to close: $DEPLOYMENT_PRS" + + for pr_number in $DEPLOYMENT_PRS; do + echo "❌ Closing deployment PR #$pr_number" + gh pr close "$pr_number" \ + --comment "πŸ€– Superseded by newer deployment (Run #${{ github.run_number }})" \ + || echo "⚠️ Failed to close PR #$pr_number (may already be closed)" + done + + echo "βœ… Closed all previous deployment PRs" + else + echo "βœ… No deployment PRs found to close" + fi + env: + GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }} + - name: Create Pull Request + id: create-pr uses: peter-evans/create-pull-request@v7 with: token: ${{ secrets.ORG_GH_TOKEN }} @@ -249,9 +332,16 @@ jobs: This PR contains automatically merged documentation from multiple branches. - **Generated from:** `branches-config.json` - **Timestamp:** ${{ github.event.head_commit.timestamp }} - **Run ID:** ${{ github.run_id }} + **Triggered by:** + - **Branch:** ${{ inputs.triggering_branch || 'N/A' }} + - **Commit:** ${{ inputs.triggering_commit_sha || 'N/A' }} + - **PR:** ${{ inputs.triggering_pr_number && format('#{0} - {1}', inputs.triggering_pr_number, inputs.triggering_pr_title) || 'N/A (direct push)' }}${{ inputs.original_pr_number && inputs.original_pr_number != inputs.triggering_pr_number && format(' (cherry-pick of #{0})', inputs.original_pr_number) || '' }} + + **Deployment Details:** + - **Generated from:** `branches-config.json` + - **Run ID:** ${{ github.run_id }} + - **Subfolder:** `${{ inputs.subfolder || '(root deployment)' }}` + - **Timestamp:** $(date) ### Changes Include: - βœ… Merged documentation from multiple branches @@ -259,12 +349,15 @@ jobs: - βœ… Updated assets and content structure - βœ… Cleaned up outdated version folders - ### Subfolder Used: - `${{ inputs.subfolder || 'docs3' }}` - --- - Please review the changes and merge when ready. + 🚦 **This PR will be processed by merge queue to ensure proper validation and ordering.** + + Previous deployment PRs have been automatically closed to prevent conflicts. + labels: | + documentation + auto-deployment + automated commit-message: | πŸ€– Auto-merge documentation from branches @@ -279,10 +372,34 @@ jobs: author: ${{ github.actor }} <${{ github.actor_id }}+${{ github.actor }}@users.noreply.github.com> committer: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> + - name: Enable auto-merge on deployment PR + if: steps.create-pr.outputs.pull-request-number + run: | + PR_NUMBER="${{ steps.create-pr.outputs.pull-request-number }}" + echo "οΏ½ Enabling auto-merge on deployment PR #$PR_NUMBER..." + + gh pr merge --squash --auto "$PR_NUMBER" + + echo "βœ… Auto-merge enabled on PR #$PR_NUMBER" + env: + GH_TOKEN: ${{ secrets.ORG_GH_TOKEN }} + - name: Create deployment summary run: | echo "## πŸ“š Documentation Deployment Summary" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY + echo "### πŸš€ Trigger Information" >> $GITHUB_STEP_SUMMARY + echo "- **Branch:** ${{ inputs.triggering_branch || 'N/A' }}" >> $GITHUB_STEP_SUMMARY + echo "- **Commit:** ${{ inputs.triggering_commit_sha || 'N/A' }}" >> $GITHUB_STEP_SUMMARY + if [ -n "${{ inputs.triggering_pr_number }}" ]; then + echo "- **PR:** #${{ inputs.triggering_pr_number }} - ${{ inputs.triggering_pr_title }}" >> $GITHUB_STEP_SUMMARY + if [ -n "${{ inputs.original_pr_number }}" ] && [ "${{ inputs.original_pr_number }}" != "${{ inputs.triggering_pr_number }}" ]; then + echo "- **Original PR:** #${{ inputs.original_pr_number }} (cherry-picked)" >> $GITHUB_STEP_SUMMARY + fi + else + echo "- **PR:** N/A (direct push)" >> $GITHUB_STEP_SUMMARY + fi + echo "" >> $GITHUB_STEP_SUMMARY echo "### βœ… Successfully merged documentation" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "**Generated files:**" >> $GITHUB_STEP_SUMMARY @@ -294,4 +411,4 @@ jobs: fi echo "" >> $GITHUB_STEP_SUMMARY - echo "**Timestamp:** $(date)" >> $GITHUB_STEP_SUMMARY \ No newline at end of file + echo "**Timestamp:** $(date)" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/eod-report-generator.yaml b/.github/workflows/eod-report-generator.yaml new file mode 100644 index 00000000..beff8169 --- /dev/null +++ b/.github/workflows/eod-report-generator.yaml @@ -0,0 +1,44 @@ +name: EOD Report Generator + +on: + workflow_dispatch: + inputs: + MAIN_REVIEWER: + description: "The GitHub user ID of the primary reviewer whose approval is required for the PR." + required: true + default: "sharadregoti" + START_DATE: + description: "The start date of report (e.g., 2025-02-21T00:00:00.000Z)." + required: true + END_DATE: + description: "The end date of report. Defaults to the current date (e.g., 2025-02-25T00:00:00.000Z)." + required: false + +jobs: + run-script: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' # Change to your required version + + - name: Install dependencies + run: | + cd scripts/eod-report-generator + npm install + + - name: Run script + run: | + cd scripts/eod-report-generator + node index.js + env: + MAIN_REVIEWER: ${{ inputs.MAIN_REVIEWER }} + START_DATE: ${{ inputs.START_DATE }} + END_DATE: ${{ inputs.END_DATE }} + GITHUB_TOKEN: ${{ secrets.TYK_SCRIPTS_TOKEN }} # GitHub Token for API calls + ANTHROPIC_KEY: ${{ secrets.ANTHROPIC_KEY }} # Org key already available diff --git a/.github/workflows/mirror-pr-to-build-deploy.yml b/.github/workflows/mirror-pr-to-build-deploy.yml index 39e8b217..c750234e 100644 --- a/.github/workflows/mirror-pr-to-build-deploy.yml +++ b/.github/workflows/mirror-pr-to-build-deploy.yml @@ -28,52 +28,38 @@ jobs: # Create new mirror PR echo "Creating new mirror PR..." - # Create PR body content - PR_BODY="**πŸ”— Auto-generated mirror PR for Mintlify preview** + # Create PR body content using printf to avoid shell parsing issues + printf '%s\n' \ + "**πŸ”— Auto-generated mirror PR for Mintlify preview**" \ + "" \ + "**Original PR:** #${{ github.event.number }}" \ + "**Author:** @${{ github.event.pull_request.user.login }}" \ + "" \ + "## Purpose" \ + "This PR provides a Mintlify preview link for reviewing documentation changes." \ + "" \ + "## Preview Link" \ + "The Mintlify preview will be available once this PR is processed." \ + "" \ + "## ⚠️ Important Notes" \ + "- **Do not merge this PR directly**" \ + "- This PR will be auto-merged when the original PR #${{ github.event.number }} is merged" \ + "- Make all comments and reviews on the original PR #${{ github.event.number }}" \ + "" \ + "## Changes" \ + "${{ github.event.pull_request.body }}" > pr_body.txt - **Original PR:** #${{ github.event.number }} - **Author:** @${{ github.event.pull_request.user.login }} + # Escape the title properly to handle special characters and spaces + ESCAPED_TITLE=$(printf '%s' "πŸ”„ Preview: ${{ github.event.pull_request.title }}" | sed 's/"/\\"/g') - ## Purpose - This PR provides a Mintlify preview link for reviewing documentation changes. - - ## Preview Link - The Mintlify preview will be available once this PR is processed. - - ## ⚠️ Important Notes - - **Do not merge this PR directly** - - This PR will be auto-merged when the original PR #${{ github.event.number }} is merged - - Make all comments and reviews on the original PR #${{ github.event.number }} - - ## Changes - ${{ github.event.pull_request.body }}" - gh pr create \ --base production \ - --head ${{ github.head_ref }} \ - --title "πŸ”„ Preview: ${{ github.event.pull_request.title }}" \ - --body "$PR_BODY" + --head "${{ github.head_ref }}" \ + --title "$ESCAPED_TITLE" \ + --body-file pr_body.txt \ + --draft echo "βœ… Mirror PR created successfully" - - # Get the mirror PR number that was just created - MIRROR_PR=$(gh pr list --base production --head ${{ github.head_ref }} --json number --jq '.[0].number // empty') - - # Comment on the original PR with link to mirror PR - if [ -n "$MIRROR_PR" ]; then - COMMENT_BODY="πŸ”— **Preview Link Available** - - This PR has an auto-generated mirror for Mintlify preview: - πŸ‘‰ **[View Preview PR #$MIRROR_PR](https://github.com/${{ github.repository }}/pull/$MIRROR_PR)** - - The Mintlify preview link will appear on the mirror PR once it's processed. - - --- - *This is an automated comment. All discussion should happen on this PR, not the mirror PR.*" - - gh pr comment ${{ github.event.number }} --body "$COMMENT_BODY" - echo "βœ… Comment added to original PR with mirror PR link" - fi else echo "πŸ”„ Mirror PR #$MIRROR_PR already exists and will be auto-updated" fi diff --git a/.github/workflows/pr_agent.yaml b/.github/workflows/pr_agent.yaml new file mode 100644 index 00000000..57f0b0a5 --- /dev/null +++ b/.github/workflows/pr_agent.yaml @@ -0,0 +1,20 @@ +on: + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + issue_comment: +jobs: + pr_agent_job: + runs-on: ubuntu-latest + name: Run pr agent on every pull request, respond to user comments + if: ${{ !github.event.pull_request.draft }} + steps: + - name: PR Agent action step + id: pragent + uses: Codium-ai/pr-agent@main + env: + OPENAI_KEY: ${{ secrets.OPENAI_KEY }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/probe-writer.yaml b/.github/workflows/probe-writer.yaml new file mode 100644 index 00000000..b7730c95 --- /dev/null +++ b/.github/workflows/probe-writer.yaml @@ -0,0 +1,24 @@ +name: Probe Writer handler + +on: + issue_comment: + types: [created] + +# Define permissions needed for the workflow +permissions: + issues: write + pull-requests: write + contents: write + +jobs: + trigger_probe_implement: + uses: buger/probe/.github/workflows/probe.yml@main + with: + command_prefix: "/writer" # Or '/ai', '/ask', etc. + allow_edit: true + prompt: ./.probe/writer.md + secrets: + ANTHROPIC_API_KEY: ${{ secrets.PROBE_ANTHROPIC_API_KEY }} + ANTHROPIC_API_URL: ${{ secrets.PROBE_ANTHROPIC_URL }} + APP_ID: ${{ secrets.PROBE_APP_ID }} + APP_PRIVATE_KEY: ${{ secrets.PROBE_APP_PRIVATE_KEY }} \ No newline at end of file diff --git a/.github/workflows/probe.yaml b/.github/workflows/probe.yaml new file mode 100644 index 00000000..965a2923 --- /dev/null +++ b/.github/workflows/probe.yaml @@ -0,0 +1,37 @@ +name: Probe handler + +on: + pull_request: + types: [opened] #[opened , labeled] + issue_comment: + types: [created] + issues: + types: [opened] #[opened, labeled] + +# Define permissions needed for the workflow +permissions: + issues: write + pull-requests: write + contents: read + +jobs: + trigger_probe_chat: + # Uncomment if you want to run on on specific lables, in this example `probe` + # if: | + # (github.event_name == 'pull_request' && github.event.action == 'opened') || + # (github.event_name == 'issues' && github.event.action == 'opened') || + # (github.event_name == 'issue_comment' && github.event.action == 'created') || + # ((github.event_name == 'pull_request' || github.event_name == 'issues') && + # github.event.action == 'labeled' && github.event.label.name == 'probe') + # Use the reusable workflow from your repository (replace and ) + uses: buger/probe/.github/workflows/probe.yml@main + # Pass required inputs + with: + command_prefix: "/probe" # Or '/ai', '/ask', etc. + # Optionally override the default npx command if the secret isn't set + # default_probe_chat_command: 'node path/to/custom/script.js' + # Pass ALL secrets from this repository to the reusable workflow + # This includes GITHUB_TOKEN, PROBE_CHAT_COMMAND (if set), ANTHROPIC_API_KEY, etc. + secrets: + ANTHROPIC_API_KEY: ${{ secrets.PROBE_ANTHROPIC_API_KEY }} + ANTHROPIC_API_URL: ${{ secrets.PROBE_ANTHROPIC_URL }} diff --git a/.github/workflows/release-bot.yaml b/.github/workflows/release-bot.yaml new file mode 100644 index 00000000..748500b8 --- /dev/null +++ b/.github/workflows/release-bot.yaml @@ -0,0 +1,189 @@ +name: Cherry-pick to Release Branch + +on: + issue_comment: + types: [created] + workflow_call: + +jobs: + cherry_pick: + runs-on: ubuntu-latest + steps: + - name: Check for release command + id: check_command + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { issue, comment } = context.payload; + if (!issue || !issue.pull_request || !comment || !comment.body.startsWith('/release to ')) { + core.setOutput('release_valid', 'false'); + return; + } + const releaseBranch = comment.body.split('/release to ')[1].trim(); + core.setOutput('release_valid', 'true'); + core.setOutput('release_branch', releaseBranch); + core.setOutput('pr_number', issue.number); + + - name: Check admin permissions + if: steps.check_command.outputs.release_valid == 'true' + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const username = context.payload.comment.user.login; + const authorAssociation = context.payload.comment.author_association; + + // Quick check: Repository owner always allowed + if (authorAssociation === 'OWNER') { + console.log(`βœ… User ${username} is repository owner`); + return; + } + + // Check for admin permission + try { + const { data: permission } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner: context.repo.owner, + repo: context.repo.repo, + username: username + }); + + if (permission.permission !== 'admin') { + core.setFailed(`❌ Only repository admins can use /release command. User ${username} has: ${permission.permission}`); + return; + } + + console.log(`βœ… User ${username} has admin permissions`); + } catch (error) { + core.setFailed(`❌ Permission check failed: ${error.message}`); + } + + - name: Install GitHub CLI (for act/local testing) + if: steps.check_command.outputs.release_valid == 'true' + run: | + sudo apt update + sudo apt install -y curl unzip gnupg + curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /usr/share/keyrings/githubcli-archive-keyring.gpg >/dev/null + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null + sudo apt update + sudo apt install -y gh + + - name: Checkout repository + if: steps.check_command.outputs.release_valid == 'true' + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set default branch variable + if: steps.check_command.outputs.release_valid == 'true' + run: echo "DEFAULT_BRANCH=${{ github.event.repository.default_branch }}" >> $GITHUB_ENV + + - name: Skip jobs if not a valid release command + if: steps.check_command.outputs.release_valid == 'false' + run: echo "Skipping cherry-pick as the release command is not valid." + continue-on-error: true + + - name: Setup Git + if: steps.check_command.outputs.release_valid == 'true' + run: | + git config --global user.email "bot@tyk.io" + git config --global user.name "Tyk Bot" + + - name: Get PR base and merge SHAs + id: pr_details + if: steps.check_command.outputs.release_valid == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + PR_NUMBER=${{ steps.check_command.outputs.pr_number }} + MERGE_COMMIT=$(gh pr view "$PR_NUMBER" --json mergeCommit --jq '.mergeCommit.oid // empty') + BASE_SHA=$(gh pr view "$PR_NUMBER" --json baseRefOid --jq '.baseRefOid // empty') + echo "MERGE_COMMIT=$MERGE_COMMIT" >> $GITHUB_ENV + echo "BASE_SHA=$BASE_SHA" >> $GITHUB_ENV + echo "MERGE_COMMIT=$MERGE_COMMIT" >> $GITHUB_OUTPUT + echo "BASE_SHA=$BASE_SHA" >> $GITHUB_OUTPUT + + - name: Cherry-pick PR into release branch + id: cherry_pick + if: steps.check_command.outputs.release_valid == 'true' + env: + GITHUB_TOKEN: ${{ secrets.ORG_GH_TOKEN }} + GITHUB_REPO: ${{ github.repository }} + GITHUB_BRANCH: ${{ steps.check_command.outputs.release_branch }} + run: | + export FOLDER=$(basename "$GITHUB_REPO") + rm -rf $FOLDER + git clone https://x-access-token:$GITHUB_TOKEN@github.com/$GITHUB_REPO + cd $FOLDER + + git checkout $GITHUB_BRANCH + git pull + + NEW_BRANCH=merge/$GITHUB_BRANCH/$MERGE_COMMIT + git branch -D $NEW_BRANCH 2>/dev/null || true + REMOTE_EXISTS=$(git ls-remote --heads origin $NEW_BRANCH | wc -l) + [ "$REMOTE_EXISTS" -gt 0 ] && git push origin --delete $NEW_BRANCH || true + + git checkout -b $NEW_BRANCH + + MERGE_FAILED=0 + git cherry-pick -x $BASE_SHA..$MERGE_COMMIT || MERGE_FAILED=$? + + git diff --quiet origin/$GITHUB_BRANCH HEAD && { + echo "No changes to cherry-pick" + echo "PR_URL=" >> $GITHUB_OUTPUT + echo "MERGE_FAILED=0" >> $GITHUB_OUTPUT + exit 0 + } + + git push origin $NEW_BRANCH --force + + TITLE=$(git log --format=%s -n 1 $MERGE_COMMIT) + BODY=$(git log --format=%B -n 1 $MERGE_COMMIT) + + PR_URL=$(gh pr create \ + --title "Merging to $GITHUB_BRANCH: $TITLE" \ + --body "$BODY" \ + --repo $GITHUB_REPO \ + --base $GITHUB_BRANCH \ + --head $NEW_BRANCH \ + $( [ "$MERGE_FAILED" -ne 0 ] && echo "--draft" )) + + echo "PR_URL=$PR_URL" >> $GITHUB_OUTPUT + echo "MERGE_FAILED=$MERGE_FAILED" >> $GITHUB_OUTPUT + + if [ "$MERGE_FAILED" -eq 0 ]; then + if [[ "$PR_URL" =~ /pull/([0-9]+) ]]; then + PR_NUMBER="${BASH_REMATCH[1]}" + gh pr merge --squash "$PR_NUMBER" --auto --subject "Merging to $GITHUB_BRANCH: $TITLE" --body "$BODY" || echo "Auto-merge failed" + fi + fi + + - name: Comment back on original PR + if: steps.check_command.outputs.release_valid == 'true' && always() + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const prUrl = '${{ steps.cherry_pick.outputs.PR_URL }}'; + const mergeFailed = '${{ steps.cherry_pick.outputs.MERGE_FAILED }}' === '1'; + let body; + + if ('${{ job.status }}' === 'success') { + if (mergeFailed) { + body = `⚠️ Cherry-pick completed with conflicts. A draft PR was created: ${prUrl}`; + } else if (prUrl) { + body = `βœ… Cherry-pick successful. A PR was created and auto-merged (if allowed): ${prUrl}`; + } else { + body = `ℹ️ Cherry-pick skipped: no changes needed in target branch.`; + } + } else { + body = '❌ Cherry-pick failed. Please check the workflow logs.'; + } + + github.rest.issues.createComment({ + issue_number: ${{ steps.check_command.outputs.pr_number }}, + owner: context.repo.owner, + repo: context.repo.repo, + body: body + }); diff --git a/.github/workflows/release-to-branches-with-label.yml b/.github/workflows/release-to-branches-with-label.yml new file mode 100644 index 00000000..0c9e6e5e --- /dev/null +++ b/.github/workflows/release-to-branches-with-label.yml @@ -0,0 +1,104 @@ +name: On Pull Request Merged to Master + +on: + pull_request: + # Only trigger on pull requests targeting main/master + branches: + - master + - main + types: + - closed + +jobs: + run-on-pr-merged: + runs-on: ubuntu-latest + + # Only run if the PR was actually merged + if: ${{ github.event.pull_request.merged == true }} + + steps: + - name: Add a comment to the merged PR (only if labeler is in the org) + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.ORG_GH_TOKEN }} + script: | + // 1. The label format: e.g., "release-1", "release-1.0" + const labelRegex = /^release-[0-9]+(\.[0-9]+)?$/; + + // 2. Get PR info + const pullRequestNumber = context.payload.pull_request.number; + const labelsOnPR = context.payload.pull_request.labels || []; + console.log("Labels on the Pull Request:", labelsOnPR.map(label => label.name)); + // 3. Get all timeline events to see who labeled the PR + const { data: prEvents } = await github.rest.issues.listEvents({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber + }); + + // 4. Filter down to "labeled" events + const labeledEvents = prEvents.filter(ev => ev.event === 'labeled'); + console.log("Labeled Events:",labeledEvents.map(event => ({ + label: event.label?.name, + user: event.actor?.login, + timestamp: event.created_at + }))); + // 5. Build a map: labelName -> last user who added it + // (We reverse to get the *most recent* labeler, if a label was added multiple times) + const labelToLastLabeler = {}; + for (const event of labeledEvents.reverse()) { + const labelName = event.label?.name; + const userName = event.actor?.login; + if (labelName && userName && !labelToLastLabeler[labelName]) { + labelToLastLabeler[labelName] = userName; + } + } + + // 6. For each label on the PR, check if it matches "release-.." + // If yes, we see who labeled it last and check their membership + for (const label of labelsOnPR) { + if (labelRegex.test(label.name)) { + const userWhoAddedLabel = labelToLastLabeler[label.name]; + + // If there's no recorded user (edge case), skip + if (!userWhoAddedLabel) { + console.log(`User not found for label: ${label.name}`); + continue; + } + + // 7. Check if the user is in the org + let isMember = false; + try { + await github.rest.orgs.checkMembershipForUser({ + org: 'TykTechnologies', + username: userWhoAddedLabel + }); + // If this call succeeds, they're a member + isMember = true; + console.log(`User '${userWhoAddedLabel}' is a member of the organization '${'TykTechnologies'}'.`); + } catch (error) { + // If 404, user is not a member. Anything else is an unexpected error. + if (error.status === 404) { + console.log(`User '${userWhoAddedLabel}' is NOT a member of the organization '${'TykTechnologies'}'.`); + }else { + console.error(`An error occurred while checking membership for user '${userWhoAddedLabel}':`, error); + throw error; + } + } + + // 8. Comment only if user is in the org + if (isMember) { + console.log(`Creating comment for label '${label.name}' on PR #${pullRequestNumber} by user '${userWhoAddedLabel}'.`); + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: pullRequestNumber, + body: `/release to ${label.name}` + }); + }else{ + console.log(`No comment created for label '${label.name}' on PR #${pullRequestNumber} because the user '${userWhoAddedLabel}' is not a member of the organization 'TykTechnologies'.`); + } + }else{ + console.log(`Label '${label.name}' does not match the expected format.`); + } + } \ No newline at end of file diff --git a/.github/workflows/site-content-analysis.yml b/.github/workflows/site-content-analysis.yml new file mode 100644 index 00000000..498b548f --- /dev/null +++ b/.github/workflows/site-content-analysis.yml @@ -0,0 +1,162 @@ +name: Site Content Analysis + +on: + workflow_dispatch: + inputs: + wait_time: + description: 'Wait time per page (seconds)' + required: false + default: '3' + type: string + timeout: + description: 'Timeout per page (seconds)' + required: false + default: '30' + type: string + +jobs: + analyze-site-content: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: production + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install system dependencies + run: | + sudo apt-get update + # Install essential packages for headless Chrome + sudo apt-get install -y \ + ca-certificates \ + fonts-liberation \ + libnss3 \ + lsb-release \ + xdg-utils \ + wget \ + gnupg + + # Install Chrome dependencies (with fallbacks for different Ubuntu versions) + sudo apt-get install -y \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libcups2 \ + libdrm2 \ + libgtk-3-0 \ + libgtk-4-1 \ + libxcomposite1 \ + libxdamage1 \ + libxrandr2 \ + libgbm1 \ + libxss1 \ + libasound2 || \ + sudo apt-get install -y \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libcups2 \ + libdrm2 \ + libgtk-3-0 \ + libxcomposite1 \ + libxdamage1 \ + libxrandr2 \ + libgbm1 \ + libxss1 \ + libasound2t64 + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install pyppeteer beautifulsoup4 + + - name: Download Chromium for Pyppeteer + run: | + python -c "import asyncio; from pyppeteer import launch; asyncio.get_event_loop().run_until_complete(launch())" + + - name: Run site content analysis + run: | + python scripts/browser_site_analyzer.py \ + --base-url https://tyk.mintlify.app \ + --docs-json docs.json \ + --output-dir site_analysis_output \ + --report-file site_analysis_report.json \ + --wait-time ${{ github.event.inputs.wait_time || '3' }} \ + --timeout ${{ github.event.inputs.timeout || '30' }} + + - name: Create summary comment + if: always() + run: | + echo "## πŸ” Site Content Analysis Results" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Analysis completed for:** https://tyk.mintlify.app" >> $GITHUB_STEP_SUMMARY + echo "**Wait time:** ${{ github.event.inputs.wait_time || '3' }} seconds" >> $GITHUB_STEP_SUMMARY + echo "**Timeout:** ${{ github.event.inputs.timeout || '30' }} seconds" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + if [ -f site_analysis_report.json ]; then + echo "### Summary Statistics" >> $GITHUB_STEP_SUMMARY + echo '```json' >> $GITHUB_STEP_SUMMARY + python -c " + import json + with open('site_analysis_report.json', 'r') as f: + data = json.load(f) + summary = data['summary'] + print(f'Total pages analyzed: {summary[\"total_pages_analyzed\"]}') + print(f'Pages with sufficient content: {summary[\"pages_with_sufficient_content\"]}') + print(f'Pages with empty/insufficient content: {summary[\"pages_with_empty_content\"]}') + print(f'Browser failures: {summary[\"browser_failures\"]}') + " >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Show problematic pages if any + python -c " + import json + with open('site_analysis_report.json', 'r') as f: + data = json.load(f) + if data['empty_pages']: + print('### ❌ Pages with Content Issues') + for page in data['empty_pages'][:10]: # Show first 10 + print(f'- **{page[\"url\"]}**: {page[\"issues\"][0] if page[\"issues\"] else \"Unknown issue\"}') + if len(data['empty_pages']) > 10: + print(f'- ... and {len(data[\"empty_pages\"]) - 10} more pages') + " >> $GITHUB_STEP_SUMMARY + else + echo "❌ Analysis failed - no report generated" >> $GITHUB_STEP_SUMMARY + fi + + echo "" >> $GITHUB_STEP_SUMMARY + echo "πŸ“„ **All analysis results are displayed above**" >> $GITHUB_STEP_SUMMARY + + - name: Fail job if critical issues found + if: always() + run: | + if [ -f site_analysis_report.json ]; then + python -c " + import json, sys + with open('site_analysis_report.json', 'r') as f: + data = json.load(f) + summary = data['summary'] + total = summary['total_pages_analyzed'] + empty = summary['pages_with_empty_content'] + + if total > 0: + empty_percentage = (empty / total) * 100 + print(f'Empty content percentage: {empty_percentage:.1f}%') + + # Fail if more than 20% of pages have empty content + if empty_percentage > 20: + print(f'❌ CRITICAL: {empty_percentage:.1f}% of pages have empty content (threshold: 20%)') + sys.exit(1) + else: + print(f'βœ… Content quality acceptable: {empty_percentage:.1f}% empty pages') + else: + print('❌ CRITICAL: No pages were analyzed') + sys.exit(1) + " + fi diff --git a/.github/workflows/trigger-docs-deploy.yml b/.github/workflows/trigger-docs-deploy.yml new file mode 100644 index 00000000..49a5c542 --- /dev/null +++ b/.github/workflows/trigger-docs-deploy.yml @@ -0,0 +1,140 @@ +name: Trigger Documentation Deployment + +on: + push: + branches: + - main + - 'release-*' + +jobs: + trigger-deploy: + runs-on: ubuntu-latest + steps: + - name: Find the actual merged PR + id: get-pr + run: | + COMMIT_SHA="${{ github.sha }}" + COMMIT_MESSAGE="${{ github.event.head_commit.message }}" + echo "Looking for PR that was merged with commit: $COMMIT_SHA" + echo "Commit message: $COMMIT_MESSAGE" + + # Search for PRs that were merged with this exact commit SHA + echo "Searching for merged PR with commit SHA: $COMMIT_SHA" + + # Use a temporary file to avoid shell parsing issues + curl -s \ + -H "Authorization: token ${{ secrets.ORG_GH_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/${{ github.repository }}/pulls?state=closed&sort=updated&direction=desc&per_page=50" > /tmp/prs.json + + # Find PR with matching merge commit SHA + ACTUAL_PR_NUMBER=$(jq -r --arg commit "$COMMIT_SHA" '.[] | select(.merge_commit_sha == $commit) | .number' /tmp/prs.json | head -1) + + if [ -n "$ACTUAL_PR_NUMBER" ] && [ "$ACTUAL_PR_NUMBER" != "null" ]; then + # Get PR details + ACTUAL_PR_TITLE=$(jq -r --arg commit "$COMMIT_SHA" '.[] | select(.merge_commit_sha == $commit) | .title' /tmp/prs.json | head -1) + PR_AUTHOR=$(jq -r --arg commit "$COMMIT_SHA" '.[] | select(.merge_commit_sha == $commit) | .user.login' /tmp/prs.json | head -1) + + echo "Found actual merged PR: #$ACTUAL_PR_NUMBER - $ACTUAL_PR_TITLE by $PR_AUTHOR" + echo "pr_number=$ACTUAL_PR_NUMBER" >> $GITHUB_OUTPUT + echo "pr_title=$ACTUAL_PR_TITLE" >> $GITHUB_OUTPUT + echo "pr_author=$PR_AUTHOR" >> $GITHUB_OUTPUT + echo "has_pr=true" >> $GITHUB_OUTPUT + + # Also try to extract original PR from commit message for context + ORIGINAL_PR=$(echo "$COMMIT_MESSAGE" | grep -oE '#[0-9]+' | head -1 | sed 's/#//') + if [ -n "$ORIGINAL_PR" ] && [ "$ORIGINAL_PR" != "$ACTUAL_PR_NUMBER" ]; then + echo "Found original PR reference in commit: #$ORIGINAL_PR" + echo "original_pr_number=$ORIGINAL_PR" >> $GITHUB_OUTPUT + echo "has_original_pr=true" >> $GITHUB_OUTPUT + else + echo "has_original_pr=false" >> $GITHUB_OUTPUT + fi + else + echo "No merged PR found via API, falling back to commit message parsing..." + + # Fallback to original method + PR_NUMBER=$(echo "$COMMIT_MESSAGE" | grep -oE '#[0-9]+' | head -1 | sed 's/#//') + + if [ -n "$PR_NUMBER" ]; then + echo "Found PR number in commit message: #$PR_NUMBER" + echo "pr_number=$PR_NUMBER" >> $GITHUB_OUTPUT + echo "has_pr=true" >> $GITHUB_OUTPUT + + # Extract PR title (everything before the PR number reference) + PR_TITLE=$(echo "$COMMIT_MESSAGE" | sed 's/ (#[0-9]\+).*//' | head -1) + echo "pr_title=$PR_TITLE" >> $GITHUB_OUTPUT + echo "has_original_pr=false" >> $GITHUB_OUTPUT + else + echo "No PR number found in commit message (direct push)" + echo "has_pr=false" >> $GITHUB_OUTPUT + echo "has_original_pr=false" >> $GITHUB_OUTPUT + fi + fi + + # Clean up temp file + rm -f /tmp/prs.json + + - name: Trigger production deployment + run: | + # Prepare the dispatch payload + if [ "${{ steps.get-pr.outputs.has_pr }}" = "true" ]; then + if [ "${{ steps.get-pr.outputs.has_original_pr }}" = "true" ]; then + # Include both actual PR and original PR + PAYLOAD=$(jq -n \ + --arg ref "production" \ + --arg pr_number "${{ steps.get-pr.outputs.pr_number }}" \ + --arg pr_title "${{ steps.get-pr.outputs.pr_title }}" \ + --arg original_pr_number "${{ steps.get-pr.outputs.original_pr_number }}" \ + --arg commit_sha "${{ github.sha }}" \ + --arg branch "${{ github.ref_name }}" \ + '{ + ref: $ref, + inputs: { + triggering_pr_number: $pr_number, + triggering_pr_title: $pr_title, + original_pr_number: $original_pr_number, + triggering_commit_sha: $commit_sha, + triggering_branch: $branch + } + }') + echo "Triggering deployment with PR #${{ steps.get-pr.outputs.pr_number }} (original: #${{ steps.get-pr.outputs.original_pr_number }})" + else + # Only actual PR + PAYLOAD=$(jq -n \ + --arg ref "production" \ + --arg pr_number "${{ steps.get-pr.outputs.pr_number }}" \ + --arg pr_title "${{ steps.get-pr.outputs.pr_title }}" \ + --arg commit_sha "${{ github.sha }}" \ + --arg branch "${{ github.ref_name }}" \ + '{ + ref: $ref, + inputs: { + triggering_pr_number: $pr_number, + triggering_pr_title: $pr_title, + triggering_commit_sha: $commit_sha, + triggering_branch: $branch + } + }') + echo "Triggering deployment with PR #${{ steps.get-pr.outputs.pr_number }}" + fi + else + PAYLOAD=$(jq -n \ + --arg ref "production" \ + --arg commit_sha "${{ github.sha }}" \ + --arg branch "${{ github.ref_name }}" \ + '{ + ref: $ref, + inputs: { + triggering_commit_sha: $commit_sha, + triggering_branch: $branch + } + }') + echo "Triggering deployment (direct push, no PR)" + fi + + curl -X POST \ + -H "Authorization: token ${{ secrets.ORG_GH_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/${{ github.repository }}/actions/workflows/deploy-docs.yml/dispatches \ + -d "$PAYLOAD" diff --git a/.github/workflows/validate-docs.yml b/.github/workflows/validate-docs.yml new file mode 100644 index 00000000..917dbd41 --- /dev/null +++ b/.github/workflows/validate-docs.yml @@ -0,0 +1,33 @@ +name: Validate Documentation + +on: + pull_request: + +jobs: + validate: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install requests + + - name: Validate documentation + run: | + echo "πŸ” Running documentation validation..." + python scripts/validate_mintlify_docs.py . --validate-redirects --verbose + + - name: Validation complete + if: success() + run: | + echo "βœ… Documentation validation passed!" + echo "All links, images, navigation, and redirects are valid." diff --git a/.probe/writer.md b/.probe/writer.md new file mode 100644 index 00000000..31a0b6eb --- /dev/null +++ b/.probe/writer.md @@ -0,0 +1,16 @@ +You are helpful senior technical writer. +Your role is to automatically assist with GitHub issues. + +Before jumping on the task or replying, you first should analyze the user request or issue details thoroughly. + +When responding: +1. Be concise but thorough in your responses. +2. If the issue description is unclear, ask clarifying questions. +3. Request any additional information you might need to better assist +4. Provide helpful information related to the query +5. Try to provide an elegant and concise solution. +6. If there are multiple different solutions or next steps, convey it in the response +7. If solution is clear, you can jump to implementation right away, if not, you can ask user a clarification question, by calling attempt_completion tool, with required details. + +When writing content: +Don’t use title case. Following sentence case conventions, like in Google and GitHub style guides. diff --git a/.vale.ini b/.vale.ini new file mode 100644 index 00000000..315ef5ab --- /dev/null +++ b/.vale.ini @@ -0,0 +1,11 @@ +StylesPath = .vale/styles, styles +MinAlertLevel = suggestion + +Packages = MDX, Google, write-good + + +[*] +BasedOnStyles = Vale, Google, write-good, mintlify-poc + +# Define your vocabulary +Vocab = mintlify-poc \ No newline at end of file diff --git a/.vale/styles/Vocab/mintlify-poc/accept.txt b/.vale/styles/Vocab/mintlify-poc/accept.txt new file mode 100644 index 00000000..39201a52 --- /dev/null +++ b/.vale/styles/Vocab/mintlify-poc/accept.txt @@ -0,0 +1,43 @@ +Tyk +Tyk's +Coprocess +OTel +API's +CVEs +JWTs +proxying +keyspace +Distroless +brotli +mutex +Subgraph +subgraph +Subgraphs +subgraphs +supergraph +supergraphs +build_id +build_ids +enums +allOf +anyOf +oneOf +allowedIPs +dataplanes +commonName +datasource +upstreams +Typename +Fieldname +mdcb +failover +orgs +misconfigured +Hashicorp +Sarama +detailed_tracing +reconnections +rawResponse +ruc +sqlite +wget \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 00000000..aa117cc7 --- /dev/null +++ b/README.md @@ -0,0 +1,72 @@ +# Documentation Validation + +This repository includes automated documentation validation to ensure quality and consistency. + +## CI/CD Validation + +### GitHub Actions +- **Trigger**: Every pull request to any branch +- **Validation**: Comprehensive documentation checks +- **Status**: Required check (blocks merge if validation fails) + +### What Gets Validated +- βœ… **Broken internal links** - All internal links point to existing files +- βœ… **Missing images** - All image references point to existing files +- βœ… **Self-referencing redirects** - No redirects where source equals destination +- βœ… **Navigation-redirect conflicts** - No conflicts between navigation and redirects +- βœ… **Invalid redirects** - No redirects with empty source or destination +- βœ… **Missing navigation files** - All navigation entries point to existing files +- βœ… **Missing redirect destinations** - All redirect destinations point to existing files + +## Local Validation + +You can run validation locally before creating a PR: + +```bash +# Full validation (recommended) +python scripts/validate_mintlify_docs.py . --validate-redirects --verbose + +# Check only links +python scripts/validate_mintlify_docs.py . --links-only + +# Check only images +python scripts/validate_mintlify_docs.py . --images-only + +# Check only redirects and navigation +python scripts/validate_mintlify_docs.py . --validate-redirects +``` + +## Setting Up Branch Protection + +To make validation required: + +1. Go to **Settings** β†’ **Branches** in your GitHub repository +2. Add a branch protection rule for your target branches (e.g., `main`, `deploy`) +3. Enable **"Require status checks to pass before merging"** +4. Select **"Validate Documentation / validate"** as a required check +5. Enable **"Require branches to be up to date before merging"** + +## Validation Script + +The `scripts/validate_mintlify_docs.py` script provides comprehensive documentation validation: + +- **No external dependencies** - Uses only Python standard library +- **Fast execution** - Typically completes in under 1 minute +- **Clear reporting** - Detailed error messages with file locations +- **Exit codes** - Returns non-zero exit code if issues found (perfect for CI) + +## Troubleshooting + +### Common Issues + +1. **Broken internal links**: Update the link path or add a redirect +2. **Missing images**: Add the image file or update the image path +3. **Navigation issues**: Ensure all navigation entries point to existing files +4. **Redirect problems**: Check redirect sources and destinations are valid + +### Getting Help + +If validation fails: +1. Check the GitHub Actions log for detailed error messages +2. Run validation locally to debug issues +3. Fix the reported issues and push updates to your PR diff --git a/advanced-configuration/other-protocols.mdx b/advanced-configuration/other-protocols.mdx new file mode 100644 index 00000000..6f3c3c98 --- /dev/null +++ b/advanced-configuration/other-protocols.mdx @@ -0,0 +1,30 @@ +--- +title: "Advanced Communication Protocols" +'og:description': "How to configure advanced communication protocols" +tags: ['gRPC', 'SSE', 'Websocket', 'Other Protocol'] +sidebarTitle: "Overview" +--- + +## Overview + +Tyk API Gateway is primarily designed to handle HTTP/HTTPS traffic, but it also provides robust support for several other protocols to accommodate modern API architectures and communication patterns. This flexibility allows developers to leverage Tyk's security, monitoring, and management capabilities across a wider range of API technologies. + +### Use Cases + +These advanced protocol capabilities enhance Tyk's usefulness beyond traditional REST APIs: + +- **Real-time Applications**: WebSockets enable bidirectional communication for chat applications, collaborative tools, and live dashboards. +- **Microservices Communication**: gRPC support facilitates efficient inter-service communication with strong typing and performance benefits. +- **Event-Driven Architectures**: SSE enables efficient server-push notifications without the overhead of maintaining WebSocket connections. + +## Supported Protocols + +Tyk currently supports the following protocols other than HTTP/HTTPS: + +1. **[TCP Proxy](/key-concepts/tcp-proxy)** + +2. **[gRPC](/key-concepts/grpc-proxy)** + +3. **[Server-Sent Events (SSE)](/advanced-configuration/sse-proxy)** + +4. **[WebSockets](/advanced-configuration/websockets)** \ No newline at end of file diff --git a/advanced-configuration/sse-proxy.mdx b/advanced-configuration/sse-proxy.mdx new file mode 100644 index 00000000..ace92671 --- /dev/null +++ b/advanced-configuration/sse-proxy.mdx @@ -0,0 +1,50 @@ +--- +title: "Server Side Events Proxy" +'og:description': "Describe how you can use Tyk as a simple Server Side Events Proxy" +tags: ['SSE Proxy', 'Other Protocol'] +sidebarTitle: "Server Side Events" +--- + +[Server-Sent Events](https://en.wikipedia.org/wiki/Server-sent_events) (SSE) is a server push technology enabling a subscribed client to receive automatic updates from a server via a long running HTTP connection. +Unlike WebSockets, SSE is a one-way communication of server to clients (WebSockets is a bidirectional communication between server and client). +As such, if you only need clients to receive data from a server, and don't require them sending messagess back, SSE could be a simpler way to make that happen. An online stock quotes, or notifications and feeds are good examples for applications that use SSE. + +## Using Tyk as a server-sent events (SSE) Proxy + +Tyk Gateway supports SSE proxying over HTTP, and can sit in the middle between the client and the SSE server and support the server sending updates to the client. + +### Setup +- Enable SSE support on the Gateway: Set `http_server_options.enable_websockets` to `true` in your Tyk Gateway config file. +- To maintain an open connection between the API consumer and the Tyk Gateway, set `http_server_options.read_timeout` and `http_server_options.write_timeout` to appropriately high values (in milliseconds). For example, you could try setting both to `2000`, but this is for you to determine in your environment. +- Set `http_server_options.flush_interval` to an appropriate value, e.g. `1`, to force Tyk to stream the response to the client every `n` seconds. + + +### Example using Tyk as an SSE proxy +For this we will need: + +* An SSE server. For this example we will use [Golang HTML 5 SSE example](https://github.com/kljensen/golang-html5-sse-example) +* An instance of the Tyk Gateway and optionally the Tyk Dashboard + +**Steps for Configuration:** +* Ensure the Gateway configurations detailed in the Setup section are set. +* Run the SSE server as per the example instructions. By default this runs on port `8000`. +``` +go run ./server.go +``` +* Publish an API with the following configuration: + 1. Set an appropriate listen path, e.g. `"listen_path": "/sse"` + 2. Strip the listen path, e.g. `"strip_listen_path": true,` + 3. Set the target url as the SSE server, e.g. the example SSE server:`"target_url": "http://host.docker.internal:8000"` + 4. Click Save, and wait for the Gateway to reload the API before testing it +* To test the protected SSE service via the API in the Tyk Gateway run: +```bash +curl http://localhost:8080/sse/events/ +``` +You should see a stream of updates from the server. In this example, you will see: + +```bash +Message: 20 - the time is 2013-03-08 21:08:01.260967 -0500 EST +Message: 21 - the time is 2013-03-08 21:08:06.262034 -0500 EST +Message: 22 - the time is 2013-03-08 21:08:11.262608 -0500 EST +``` + diff --git a/advanced-configuration/transform-traffic/looping.mdx b/advanced-configuration/transform-traffic/looping.mdx new file mode 100644 index 00000000..536b0951 --- /dev/null +++ b/advanced-configuration/transform-traffic/looping.mdx @@ -0,0 +1,102 @@ +--- +title: "Looping" +order: 5 +sidebarTitle: "What is Internal Looping?" +--- + +## Overview + +If you need to redirect your URL to *another endpoint* in the api or *another api in the gateway* using [URL Rewriting](/transform-traffic/url-rewriting#url-rewrite-middleware), you can run the request pipeline one more time, internally instead of redirect it to a HTTP endpoint through the network. This is called looping. This is very performant because Tyk will not do another network call when a loop is detected. + +In order to specify a loop, in the target URL you specify a string in the protocol schema `tyk://` as shown below: + +This syntax of `tyk` in the schema protocol and `self` in the domian will loop the request to another endpoint under the current api: +``` +tyk://self/ +``` + +You can also loop to another API as by specifying the API name or id (instead of `self`): +``` +tyk:/// +``` + +Combined with our advanced URL rewriter rules, it can be turned into a powerful logical block, replacing the need for writing middleware or virtual endpoints in many cases. + + +## Example Use Cases + +### Multiple Auth Types for a single API + +Imagine you have a legacy API that has existing authentication strategies. We can pretend it's using basic authentication. You've decided to bring this API into your APIM ecosystem, and also begin to use OAuth2 for your API. But also we need to support existing users who have basic auth credentials. Finally, it's important that we expose a single ingress to our customers for that one API, instead of multiple listen paths for each authentication type. + +We can use looping to achieve this. We can set up triggers in URL Rewrite plugin, where based off a specific header, Tyk will loop the request to a specific API. + +For example, let's see the following use case: +Looping example + +#### 1. A request comes into the ingress API. This has two rules: +- If `Header == "Authorization: Bearer"`, loop to the OAuth API +- If `Header == "Authorization: Basic"`, loop to the Basic Auth API + +1. The ingress API is marked "keyless" as Tyk doesn't perform any authentication here. +2. We add rate limiting option to the loop via `?check_limits=true` + +#### 2. The inner APIs perform authentication, and loop to the north-bound API + +These APIs are marked internal, can only be accessed from within loop context. + +#### 3. The north-bound API, marked open keyless, does transformations, etc, then reverse proxies to the backend API. + +1. This API is marked internal, can only be accessed from within loop context. +2. This API is marked "keyless" as Tyk doesn't perform any authentication here. + +## Advanced Configuration + +You can add one or more of the following configurations as query parameters to your looping URL. + +### Rate Limiting in looping + +In looping context, rate limiting (quotas as well) is not checked except when explicitly turned on. You need to add the following query param: +``` +?check_limits=true +``` + +For example: + +``` +tyk://123/myendpoint?check_limits=true +``` + +### HTTP Method transformation in looping + +You can tell Tyk to modify the HTTP verb during looping by adding the following query param: +``` +?method=GET +``` + +For example: + +``` +tyk://123/myendpoint?method=GET +``` + +### Loop Limiting + +In order to avoid endless recursion, there's a default limit loop level of 5 which is set in the request level (i.e. set per request). +In case the loop level has gone beiod the allowed limit the user will get the error `"Loop level too deep. Found more than %d loops in single request"`. +You can set the loop level limit with a query param as shown below. Please note that you can only set it once per request. After that, you can't overwrite with a new loop level limit. + + +Tell Tyk to limit the number of loops by adding the following query param: +``` +?loop_limit={int} +``` + +For example: + +``` +tyk://123/myendpoint?loop_limit={int} +``` + + + diff --git a/advanced-configuration/transform-traffic/soap-rest.mdx b/advanced-configuration/transform-traffic/soap-rest.mdx new file mode 100644 index 00000000..ce79b0dd --- /dev/null +++ b/advanced-configuration/transform-traffic/soap-rest.mdx @@ -0,0 +1,194 @@ +--- +title: "Transformation Use Case: SOAP To REST" +'og:description': "How to transform SOAP API to REST API in Tyk" +tags: ['Traffic Transformation', 'SOAP', 'REST', 'SOAP to REST'] +sidebarTitle: "SOAP To REST" +--- + +You can transform an existing SOAP service to a JSON REST service. This can be done from the Tyk Dashboard with no coding involved and should take around 10 minutes to perform the transform. + +We also have a video which walks you through the SOAP to REST transform. + + + +## Prerequisites + +An existing SOAP service and the WSDL definition. For this example, we will use: + +- Upstream Target - [https://www.dataaccess.com/webservicesserver/numberconversion.wso](https://www.dataaccess.com/webservicesserver/numberconversion.wso) +- The WSDL definition from - [https://www.dataaccess.com/webservicesserver/numberconversion.wso?WSDL](https://www.dataaccess.com/webservicesserver/numberconversion.wso?WSDL) +- Postman Client (or other endpoint testing tool) + +## Steps for Configuration + +1. **Import the WSDL API** + + 1. Select APIs from the System Management menu + + APIs Menu + + 2. Click Import API + + Import API + + 3. Select **From WSDL** from the Import an API Definition window + 4. In the **Upstream Target** field, enter `https://www.dataaccess.com/webservicesserver/numberconversion.wso` as listed in the Prerequisites. + 5. Paste the WSDL definition from the link in Prerequisites + 6. Click **Generate API**. You should now have an API named `NumberConversion` in your API list + + NumberService API + +2. **Add the transforms to an Endpoint** + + 1. From the API list, select Edit from the Actions menu for the `NumberConversion` API + 2. Select the **Endpoint Designer** tab. You should see 2 POST endpoints that were imported. We will apply the transforms to the `NumberToWords` endpoint. + + Endpoints + + 3. Expand the `NumberToWords` endpoint. The following plugins should have been added as part of the import process. + + - URL rewrite + - Track endpoint + + + + + To make the URL a little friendlier, we're going to amend the Relative Path to just `/NumberToWords`. Update your API after doing this. + + + + 4. Add the following plugins from the **Plugins** drop-down list: + + - Body transform + - Modify headers + +3. **Modify the Body Transform Plugin** + + **Set up the Request** + + We use the `{{.FieldName}}` Golang template syntax to access the JSON request. For this template we will use `{{.numberToConvert}}`. + + 1. Expand the Body transform plugin. From the Request tab, copy the following into the Template section: + + ```xml expandable + + + + + {{.numberToConvert}} + + + + ``` + + 2. In the Input field, enter the following: + + ```json + { + "numberToConvert": 35 + } + ``` + + + + The '35' integer can be any number you want to convert + + + + 3. Click **Test**. You should get the following in the Output field: + + ```xml + + + + + 35 + + + + ``` + + **Set up the Response** + + Again, for the response, we will be using the `{{.FieldName}}` syntax as the following `{{.Envelope.Body.NumberToDollarsResponse.NumberToDollarsResult}}` + + 1. For the Input Type, select XML + + Response Input Type + + 2. In the Template section enter: + + ```yaml + { + "convertedNumber": "{{.Envelope.Body.NumberToDollarsResponse.NumberToDollarsResult}}" + } + ``` + 3. Enter the following into the input field: + + ```xml + + + + thirty five dollars + + + + ``` + 4. Click Test. You should get the following in the Output field: + + ```json + { + "convertedNumber": "thirty five dollars" + } + ``` + +5. **Change the Content-Type Header** + + We now need to change the `content-type` header to allow the SOAP service to receive the payload in XML. We do this by using the **Modify header** plugin + + 1. Expand the Modify Header plugin + 2. From the **Request** tab enter the following in the **Add this header** section + + - Header Name: `content-type` + - Header Value: `text/xml` + + 3. Click Add + + Modify Header Request + + 4. From the **Response** tab enter the following in the **Add this header** section + + - Header Name: `content-type` + - Header Value: `application/json` + + Modify Header Response + + 5. Click **Add** + 6. Click **Update** + + Update API + +## Testing the Endpoint + +You now need to test the endpoint. We are going to use Postman. + + + +We have not set up any Authentication for this API, it has defaulted to `Open (Keyless)`. + + + + +1. Copy the URL for your NumberConversion API with the NumberToWords endpoint - `https://tyk-url/numberconversion/NumberToWords/` +2. Paste it as a POST URL in the Postman URL Request field +3. Enter the following as a raw Body request + +```json +{ + "numberToConvert": 35 +} +``` +Your Postman request should look similar to below (apart from the URL used) + +Postman + diff --git a/advanced-configuration/websockets.mdx b/advanced-configuration/websockets.mdx new file mode 100644 index 00000000..6bdc7dd8 --- /dev/null +++ b/advanced-configuration/websockets.mdx @@ -0,0 +1,58 @@ +--- +title: "Websockets" +'og:description': "How to use websockets in Tyk" +tags: ['websockets', 'Other Protocol'] +sidebarTitle: "Websockets" +--- + +As from Tyk gateway v2.2, Tyk supports transparent WebSocket connection upgrades. To enable this feature, set the `enable_websockets` value to `true` in your `tyk.conf` file. + +WebSocket proxying is transparent, Tyk will not modify the frames that are sent between client and host, and rate limits are on a per-connection, not per-frame basis. + +The WebSocket upgrade is the last middleware to fire in a Tyk request cycle, and so can make use of HA capabilities such as circuit breakers and enforced timeouts. + +Tyk needs to decrypt the inbound and re-encrypt the outbound for the copy operations to work, Tyk does not just pass through the WebSocket. When the target is on default SSL port you must explicitly specify the target url for the API: + +```{.copyWrapper} +https://target:443/ +``` + +## WebSocket Example + +We are going to set up Tyk with a WebSocket proxy using our [Tyk Pro Docker Demo](https://github.com/TykTechnologies/tyk-pro-docker-demo) installation. + +We will be using the [Postman WebSocket Echo Service](https://blog.postman.com/introducing-postman-websocket-echo-service/) to test the connection. + +**Steps for Configuration** + +1. **Setup the API in Tyk** + + Create a new API in Tyk. For this demo we are going to select Open (Keyless) as the **Authentication mode**. + + Set the **Target URL** to `wss://ws.postman-echo.com/raw` + +2. **Test the Connection** + + 1. From Postman, select **File > New > WebSocket Request** (or from **Workspace > New > WebSocket Request** if using the web based version). + + Postman WebSocket Request + + 2. Enter your Tyk API URL in the **Enter server URL** field (minus the protocol). + 3. Enter some text in the **New Message** field and click **Send**. + 4. You will see a successful connection. + + Postman WebSocket Connection Result + + + + + If your API uses an Authentication mode other than Open (Keyless), add the details in the Header tab. + + + +An example Header configuration for using an Authentication Token with an API: + +Postman WebSocket Connection Result with Authorization token + +See the [Access an API](/api-management/gateway-config-managing-classic#access-an-api) tutorial for details on adding an Authentication Token to your APIs. + diff --git a/ai-management/ai-studio/ai-portal.mdx b/ai-management/ai-studio/ai-portal.mdx new file mode 100644 index 00000000..5ae7194c --- /dev/null +++ b/ai-management/ai-studio/ai-portal.mdx @@ -0,0 +1,125 @@ +--- +title: "AI Portal" +'og:description': "How AI Portal works?" +tags: ['AI Studio', 'AI Management', 'AI Portal'] +sidebarTitle: "AI Portal" +--- + +The Tyk AI Studio's AI Portal provides a user-friendly web interface where end users can interact with configured AI capabilities. It serves as the primary access point for users to engage with Chat Experiences, view documentation, and manage their account settings. + +## Purpose + +The main goals of the AI Portal are: + +* **Unified User Experience:** Offer a cohesive interface for accessing all AI capabilities configured within Tyk AI Studio. +* **Self-Service Access:** Enable users to independently access and utilize AI features without administrator intervention. +* **Contextual Documentation:** Provide integrated documentation and guidance for available AI services. +* **Account Management:** Allow users to manage their own profile settings and view usage information. +* **Secure Access Control:** Enforce permissions based on teams and organizational policies. + +## Key Features + +* **Chat Interface:** Access to all [Chat Experiences](/ai-management/ai-studio/chat-interface) the user has permission to use, with a clean, intuitive UI for conversational interactions. +* **Resource Catalogues:** Browse and subscribe to available LLMs, Data Sources, and [Tools](/ai-management/ai-studio/tools) through dedicated catalogue interfaces. +* **Application Management:** Create and manage [Apps](/ai-management/ai-studio/apps) that integrate LLMs, tools, and data sources for API access. +* **Documentation Hub:** Integrated documentation for available AI services, tools, and data sources. +* **User Profile Management:** Self-service capabilities for updating profile information and preferences. +* **History & Favorites:** Access to past chat sessions and ability to bookmark favorite conversations. +* **Responsive Design:** Optimized for both desktop and mobile devices for consistent access across platforms. +* **Customizable Themes:** Support for light/dark mode and potentially organization-specific branding. +* **Notifications:** System alerts and updates relevant to the user's AI interactions. + +## Using the AI Portal + +Users access the AI Portal through a web browser at the configured URL for their Tyk AI Studio installation. + +1. **Authentication:** Users log in using their credentials (username/password, SSO, or other configured authentication methods). +2. **Home Dashboard:** Upon login, users see a dashboard with available Chat Experiences and recent activity. +3. **Resource Discovery:** Users can browse catalogues of available LLMs, Data Sources, and Tools to which they have access. +4. **Application Creation:** Users can create Apps by selecting and subscribing to the LLMs, tools, and data sources they need. +5. **Chat Selection:** Users can select from available Chat Experiences to start or continue conversations. +6. **Documentation Access:** Users can browse integrated documentation to learn about available capabilities. +7. **Profile Management:** Users can update their profile settings, preferences, and view usage statistics. + + AI Portal Dashboard + +## Configuration (Admin) + +Administrators configure the AI Portal through the Tyk AI Studio admin interface: + +* **Portal Branding:** Customize logos, colors, and themes to match organizational branding. +* **Available Features:** Enable or disable specific portal features (chat, documentation, etc.). +* **Authentication Methods:** Configure login options (local accounts, SSO integration, etc.). +* **Default Settings:** Set system-wide defaults for user experiences. +* **Access Control:** Manage which teams can access the portal and specific features within it. +* **Custom Content:** Add organization-specific documentation, welcome messages, or announcements. + + Portal Configuration + +## API Access + +While the AI Portal primarily provides a web-based user interface, it is built on top of the same APIs that power the rest of Tyk AI Studio. Developers can access these APIs directly for custom integrations: + +* **Authentication API:** `/api/v1/auth/...` endpoints for managing user sessions. +* **Chat API:** `/api/v1/chat/...` endpoints for programmatic access to chat functionality. +* **User Profile API:** `/api/v1/users/...` endpoints for managing user information. +* **Datasource API:** `/datasource/{dsSlug}` endpoint for directly querying configured data sources. +* **Tools API:** `/api/v1/tools/...` endpoints for discovering and invoking available tools. +* **Applications API:** `/api/v1/apps/...` endpoints for managing user applications and their resource subscriptions. + +### Datasource API + +The Datasource API allows direct semantic search against configured vector stores: + +* **Endpoint:** `/datasource/{dsSlug}` (where `{dsSlug}` is the datasource identifier) +* **Method:** POST +* **Authentication:** Bearer token required +* **Request Format:** + ```json + { + "query": "your search query here", + "n": 5 // optional, number of results to return (default: 3) + } + ``` +* **Response Format:** + ```json + { + "documents": [ + { + "content": "text content of the document chunk", + "metadata": { + "source": "filename.pdf", + "page": 42 + } + }, + // additional results... + ] + } + ``` + +**Important Note:** The endpoint does not accept a trailing slash. Use `/datasource/{dsSlug}` and not `/datasource/{dsSlug}/`. + +### Tools API + +The Tools API provides programmatic access to available tools and their capabilities: + +* **Tool Discovery:** `/api/v1/tools/` - List available tools and their specifications +* **Tool Invocation:** `/api/v1/tools/{toolId}/invoke` - Execute specific tool operations +* **Tool Documentation:** `/api/v1/tools/{toolId}/docs` - Retrieve tool usage documentation + +### MCP (Model Context Protocol) Access + +Tools can also be accessed through the **Model Context Protocol (MCP)**, providing standardized tool integration: + +* **MCP Server Endpoint:** `/mcp` - Connect MCP-compatible clients to access tools +* **Protocol Compliance:** Supports the full MCP specification for tool discovery and execution +* **Client Libraries:** Compatible with popular MCP client implementations across different programming languages + +This multi-protocol approach enables developers to: +* Use familiar MCP tooling and libraries +* Integrate with existing MCP-enabled workflows +* Maintain consistency across different AI platforms and tools + +This API-first approach ensures that all functionality available through the AI Portal can also be accessed programmatically for custom applications or integrations. + +The AI Portal serves as the primary touchpoint for end users interacting with AI capabilities managed by Tyk AI Studio, providing a secure, intuitive, and feature-rich experience. diff --git a/ai-management/ai-studio/ai-studio-swagger.mdx b/ai-management/ai-studio/ai-studio-swagger.mdx new file mode 100644 index 00000000..d32855e6 --- /dev/null +++ b/ai-management/ai-studio/ai-studio-swagger.mdx @@ -0,0 +1,7 @@ +--- +title: "Tyk AI Studio API" +'og:description': "Tyk AI Studio API" +tags: ['OpenAPI Spec for AI Studio', 'Tyk AI Studio OAS', 'Tyk AI Portal REST'] +sidebarTitle: "API Documentation" +--- + diff --git a/ai-management/ai-studio/analytics.mdx b/ai-management/ai-studio/analytics.mdx new file mode 100644 index 00000000..6c2b16ad --- /dev/null +++ b/ai-management/ai-studio/analytics.mdx @@ -0,0 +1,76 @@ +--- +title: "Analytics & Monitoring" +'og:description': "How to configure analytics in Tyk AI Studio?" +tags: ['AI Studio', 'AI Management', 'Analytics', 'Monitoring'] +sidebarTitle: "Analytics & Monitoring" +--- + +Tyk AI Studio incorporates an Analytics System designed to collect, aggregate, and provide insights into the usage, cost, and performance of the platform's core components, particularly LLMs, Tools, and Chat interactions. + +## Purpose + +The Analytics System serves several key purposes: + +* **Cost Tracking:** Monitor spending associated with different LLM providers and models. +* **Usage Monitoring:** Understand how users and applications are interacting with LLMs and Tools. +* **Performance Analysis:** Track metrics like latency and token counts for LLM requests. +* **Auditing & Debugging:** Provide detailed logs of interactions for troubleshooting and security analysis. +* **Reporting:** Offer data points for dashboards and reports for administrators and potentially end-users. + +## Data Collection + +The primary point of data collection is the **[Proxy & API Gateway](/ai-management/ai-studio/proxy)**. As requests flow through the proxy: + +1. **Request Details:** Information about the incoming request is captured (e.g., user ID, application ID, requested LLM/route, timestamp). +2. **LLM Interaction:** Details about the interaction with the backend LLM are recorded (e.g., model used, prompt tokens, completion tokens, latency). +3. **Cost Calculation:** Using data from the [Model Pricing System](/ai-management/ai-studio/llm-management), the cost of the interaction is calculated based on token counts. +4. **Tool Usage:** If the interaction involved [Tools](/ai-management/ai-studio/tools), relevant details might be logged (e.g., which tool was called, success/failure). +5. **Chat Context:** For interactions originating from the [Chat Interface](/ai-management/ai-studio/chat-interface), metadata about the chat session might be included. + +## Architecture + +* **Asynchronous Ingestion:** To minimize impact on request latency, analytics data is typically collected by the Proxy and sent asynchronously to a dedicated analytics database or processing pipeline. +* **Data Storage:** A suitable database (e.g., time-series database like InfluxDB, relational database like PostgreSQL, or a data warehouse) stores the aggregated analytics records. +* **API Endpoints:** Tyk AI Studio exposes internal API endpoints that allow the Admin UI (and potentially other authorized services) to query the aggregated analytics data. + +## Key Metrics Tracked (Examples) + +* **Per LLM Request:** + * Timestamp + * User ID / API Key ID + * LLM Configuration ID / Route ID + * Model Name + * Prompt Tokens + * Completion Tokens + * Total Tokens + * Calculated Cost + * Latency (ms) + * Success/Error Status +* **Per Tool Call (if applicable):** + * Timestamp + * Tool ID + * Success/Error Status + * Latency (ms) +* **Aggregated Metrics:** + * Total cost per user/application/LLM over time. + * Total requests per user/application/LLM over time. + * Average latency per LLM. + * Most frequently used models/tools. + +## Monitoring & Dashboards (Admin) + +Administrators typically access analytics data via dashboards within the Tyk AI Studio UI. + +* **Overview:** High-level summaries of cost, usage, and requests. +* **Filtering & Grouping:** Ability to filter data by time range, user, application, LLM configuration, etc. +* **Visualizations:** Charts and graphs showing trends in cost, token usage, request volume, and latency. +* **Detailed Logs:** Access to raw or near-raw event logs for specific interactions (useful for debugging). + + Analytics Dashboard + +## Integration with Other Systems + +* **[Budget Control](/ai-management/ai-studio/llm-management):** Analytics data (specifically cost) is likely used by the Budget Control system to track spending against defined limits. +* **[Model Pricing](/ai-management/ai-studio/llm-management):** The pricing definitions are crucial for calculating the cost metric within the analytics system. + +By providing detailed analytics, Tyk AI Studio enables organizations to effectively manage costs, understand usage patterns, and ensure the optimal performance of their AI interactions. diff --git a/ai-management/ai-studio/apps.mdx b/ai-management/ai-studio/apps.mdx new file mode 100644 index 00000000..3f8c2585 --- /dev/null +++ b/ai-management/ai-studio/apps.mdx @@ -0,0 +1,298 @@ +--- +title: "Apps View for Tyk AI Studio" +'og:description': "How to configure apps in AI Studio?" +tags: ['AI Studio', 'AI Management', 'Apps'] +--- + +The **Apps View** is used to manage user-created applications that interact with Large Language Models (LLMs) and data sources via the Tyk AI Gateway. These apps provide a mechanism for users to define and encapsulate the functionality and resources they need for API interactions. Below is a detailed overview of the Apps View and its functionality. + +--- + +#### **Apps List Overview** + +1. **Name**: + - The name of the application created by the user (e.g., `My New App`, `Jira Task Refiner`). + +2. **Description**: + - A brief explanation of the app's purpose or functionality (e.g., "Experiment to refine tasks for Jira"). + +3. **User**: + - The name of the user who created or owns the application. + +4. **Actions**: + - A menu (three-dot icon) for performing app-related actions, such as: + - Viewing app details. + - Editing the app configuration. + - Deleting the app. + +--- + +#### **Features** + +1. **Add App Button**: + - A green button labeled **+ ADD APP**, located in the top-right corner. Clicking this button opens a form for creating a new app. + +2. **Pagination Control**: + - Located at the bottom-left, this dropdown allows administrators to adjust the number of apps displayed per page. + +--- + +#### **Purpose of Apps** + +1. **Encapsulation of Resources**: + - Apps bundle together the LLMs, tools, and data sources a user needs to access through the AI Gateway. + - Users can subscribe to tools from the [AI Portal](/ai-management/ai-studio/ai-portal) tool catalogue, similar to how they access LLMs and data sources. + +2. **RESTful Access via Credentials**: + - Each app is linked to a set of credentials that users can use to authenticate API requests. + - Credentials must be activated by an administrator after an app is submitted. + +3. **Integration with Upstream LLMs and Tools**: + - Apps allow users to access LLMs through the following methods: + - **Vendor-Native SDKs**: Direct interaction with the upstream provider's SDK. + - **OpenAI-Compatible API**: Requests translated into a standardized API format. + - Tools can be accessed through multiple protocols: + - **REST APIs**: Direct HTTP API access to tool endpoints. + - **MCP (Model Context Protocol)**: Standardized tool interaction protocol for enhanced compatibility. + +4. **Centralized Administration**: + - Administrators can manage all user-created apps, ensuring proper governance and access control. + +--- + +#### **Example Workflow** + +1. **App Creation**: + - A user creates an app and specifies the LLMs, tools, and data sources required for their use case. + - Tools can be discovered and subscribed to through the AI Portal's tool catalogue interface. + +2. **Credential Activation**: + - Once the app is submitted, an admin reviews and activates the app's credentials, enabling the app to interact with the AI Gateway. + +3. **API Integration**: + - The user integrates their app with their systems using the provided credentials and API endpoints. + - Multiple access methods are available: REST APIs, OpenAI-compatible APIs, and MCP protocol. + +4. **Real-Time Usage**: + - The app facilitates communication with the specified resources via the AI Gateway using the chosen protocol. + +--- + +#### **Benefits** + +1. **Streamlined Access**: + - Users can consolidate all the resources they need into a single app, simplifying integration workflows. + +2. **Governance and Security**: + - Admin-controlled credential activation ensures that only approved apps can access the AI Gateway. + +3. **Flexibility**: + - Support for both vendor-native SDKs and OpenAI-compatible APIs allows users to integrate apps into diverse environments. + +4. **Centralized Management**: + - Admins can oversee all user-created apps, ensuring compliance with organizational policies. + +--- + +The **Apps View** is a key feature of the Tyk AI Studio, enabling users to define, configure, and securely interact with LLMs and data sources through the AI Gateway while ensuring robust governance and control. + +### App Details and Proxy Logs + +The **App Details View** provides a comprehensive overview of a specific app, including its performance metrics, usage data, and traffic logs. This view is designed to help administrators monitor the app's activity and ensure compliance with governance policies. + +--- + +#### **Sections and Features** + +### **App Token Usage and Cost** +- **Graph**: + - Displays token usage and cost over a specified date range. + - Tracks the app's performance by visualizing the volume of tokens consumed and associated costs. + +- **Date Range Selector**: + - Allows filtering of token usage and cost data for specific start and end dates. + +--- + +### **App Information** +1. **Name**: + - The name of the app (e.g., `Jira Task Refiner`). + +2. **Description**: + - A summary of the app's purpose (e.g., "Experiment to refine tasks for Jira"). + +3. **User**: + - The creator or owner of the app (e.g., `Jeffy Mathew`). + +4. **LLMs**: + - The LLMs used by the app (if specified). + +5. **Datasources**: + - Lists the data sources configured for use with this app (e.g., `Tyk Documentation`). + +6. **Tools**: + - Shows the tools subscribed to and available for this app (e.g., `JIRA API`, `Weather Service`). + +--- + +### **Credential Information** +1. **Key ID**: + - The unique identifier for the app's credentials. + +2. **Secret**: + - The app's secret key, masked for security purposes. + +3. **Active Status**: + - Indicates whether the app's credentials are active. + - Admins must activate credentials after the app is submitted. + +--- + +### **Proxy Logs** +- **Purpose**: + - Provides a detailed log of inbound and outbound requests processed by the app for governance and troubleshooting purposes. + +- **Columns**: + - **Timestamp**: The date and time of the request. + - **Vendor**: The upstream LLM vendor handling the request. + - **Response Code**: The HTTP status code of the response. + - **Request**: Details of the inbound request. + - **Response**: The response returned by the upstream LLM or tool. + +- **Pagination**: + - Allows administrators to navigate through logs in batches. + +--- + +### **Action Buttons** +1. **Edit App**: + - Opens the app configuration form for editing details such as name, description, LLMs, and data sources. + +2. **Back to Apps**: + - Navigates back to the **Apps List View**. + +--- + +#### **Purpose of the App Details View** + +1. **Monitoring Performance**: + - The token usage and cost graph provides insight into how efficiently the app is utilizing resources. + +2. **Governance and Compliance**: + - Proxy logs and credential management ensure transparency and compliance with organizational policies. + +3. **Troubleshooting**: + - Administrators can use detailed request and response logs to identify and resolve issues. + +4. **Security**: + - Credential status and masking ensure that sensitive information is handled securely. + +--- + +This **App Details View** is an essential feature for administrators to monitor, manage, and ensure the secure operation of user-created apps within the Tyk AI Studio. + +### App Editing and Credential Approval + +The **App Edit View** allows administrators to modify app details and manage credentials for user-created applications. Most importantly, this view provides controls to **approve or reject** an app by activating or deactivating its credentials. + +--- + +#### **Sections and Features** + +### **App Details** +1. **Name** *(Required)*: + - Editable field for the app's name (e.g., `Jira Task Refiner`). + +2. **Description** *(Optional)*: + - A short summary of the app's functionality or purpose (e.g., "Experiment to refine tasks for Jira"). + +3. **User** *(Read-Only)*: + - Displays the user who created the app (e.g., `Jeffy Mathew`). + +4. **LLMs** *(Dropdown)*: + - Select or modify the Large Language Models associated with this app. + - Example: OpenAI's GPT-4, Anthropic's Claude. + +5. **Datasources** *(Dropdown)*: + - Add or remove data sources the app can access. + - Example: `Tyk Documentation`. + +6. **Tools** *(Dropdown)*: + - Subscribe to or remove tools the app can use. + - Tools are available from the AI Portal tool catalogue. + - Example: `JIRA API`, `Weather Service`. + +--- + +### **Credential Information** +1. **Key ID** *(Read-Only)*: + - The unique identifier for the app's credentials. + +2. **Secret** *(Masked)*: + - A secure key that is masked by default but used for API integration. + +3. **Active Toggle**: + - **Purpose**: Approve or reject the app by activating or deactivating its credentials. + - **States**: + - **Inactive**: The app is not approved, and credentials cannot be used. + - **Active**: The app is approved, enabling API access via the AI Gateway. + +--- + +### **Action Buttons** +1. **Update App**: + - Saves changes made to the app's details and credentials. + +2. **Back to Apps**: + - Navigates back to the **Apps List View** without saving changes. + +--- + +#### **Use Cases** + +1. **Approval Workflow**: + - After a user submits an app, administrators can review its configuration and activate the credentials if it complies with governance policies. + +2. **Editing App Details**: + - Admins can modify the app's name, description, associated LLMs, and data sources to refine its configuration. + +3. **Credential Management**: + - Credentials can be deactivated if the app no longer complies with organizational requirements. + +4. **Governance and Security**: + - Ensures apps have controlled access to LLMs and data sources, adhering to organizational policies. + +--- + +#### **Purpose and Benefits** + +1. **Enhanced Control**: + - Enables administrators to manage the lifecycle of user-created apps, ensuring proper governance. + +2. **Simplified Approval Process**: + - The credential activation toggle streamlines the process of approving or rejecting apps. + +3. **Secure Integration**: + - Ensures only approved apps can interact with the AI Gateway, protecting sensitive data and resources. + +--- + +The **App Edit View** is a critical feature for managing user-created apps in the Tyk AI Studio, providing administrators with full control over app configuration, approval, and access management. + +### Privacy Level Validation + +When creating or updating an app, the system validates that the privacy levels of selected datasources are not higher than the privacy levels of selected LLMs. This ensures data governance and prevents sensitive data from being exposed to less secure LLMs. + +Privacy levels define how data is protected by controlling LLM access based on its sensitivity. LLM providers with lower privacy levels can't access higher-level data sources and tools, ensuring secure and appropriate data handling. + +The system works with 4 privacy levels from low to high: +- Public – Safe to share (e.g., blogs, press releases). +- Internal – Company-only info (e.g., reports, policies). +- Confidential – Sensitive business data (e.g., financials, strategies). +- Restricted (PII) – Personal data (e.g., names, emails, customer info). + +If you attempt to create or update an app with datasources that have higher privacy requirements (levels) than the selected LLMs, you'll receive an error message: "Datasources have higher privacy requirements than the selected LLMs. Please select LLMs with equal or higher privacy levels." + +To resolve this issue, either: +1. Select LLMs with higher privacy levels that match or exceed your datasource requirements +2. Use datasources with lower privacy requirements that are compatible with your selected LLMs diff --git a/ai-management/ai-studio/budgeting.mdx b/ai-management/ai-studio/budgeting.mdx new file mode 100644 index 00000000..06c4b3ef --- /dev/null +++ b/ai-management/ai-studio/budgeting.mdx @@ -0,0 +1,60 @@ +--- +title: "Budget Control" +'og:description': "How to configure budgets in Tyk AI Studio?" +tags: ['AI Studio', 'AI Management', 'Budget Control'] +sidebarTitle: "Budget Control" +--- + +Tyk AI Studio provides a Budget Control system to help organizations manage and limit spending on Large Language Model (LLM) usage. + +## Purpose + +The primary goals of the Budget Control system are: + +* **Prevent Overspending:** Set hard limits on costs associated with LLM API calls. +* **Cost Allocation:** Track and enforce spending limits at different granularities (e.g., per organization, per specific LLM configuration). +* **Predictability:** Provide better predictability for monthly AI operational costs. + +## Scope & Configuration + +Budgets are typically configured by administrators and applied at specific levels: + +* **Organization Level:** A global budget limit for all LLM usage within the organization. +* **LLM Configuration Level:** A specific budget limit tied to a particular LLM setup (e.g., a dedicated budget for a high-cost `gpt-4` configuration). +* **(Potentially) Application/User Level:** Granular budgets might be assignable to specific applications or teams (depending on implementation specifics). + +**Configuration Parameters:** + +* **Limit Amount:** The maximum monetary value allowed (e.g., $500). +* **Currency:** The currency the budget is defined in (e.g., USD). +* **Time Period:** The reset interval for the budget, typically monthly (e.g., resets on the 1st of each month). +* **Scope:** Which entity the budget applies to (Organization, specific LLM Configuration ID, etc.). + +Administrators configure these budgets via the Tyk AI Studio UI or API. + + Budget Config UI + +## Enforcement + +Budget enforcement primarily occurs at the **[Proxy & API Gateway](/ai-management/ai-studio/proxy)**: + +1. **Request Received:** The Proxy receives a request destined for an LLM. +2. **Cost Estimation:** Before forwarding the request, the Proxy might estimate the potential maximum cost (or rely on post-request cost calculation). +3. **Budget Check:** The Proxy checks the current spending against all applicable budgets (e.g., the specific LLM config budget AND the overall organization budget) for the current time period. +4. **Allow or Deny:** + * If the current spending plus the estimated/actual cost of the request does *not* exceed the limit(s), the request is allowed to proceed. + * If the request *would* cause a budget limit to be exceeded, the request is blocked, and an error is returned to the caller. + +## Integration with Other Systems + +* **[Analytics & Monitoring](/ai-management/ai-studio/analytics):** The Analytics system provides the cost data used to track spending against budgets. The current spent amount for a budget period is derived from aggregated analytics data. +* **[Model Pricing](/ai-management/ai-studio/llm-management):** The pricing definitions are essential for the Analytics system to calculate costs accurately, which in turn feeds the Budget Control system. +* **[Notification System](/ai-management/ai-studio/notifications):** Budgets can be configured to trigger notifications when spending approaches or reaches defined thresholds (e.g., alert admin when 80% of budget is consumed, notify user/admin when budget is exceeded). + +## Benefits + +* **Financial Control:** Prevents unexpected high bills from LLM usage. +* **Resource Management:** Ensures fair distribution of AI resources according to allocated budgets. +* **Accountability:** Tracks spending against specific configurations or organizational units. + +Budget Control is a critical feature for organizations looking to adopt AI technologies responsibly and manage their operational costs effectively. diff --git a/ai-management/ai-studio/call-settings.mdx b/ai-management/ai-studio/call-settings.mdx new file mode 100644 index 00000000..fbbe5aff --- /dev/null +++ b/ai-management/ai-studio/call-settings.mdx @@ -0,0 +1,112 @@ +--- +title: "LLM Call Settings" +'og:description': "How to configure LLMs in AI Studio?" +tags: ['AI Studio', 'AI Management', 'LLMs', 'Large Language Models', 'LLM Call Settings'] +--- + +The **LLM Call Settings** section allows administrators to configure default runtime parameters for Large Language Models (LLMs) used in chat interactions and middleware system function calls. These settings provide control over how the LLM processes inputs and generates outputs. It is important to note that these settings are not utilized in the AI Gateway proxy since applications are expected to define their own model configurations. + +--- + +#### **Table Overview** +The table lists all configured call settings for available LLMs with the following columns: + +1. **Model Name**: + The specific name or version of the LLM for which the call settings apply (e.g., `claude-3.5-sonnet-20240620`, `gpt-4o`). + +2. **Temperature**: + A numerical value (e.g., `0.7`, `0.1`) that controls the randomness of the LLM's responses: + - Higher values (e.g., `0.7`) produce more creative and varied outputs. + - Lower values (e.g., `0.1`) generate more deterministic and focused responses. + +3. **Max Tokens**: + The maximum number of tokens the LLM can generate in a single response. This sets a hard limit on the length of the output to ensure efficiency and prevent excessive usage. + +4. **Actions**: + A menu (three-dot icon) with quick actions to edit or delete the call settings for a specific model. + +--- + +#### **Features** +1. **Add LLM Call Setting Button**: + A green button labeled **+ ADD LLM CALL SETTING**, located at the top-right of the view. Clicking this button opens a form to define call settings for a new model. + +2. **Pagination Dropdown**: + Found at the bottom-left corner, this dropdown allows users to control how many call settings are displayed per page (e.g., 10, 20, etc.). + +--- + +#### **Use Cases** +- **Chats**: + These settings control how the LLM responds in conversational interfaces within the Chat Room feature, allowing administrators to fine-tune the user experience. + +- **Middleware Function Calls**: + The settings guide LLM behavior in automated backend processes where the LLM is used for tasks such as data generation or content analysis. + +--- + +#### **Quick Insights** +The **LLM Call Settings** section provides administrators with granular control over LLM behavior during runtime. While the settings are not used in the proxy, they are crucial for managing system-level and chat-specific interactions, ensuring consistent performance and efficiency. This section enables streamlined configuration for application-level integration of LLMs. + +### Edit/Create Call Settings + +The **Edit/Create LLM Call Settings View** enables administrators to configure or update call-time options for a specific Large Language Model (LLM). These settings determine how the LLM processes inputs and generates outputs in chat interactions or middleware system function calls. Below is an explanation of each field and its purpose: + +--- + +#### **Form Fields and Descriptions** + +1. **Model Preset** *(Dropdown)*: + - Allows administrators to select a pre-configured preset for the LLM, or choose "Custom" to manually configure all settings. + +2. **Model Name** *(Required)*: + - Specifies the name of the LLM model these settings apply to (e.g., `claude-3.5-sonnet-20240620`). + +3. **Temperature** *(Decimal, 0.0 to 1.0)*: + - Controls the randomness of the model's responses: + - **Higher values** (e.g., `0.7`): More creative and varied outputs. + - **Lower values** (e.g., `0.1`): More deterministic and repetitive outputs. + +4. **Max Tokens** *(Integer)*: + - Defines the maximum number of tokens the LLM can generate in its response. + - Helps to limit response length for efficiency and control. + +5. **Top P** *(Decimal, 0.0 to 1.0)*: + - Controls nucleus sampling, a method to limit token selection to the most probable subset: + - **Higher values** (e.g., `0.9`): Includes more variability in token choices. + - **Lower values** (e.g., `0.1`): Focuses on the most likely tokens. + +6. **Top K** *(Integer)*: + - Limits token selection to the top `K` most probable tokens at each step: + - **Higher values** allow for more varied responses. + - **Lower values** restrict outputs to fewer options. + +7. **Min Length** *(Integer)*: + - Sets the minimum number of tokens that must be included in the model's response. + +8. **Max Length** *(Integer)*: + - Specifies the upper limit for the length of the response (e.g., 200,000 tokens). + +9. **Repetition Penalty** *(Decimal)*: + - Penalizes repeated tokens to prevent the model from generating repetitive responses. + - **Higher values** (e.g., `1.5`): Stronger penalty for repetition. + - **Lower values** (e.g., `1.0`): Little or no penalty. + +10. **System Prompt** *(Optional)*: + - A predefined instruction for the LLM that sets the tone or context for its responses. + Example: + *"You are a helpful AI assistant specializing in API management and related topics. Respond in markdown and cite sources where appropriate."* + +--- + +#### **Action Buttons** +1. **Update LLM Call Settings / Create LLM Call Settings**: + - Saves the configuration or creates a new call settings entry. This button becomes active only when all required fields are valid. + +2. **Back to LLM Call Settings**: + - A link in the top-right corner to return to the main LLM Call Settings view without saving changes. + +--- + +#### **Purpose** +This interface provides granular control over runtime parameters, allowing administrators to customize the LLM's behavior for different use cases. These settings are critical for ensuring that the model generates responses tailored to specific operational or user requirements. diff --git a/ai-management/ai-studio/catalogs.mdx b/ai-management/ai-studio/catalogs.mdx new file mode 100644 index 00000000..07e1a0b4 --- /dev/null +++ b/ai-management/ai-studio/catalogs.mdx @@ -0,0 +1,76 @@ +--- +title: "Catalogs View for Tyk AI Studio" +'og:description': "How to manage catalogs and in AI Studio?" +tags: ['AI Studio', 'AI Management', 'Catalogs'] +--- + +The **Catalogs View** provides administrators with a centralized interface to group and manage related resources such as Large Language Models (LLMs), Data Sources, and Tools. By organizing these resources into catalogs, administrators can efficiently manage access control and simplify the assignment of resources to user groups. + +--- + +#### **Catalogs Overview** + +1. **Purpose**: + - Catalogs group similar resources (LLMs, Data Sources, or Tools) for streamlined management. + - Access to these resources is controlled by assigning catalogs to specific teams. + +2. **Use Cases**: + - Simplify resource management by categorizing related LLMs, Data Sources, or Tools into a single catalog. + - Apply consistent access control policies by assigning catalogs to teams. + - Manage large quantities of resources efficiently in a growing environment. + +--- + +#### **Catalogs View Layout** + +1. **Columns**: + - **Name**: + - The name of the catalog (e.g., `Main Catalogue`). + - **Resources**: + - Lists the resources included in the catalog: + - For LLMs: Displays the LLMs included (e.g., `Anthropic`, `OpenAI GPT-4`). + - For Data Sources: Shows vector or relational data sources. + - For Tools: Lists available tools (e.g., APIs, web scrapers). + - **Actions**: + - A dropdown menu to perform actions such as editing or deleting the catalog. + +2. **Add Catalog Button**: + - Located at the top-right of the view, this green button (**+ ADD CATALOG**) opens a form to create a new catalog. + +3. **Pagination Control**: + - Adjust the number of catalogs displayed per page using the dropdown at the bottom-left. + +--- + +#### **Cross-Resource Applicability** + +This view applies to the following catalog types: + +1. **LLM Catalogs**: + - Group and manage collections of LLMs, enabling administrators to easily control access to models based on their functionality or vendor. + +2. **Data Source Catalogs**: + - Organize vector or structured data sources into catalogs for efficient assignment and governance. + +3. **Tool Catalogs**: + - Categorize tools (e.g., APIs, search utilities) into logical groups to streamline their use in chat rooms or applications. + +--- + +#### **Benefits** + +1. **Centralized Management**: + - Simplifies resource organization and reduces redundancy. + +2. **Efficient Governance**: + - Allows for consistent application of access control policies across groups. + +3. **Scalability**: + - Handles large numbers of resources effectively as organizational needs evolve. + +4. **Flexible Integration**: + - Ensures that related resources are grouped for seamless assignment to teams or chat rooms. + +--- + +This generalized view supports catalog management for all resource types, providing a uniform approach to simplify administration while maintaining security and organizational efficiency. diff --git a/ai-management/ai-studio/chat-interface.mdx b/ai-management/ai-studio/chat-interface.mdx new file mode 100644 index 00000000..464648df --- /dev/null +++ b/ai-management/ai-studio/chat-interface.mdx @@ -0,0 +1,61 @@ +--- +title: "Chat Interface" +'og:description': "How AI Studios Chat Interface works?" +tags: ['AI Studio', 'AI Management', 'Chat Interface'] +sidebarTitle: "Chat Interface" +--- + +# Chat Interface + +Tyk AI Studio's Chat Interface provides a secure and interactive environment for users to engage with Large Language Models (LLMs), leveraging integrated tools and data sources. It serves as the primary front-end for conversational AI interactions within the platform. + +## Purpose + +The main goals of the Chat Interface are: + +* **User-Friendly Interaction:** Offer an intuitive web-based chat experience for users of all technical levels. +* **Unified Access:** Provide a single point of access to various configured LLMs, Tools, and Data Sources. +* **Context Management:** Maintain conversation history and manage context, including system prompts and retrieved data (RAG). +* **Secure & Governed:** Enforce access controls based on teams and apply configured Filters. + +## Key Features + +* **Chat Sessions:** Each conversation happens within a session, preserving history and context. +* **Streaming Responses:** LLM responses are streamed back to the user for a more interactive feel. +* **Tool Integration:** Seamlessly uses configured [Tools](/ai-management/ai-studio/tools) when the LLM determines they are necessary to fulfill a user's request. The available tools depend on the Chat Experience configuration and the user's group permissions. +* **Data Source (RAG) Integration:** Can automatically query configured [Data Sources](/ai-management/ai-studio/datasources-rag) to retrieve relevant information (Retrieval-Augmented Generation) to enhance LLM responses. The available data sources depend on the Chat Experience configuration and the user's group permissions. +* **System Prompts:** Administrators can define specific system prompts for different Chat Experiences to guide the LLM's persona, tone, and behavior. +* **History:** Users can view their past chat sessions. +* **File Upload (Context):** Users might be able to upload files directly within a chat to provide temporary context for the LLM (depending on configuration). +* **Access Control:** Users only see and can interact with Chat Experiences assigned to their Teams. + +## Using the Chat Interface + +Users access the Chat Interface through the Tyk AI Studio web UI. + +1. **Select Chat Experience:** Users choose from a list of available Chat Experiences (pre-configured chat environments) they have access to. +2. **Interact:** Users type their prompts or questions. +3. **Receive Responses:** The LLM processes the request, potentially using tools or data sources behind the scenes, and streams the response back. + + Chat UI + +## Configuration (Admin) + +Administrators configure the available "Chat Experiences" (formerly known as Chat Rooms) via the UI or API. Configuration involves: + +* **Naming:** Giving the Chat Experience a descriptive name. +* **Assigning LLM:** Linking to a specific [LLM Configuration](/ai-management/ai-studio/llm-management). +* **Enabling Tools:** Selecting which [Tool Catalogues](/ai-management/ai-studio/tools) are available. +* **Enabling Data Sources:** Selecting which [Data Source Catalogues](/ai-management/ai-studio/datasources-rag) are available. +* **Setting System Prompt:** Defining the guiding prompt for the LLM. +* **Applying Filters:** Associating specific [Filters](/ai-management/ai-studio/filters) for governance. +* **Assigning Groups:** Determining which Teams can access this Chat Experience. +* **Enabling/Disabling Features:** Toggling features like file uploads or direct tool usage. + + Chat Config + +## API Access + +Beyond the UI, Tyk AI Studio provides APIs (`/api/v1/chat/...`) for programmatic interaction with the chat system, allowing developers to build custom applications or integrations that leverage the configured Chat Experiences. + +This comprehensive system provides a powerful yet controlled way for users to interact with AI capabilities managed by Tyk AI Studio. diff --git a/ai-management/ai-studio/configuration.mdx b/ai-management/ai-studio/configuration.mdx new file mode 100644 index 00000000..3ae41295 --- /dev/null +++ b/ai-management/ai-studio/configuration.mdx @@ -0,0 +1,74 @@ +--- +title: "Initial Configuration" +'og:description': "Configuration of Tyk AI Studio" +tags: ['AI Studio', 'AI Management'] +sidebarTitle: "Initial Configuration" +--- + +This guide covers the essential first steps to take within the Tyk AI Studio UI after successfully deploying the platform using the [Installation Guide](/ai-management/ai-studio/deployment-k8s). + +## 1. First Login + +1. **Access the UI:** Open your web browser and navigate to the `SITE_URL` (or the Ingress host configured for `app.yourdomain.com`) specified during deployment. +2. **Admin Credentials:** Log in using the initial administrator credentials. This is typically: + * **Email:** The `ADMIN_EMAIL` configured during deployment (e.g., `admin@yourdomain.com`). + * **Password:** Check the deployment logs or default configuration for the initial admin password setup. You may be required to set a new password upon first login. + + Login Screen + +## 2. Configure Your First LLM + +One of the most common initial steps is connecting Tyk AI Studio to an LLM provider. + +1. **Navigate to LLM Management:** In the admin UI sidebar, find the section for LLM Management (or similar) and select it. +2. **Add LLM Configuration:** Click the button to add a new LLM Configuration. +3. **Select Provider:** Choose the LLM provider you want to connect (e.g., OpenAI, Anthropic, Azure OpenAI). +4. **Enter Details:** + * **Configuration Name:** Give it a recognizable name (e.g., `OpenAI-GPT-4o`). + * **Model Name(s):** Specify the exact model identifier(s) provided by the vendor (e.g., `gpt-4o`, `gpt-4-turbo`). + * **API Key (Using Secrets):** + * **IMPORTANT:** Do *not* paste your API key directly here. Instead, use [Secrets Management](/ai-management/ai-studio/secrets). + * If you haven't already, go to the **Secrets** section in the admin UI and create a new secret: + * **Variable Name:** `OPENAI_API_KEY` (or similar) + * **Secret Value:** Paste your actual OpenAI API key here. + * Save the secret. + * Return to the LLM Configuration screen. + * In the API Key field, enter the secret reference: `$SECRET/OPENAI_API_KEY` (using the exact Variable Name you created). + * **Other Parameters:** Configure any other provider-specific settings (e.g., Base URL for Azure/custom endpoints, default temperature, etc.). +5. **Save:** Save the LLM configuration. + + LLM Config UI + +This LLM is now available for use within Tyk AI Studio, subject to [User/Group permissions](/ai-management/ai-studio/user-management). + +For more details, see the [LLM Management](/ai-management/ai-studio/llm-management) documentation. + +## 3. Verify Core System Settings + +While most core settings are configured during deployment, you can usually review them within the administration UI: + +* **Site URL:** Check that the base URL for accessing the portal is correct. +* **Email Configuration:** If using features like user invites or notifications, ensure SMTP settings are correctly configured and test email delivery if possible ([Notifications](/ai-management/ai-studio/notifications)). + +## 4. Configuration Reference (Deployment) + +Remember that fundamental system parameters are typically set via environment variables or Helm values *during deployment*. This includes: + +* Database Connection (`DATABASE_TYPE`, `DATABASE_URL`) +* License Key (`TYK_AI_LICENSE`) +* Secrets Encryption Key (`TYK_AI_SECRET_KEY`) +* Base URL (`SITE_URL`) +* Email Server Settings (`SMTP_*`, `FROM_EMAIL`, `ADMIN_EMAIL`) +* Registration Settings (`ALLOW_REGISTRATIONS`, `FILTER_SIGNUP_DOMAINS`) + +Refer to the **Configuration Options** detailed within the [Installation Guide](/ai-management/ai-studio/deployment-k8s) for specifics on setting these values during the deployment process. + +## Next Steps + +With the initial configuration complete, you can now: + +* Explore [User Management](/ai-management/ai-studio/user-management) to create users and groups. +* Set up [Tools](/ai-management/ai-studio/tools) for external API integration. +* Configure [Data Sources](/ai-management/ai-studio/datasources-rag) for RAG. +* Define [Filters](/ai-management/ai-studio/filters) for custom request/response logic. +* Try out the [Chat Interface](/ai-management/ai-studio/chat-interface). diff --git a/ai-management/ai-studio/core-concepts.mdx b/ai-management/ai-studio/core-concepts.mdx new file mode 100644 index 00000000..3b4f943d --- /dev/null +++ b/ai-management/ai-studio/core-concepts.mdx @@ -0,0 +1,38 @@ +--- +title: "Core Concepts" +'og:description': "Core Concepts of AI Studio" +tags: ['AI Studio', 'AI Management', 'Core Concepts'] +sidebarTitle: "Core Concepts" +--- + +Welcome to Tyk AI Studio! Before diving into specific features, understanding these core concepts will help you navigate the platform and its capabilities. + +## Key Components & Philosophy + +Tyk AI Studio is designed as a secure, observable, and extensible gateway for interacting with Large Language Models (LLMs) and other AI services. Key architectural pillars include: + +* **[AI Gateway](/ai-management/ai-studio/proxy):** The central gateway managing all interactions between your applications and various LLM providers. It enforces policies, logs activity, and handles vendor abstraction. +* **AI Portal:** Empowers developers with a curated catalog of AI tools and services for faster innovation. +* **[Chat](/ai-management/ai-studio/chat-interface):** Provides a secure and interactive environment for users to engage with LLMs, leveraging integrated tools and data sources. +* **[User Management & RBAC](/ai-management/ai-studio/user-management):** Securely manages users, groups, and permissions. Access to resources like LLMs, Tools, and Data Sources is controlled via group memberships. +* **Extensibility ([Tools](/ai-management/ai-studio/tools) & [Data Sources](/ai-management/ai-studio/datasources-rag)):** Allows integrating external APIs (Tools) and vector databases (Data Sources) into LLM workflows (e.g., for Retrieval-Augmented Generation - RAG). +* **Policy Enforcement ([Filters](/ai-management/ai-studio/filters)):** Intercept and modify LLM requests/responses using custom scripts to enforce specific rules or data transformations. +* **Configuration over Code:** Many aspects like LLM parameters, Filters, and [Budgets](/ai-management/ai-studio/budgeting) are configured through the UI/API rather than requiring code changes. +* **Security First:** Features like [Secrets Management](/ai-management/ai-studio/secrets), [SSO integration](/ai-management/ai-studio/sso), and fine-grained access control are integral to the platform. +* **Observability:** Includes systems for [Analytics & Monitoring](/ai-management/ai-studio/analytics) and [Notifications](/ai-management/ai-studio/notifications) to track usage, costs, and system events. + +## Core Entities + +Understanding these entities is crucial: + +* **[User](/ai-management/ai-studio/user-management#core-concepts):** Represents an individual interacting with Tyk AI Studio, managed within the User Management system. +* **[Group](/ai-management/ai-studio/user-management#core-concepts):** Collections of users, the primary mechanism for assigning access rights to resources via RBAC. +* **[API Key](/ai-management/ai-studio/user-management#core-concepts):** Credentials generated by Users to allow applications or scripts programmatic access to Tyk AI Studio APIs (like the Proxy), inheriting the User's permissions. +* **[LLM Configuration](/ai-management/ai-studio/llm-management):** Represents a specific LLM provider and model setup (e.g., OpenAI GPT-4, Anthropic Claude 3), including parameters and potentially associated [pricing](/ai-management/ai-studio/llm-management) and [budgets](/ai-management/ai-studio/budgeting). +* **[Tool](/ai-management/ai-studio/tools):** Definitions of external APIs (via OpenAPI spec) that can be invoked by LLMs during chat sessions to perform actions or retrieve external data. +* **[Data Source](/ai-management/ai-studio/datasources-rag):** Connections to vector databases or other data repositories used for Retrieval-Augmented Generation (RAG) within chat sessions. +* **[Catalogue](/ai-management/ai-studio/tools) ([Tools](/ai-management/ai-studio/tools) / [Data Sources](/ai-management/ai-studio/datasources-rag)):** Collections that group related Tools or Data Sources for easier management and assignment to Groups for access control. +* **[Secret](/ai-management/ai-studio/secrets):** Securely stored credentials (API keys, tokens) referenced indirectly (e.g., `$SECRET/MY_KEY`) in configurations like LLMs, Tools, or Data Sources. +* **[Filter](/ai-management/ai-studio/filters):** Custom logic (using Tengo scripts) associated with specific execution points (e.g., pre/post LLM request) to intercept and modify requests/responses. + +This page provides a high-level overview. Click the links above or use the sidebar to navigate to the detailed documentation for each feature. diff --git a/ai-management/ai-studio/dashboard.mdx b/ai-management/ai-studio/dashboard.mdx new file mode 100644 index 00000000..71dc9040 --- /dev/null +++ b/ai-management/ai-studio/dashboard.mdx @@ -0,0 +1,55 @@ +--- +title: "Dashboard Overview for Tyk AI Studio" +'og:description': "Overview of dashboard in AI Studio?" +tags: ['AI Studio', 'AI Management', 'Dashboard'] +--- + +This dashboard provides an overview of user engagement, cost analysis, and tool/model usage for the Tyk AI Studio. Below is a breakdown of the various elements and their significance: + +--- + +#### **Conversations** +This section monitors user interaction trends over a selected time period. + +- **Unique Users per Day**: + A line graph displaying the daily count of unique users engaging with the platform. This metric helps track active user trends. + +- **Chat Interactions per Day**: + A line graph showing the number of chat interactions per day. This reflects user activity levels and engagement with AI-powered chat features. + +--- + +#### **Cost Analysis** +This section offers insights into the expenditure associated with the usage of Large Language Models (LLMs). + +- **Cost Analysis by Currency**: + A line chart representing the cost incurred over time, broken down by currency. This visual allows for tracking fluctuations in platform expenditure. + +- **Total Cost per Vendor and Model**: + A table summarizing costs by vendor and model, including: + - **Vendor**: The provider of the LLM (e.g., Anthropic, OpenAI). + - **Model**: The specific LLM used (e.g., `claude-3.5`, `gpt-4.0`). + - **Total Cost**: The cumulative cost for the model. + - **Currency**: The unit of currency in which the cost is calculated (e.g., USD). + +--- + +#### **Model and Tool Usage** +This section highlights the utilization statistics for LLM models and tools within the platform. + +- **Most Used LLM Models**: + A bar chart showcasing the frequency of usage for different LLM models. This data provides insights into which models are most relied upon. + +- **Tool Usage Statistics**: + A bar chart detailing the frequency of tool usage across various functionalities, such as vector searches, sandbox environments, and data management. This helps identify popular tools and areas for optimization. + +--- + +#### **Additional Features** +- **Date Selector**: Located in the top-right corner, this allows users to define a custom date range for analyzing trends across the dashboard. + +- **Feedback Button**: A floating button labeled "Feedback" to enable users to report issues or provide suggestions regarding the dashboard. + +--- + +This dashboard serves as a comprehensive monitoring and analytical tool for administrators and users to evaluate platform performance, user engagement, and cost efficiency. diff --git a/ai-management/ai-studio/datasources-rag.mdx b/ai-management/ai-studio/datasources-rag.mdx new file mode 100644 index 00000000..14923de9 --- /dev/null +++ b/ai-management/ai-studio/datasources-rag.mdx @@ -0,0 +1,200 @@ +--- +title: "Data Sources & RAG" +'og:description': "How to integrate Data Sources & RAG in Tyk AI Studio?" +tags: ['AI Studio', 'AI Management', 'Datasources', 'RAG'] +sidebarTitle: "Data Sources & RAG" +--- + +Tyk AI Studio's Data Source system connects the platform to external knowledge bases, primarily vector stores, enabling **Retrieval-Augmented Generation (RAG)**. This allows Large Language Models (LLMs) to access and utilize specific information from your documents, grounding their responses in factual data. + +## Purpose + +The primary goal is to enhance LLM interactions by: + +* **Providing Context:** Injecting relevant information retrieved from configured data sources directly into the LLM prompt. +* **Improving Accuracy:** Reducing hallucinations and grounding LLM responses in specific, verifiable data. +* **Accessing Private Knowledge:** Allowing LLMs to leverage internal documentation, knowledge bases, or other proprietary information. + +## Core Concepts + +* **Data Source:** A configuration in Tyk AI Studio that defines a connection to a specific knowledge base (typically a vector store) and the associated embedding service used to populate it. +* **Vector Store Abstraction:** Tyk AI Studio provides a unified interface to interact with various vector database types (e.g., Pinecone, Milvus, ChromaDB). Administrators configure the connection details for their chosen store. +* **Embedding Service:** Text needs to be converted into numerical vector embeddings before being stored and searched. Administrators configure the embedding service (e.g., OpenAI `text-embedding-ada-002`, a local Sentence Transformer model via an API endpoint) and its credentials (using [Secrets Management](/ai-management/ai-studio/secrets)). +* **File Processing:** Administrators upload documents (e.g., PDF, TXT, DOCX) to a Data Source configuration. Tyk AI Studio automatically: + * Chunks the documents into smaller, manageable pieces. + * Uses the configured Embedding Service to convert each chunk into a vector embedding. + * Stores the text chunk and its corresponding embedding in the configured Vector Store. +* **RAG (Retrieval-Augmented Generation):** The core process where: + 1. A user's query in the [Chat Interface](/ai-management/ai-studio/chat-interface) is embedded using the same embedding service. + 2. This query embedding is used to search the relevant vector store(s) for the most similar text chunks (based on vector similarity). + 3. The retrieved text chunks are added as context to the prompt sent to the LLM. + 4. The LLM uses this context to generate a more informed and relevant response. +* **Data Source Catalogues:** Similar to Tools, Data Sources are grouped into Catalogues for easier management and assignment to teams. +* **Privacy Levels:** Each Data Source has a privacy level. It can only be used in RAG if its level is less than or equal to the privacy level of the [LLM Configuration](/ai-management/ai-studio/llm-management) being used, ensuring data governance. + + Privacy levels define how data is protected by controlling LLM access based on its sensitivity: + - Public – Safe to share (e.g., blogs, press releases). + - Internal – Company-only info (e.g., reports, policies). + - Confidential – Sensitive business data (e.g., financials, strategies). + - Restricted (PII) – Personal data (e.g., names, emails, customer info). + +## How RAG Works in the Chat Interface + +When RAG is enabled for a Chat Experience: + +1. User sends a prompt. +2. Tyk AI Studio embeds the user's prompt using the configured embedding service for the relevant Data Source(s). +3. Tyk AI Studio searches the configured Vector Store(s) using the prompt embedding to find relevant text chunks. +4. The retrieved chunks are formatted and added to the context window of the LLM prompt. +5. The combined prompt (original query + retrieved context) is sent to the LLM. +6. The LLM generates a response based on both the query and the provided context. +7. The response is streamed back to the user. + +## Creating & Managing Data Sources (Admin) + +Administrators configure Data Sources via the UI or API: + +1. **Define Data Source:** Provide a name, description, and privacy level. +2. **Configure Vector Store:** + * Select the database type (e.g., `pinecone`). + * Provide connection details (e.g., endpoint/connection string, namespace/index name). + * Reference a [Secret](/ai-management/ai-studio/secrets) containing the API key/credentials. +3. **Configure Embedding Service:** + * Select the vendor/type (e.g., `openai`, `local`). + * Specify the model name (if applicable). + * Provide the service URL (if applicable, for local models). + * Reference a [Secret](/ai-management/ai-studio/secrets) containing the API key (if applicable). +4. **Upload Files:** Upload documents to be chunked, embedded, and indexed into the vector store. + + Datasource Config + +## Organizing & Assigning Data Sources (Admin) + +* **Create Catalogues:** Group related Data Sources into Catalogues (e.g., "Product Docs", "Support KB"). +* **Assign to Groups:** Assign Data Source Catalogues to specific Teams. + + Catalogue Config + +## Using Data Sources (User) + +Data Sources are primarily used implicitly via RAG within the [Chat Interface](/ai-management/ai-studio/chat-interface). + +A Data Source will be used for RAG if: + +1. The specific Chat Experience configuration includes the relevant Data Source Catalogue. +2. The user belongs to a Team that has been assigned that Data Source Catalogue. +3. The Data Source's privacy level is compatible with the LLM being used. + +## Programmatic Access via API + +Tyk AI Studio provides a direct API endpoint for querying configured Data Sources programmatically: + +### Datasource API Endpoint + +* **Endpoint:** `/datasource/{dsSlug}` (where `{dsSlug}` is the datasource identifier) +* **Method:** POST +* **Authentication:** Bearer token required in the Authorization header + +### Request Format + +```json +{ + "query": "your semantic search query here", + "n": 5 // optional, number of results to return (default: 3) +} +``` + +### Response Format + +```json expandable +{ + "documents": [ + { + "content": "text content of the document chunk", + "metadata": { + "source": "filename.pdf", + "page": 42 + } + }, + // additional results... + ] +} +``` + +### Example Usage + +#### cURL + +```bash +curl -X POST "https://your-tyk-instance/datasource/product-docs" \ + -H "Authorization: Bearer YOUR_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"query": "How do I configure authentication?", "n": 3}' +``` + +#### Python + +```python expandable +import requests + +url = "https://your-tyk-instance/datasource/product-docs" +headers = { + "Authorization": "Bearer YOUR_TOKEN", + "Content-Type": "application/json" +} +payload = { + "query": "How do I configure authentication?", + "n": 3 +} + +response = requests.post(url, json=payload, headers=headers) +results = response.json() + +for doc in results["documents"]: + print(f"Content: {doc['content']}") + print(f"Source: {doc['metadata']['source']}") + print("---") +``` + +#### JavaScript + +```javascript expandable +async function queryDatasource() { + const response = await fetch('https://your-tyk-instance/datasource/product-docs', { + method: 'POST', + headers: { + 'Authorization': 'Bearer YOUR_TOKEN', + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ + query: 'How do I configure authentication?', + n: 3 + }) + }); + + const data = await response.json(); + + data.documents.forEach(doc => { + console.log(`Content: ${doc.content}`); + console.log(`Source: ${doc.metadata.source}`); + console.log('---'); + }); +} +``` + +### Common Issues and Troubleshooting + +1. **Trailing Slash Error:** The endpoint does not accept a trailing slash. Use `/datasource/{dsSlug}` and not `/datasource/{dsSlug}/`. + +2. **Authentication Errors:** Ensure your Bearer token is valid and has not expired. The token must have permissions to access the specified datasource. + +3. **404 Not Found:** Verify that the datasource slug is correct and that the datasource exists and is properly configured. + +4. **403 Forbidden:** Check that your user account has been granted access to the datasource catalogue containing this datasource. + +5. **Empty Results:** If you receive an empty documents array, try: + - Reformulating your query to better match the content + - Increasing the value of `n` to get more results + - Verifying that the datasource has been properly populated with documents + +This API endpoint allows developers to build custom applications that leverage the semantic search capabilities of configured vector stores without needing to implement the full RAG pipeline. diff --git a/ai-management/ai-studio/deployment-k8s.mdx b/ai-management/ai-studio/deployment-k8s.mdx new file mode 100644 index 00000000..e68d93ad --- /dev/null +++ b/ai-management/ai-studio/deployment-k8s.mdx @@ -0,0 +1,671 @@ +--- +title: "Installation (Kubernetes)" +'og:description': "Install Tyk AI Studio" +tags: ['AI Studio', 'AI Management'] +sidebarTitle: "Installation (Kubernetes)" +--- + +This guide explains how to deploy Tyk AI Studio, a secure and extensible AI gateway, using pure Kubernetes manifests. + +## Prerequisites + +- Kubernetes 1.16+ +- kubectl configured with access to your cluster +- A `TYK_AI_LICENSE` string from Tyk Technologies (contact support@tyk.io or your account manager to obtain) +- A securely generated `TYK_AI_SECRET_KEY` string for secrets encryption +- If using SSL/TLS: cert-manager installed in your cluster + +*Note: The following examples use placeholder values (e.g., `your-domain.com`, `your-secret-key`). Remember to replace these with your actual configuration values.* + +## Installation Options + +Tyk AI Studio can be deployed in several configurations: + +1. Local Development +2. Production without TLS +3. Production with TLS +4. Production with External Database + +### Option 1: Local Development Setup + +1. Create a `local-deployment.yaml` file: + +```yaml expandable +apiVersion: v1 +kind: Namespace +metadata: + name: tyk-ai-studio +--- +apiVersion: v1 +kind: Secret +metadata: + name: tyk-ai-config + namespace: tyk-ai-studio +type: Opaque +stringData: + ALLOW_REGISTRATIONS: "true" + ADMIN_EMAIL: "admin@localhost" + SITE_URL: "http://localhost:32580" + FROM_EMAIL: "noreply@localhost" + DEV_MODE: "true" + DATABASE_TYPE: "postgres" + TYK_AI_SECRET_KEY: "your-secret-key" + TYK_AI_LICENSE: "your-license" + DATABASE_URL: "postgres://postgres:localdev123@postgres:5432/tyk-ai-studio" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: tyk-ai-studio +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:13 + env: + - name: POSTGRES_DB + value: "tyk-ai-studio" + - name: POSTGRES_USER + value: "postgres" + - name: POSTGRES_PASSWORD + value: "localdev123" + ports: + - containerPort: 5432 + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + volumes: + - name: postgres-data + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: tyk-ai-studio +spec: + selector: + app: postgres + ports: + - port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tyk-ai-studio + namespace: tyk-ai-studio +spec: + replicas: 1 + selector: + matchLabels: + app: tyk-ai-studio + template: + metadata: + labels: + app: tyk-ai-studio + spec: + containers: + - name: ai-studio + image: tykio/ai-studio:latest + envFrom: + - secretRef: + name: tyk-ai-config + ports: + - containerPort: 8080 + - containerPort: 9090 +--- +apiVersion: v1 +kind: Service +metadata: + name: tyk-ai-studio + namespace: tyk-ai-studio +spec: + type: NodePort + selector: + app: tyk-ai-studio + ports: + - name: http + port: 8080 + targetPort: 8080 + nodePort: 32580 + - name: gateway + port: 9090 + targetPort: 9090 + nodePort: 32590 +``` + +2. Deploy the application: + +```bash +kubectl apply -f local-deployment.yaml +``` + +3. Access the application: +- Web Interface: http://localhost:32580 +- Gateway: http://localhost:32590 + +### Option 2: Production without TLS + +For a production deployment without TLS certificates: + +1. Create `production-no-tls.yaml`: + +```yaml expandable +apiVersion: v1 +kind: Namespace +metadata: + name: tyk-ai-studio +--- +apiVersion: v1 +kind: Secret +metadata: + name: tyk-ai-config + namespace: tyk-ai-studio +type: Opaque +stringData: + ALLOW_REGISTRATIONS: "true" + ADMIN_EMAIL: "admin@yourdomain.com" + SITE_URL: "http://app.yourdomain.com" + FROM_EMAIL: "noreply@yourdomain.com" + DEV_MODE: "false" + DATABASE_TYPE: "postgres" + TYK_AI_SECRET_KEY: "your-production-key" + TYK_AI_LICENSE: "your-production-license" + DATABASE_URL: "postgres://your-db-user:your-db-password@your-db-host:5432/tyk-ai-studio" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tyk-ai-studio + namespace: tyk-ai-studio +spec: + replicas: 2 + selector: + matchLabels: + app: tyk-ai-studio + template: + metadata: + labels: + app: tyk-ai-studio + spec: + containers: + - name: ai-studio + image: tykio/ai-studio:latest + envFrom: + - secretRef: + name: tyk-ai-config + ports: + - containerPort: 8080 + - containerPort: 9090 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: tyk-ai-studio + namespace: tyk-ai-studio +spec: + selector: + app: tyk-ai-studio + ports: + - name: http + port: 8080 + targetPort: 8080 + - name: gateway + port: 9090 + targetPort: 9090 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: tyk-ai-studio-ingress + namespace: tyk-ai-studio + annotations: + kubernetes.io/ingress.class: nginx +spec: + rules: + - host: app.yourdomain.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: tyk-ai-studio + port: + number: 8080 + - host: gateway.yourdomain.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: tyk-ai-studio + port: + number: 9090 +``` + +2. Deploy: + +```bash +kubectl apply -f production-no-tls.yaml +``` + +### Option 3: Production with TLS + +For a secure production deployment with TLS: + +1. Create `production-tls.yaml`: + +```yaml expandable +apiVersion: v1 +kind: Namespace +metadata: + name: tyk-ai-studio +--- +apiVersion: v1 +kind: Secret +metadata: + name: tyk-ai-config + namespace: tyk-ai-studio +type: Opaque +stringData: + ALLOW_REGISTRATIONS: "true" + ADMIN_EMAIL: "admin@yourdomain.com" + SITE_URL: "https://app.yourdomain.com" + FROM_EMAIL: "noreply@yourdomain.com" + DEV_MODE: "false" + DATABASE_TYPE: "postgres" + TYK_AI_SECRET_KEY: "your-production-key" + TYK_AI_LICENSE: "your-production-license" + DATABASE_URL: "postgres://user:password@your-production-db:5432/tyk-ai-studio" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tyk-ai-studio + namespace: tyk-ai-studio +spec: + replicas: 2 + selector: + matchLabels: + app: tyk-ai-studio + template: + metadata: + labels: + app: tyk-ai-studio + spec: + containers: + - name: ai-studio + image: tykio/ai-studio:latest + envFrom: + - secretRef: + name: tyk-ai-config + ports: + - containerPort: 8080 + - containerPort: 9090 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: tyk-ai-studio + namespace: tyk-ai-studio +spec: + selector: + app: tyk-ai-studio + ports: + - name: http + port: 8080 + targetPort: 8080 + - name: gateway + port: 9090 + targetPort: 9090 +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: app-tls-certificate + namespace: tyk-ai-studio +spec: + secretName: app-tls-secret + issuerRef: + name: letsencrypt-prod + kind: ClusterIssuer + dnsNames: + - app.yourdomain.com +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: gateway-tls-certificate + namespace: tyk-ai-studio +spec: + secretName: gateway-tls-secret + issuerRef: + name: letsencrypt-prod + kind: ClusterIssuer + dnsNames: + - gateway.yourdomain.com +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: tyk-ai-studio-ingress + namespace: tyk-ai-studio + annotations: + kubernetes.io/ingress.class: nginx + nginx.ingress.kubernetes.io/ssl-redirect: "true" +spec: + tls: + - hosts: + - app.yourdomain.com + secretName: app-tls-secret + - hosts: + - gateway.yourdomain.com + secretName: gateway-tls-secret + rules: + - host: app.yourdomain.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: tyk-ai-studio + port: + number: 8080 + - host: gateway.yourdomain.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: tyk-ai-studio + port: + number: 9090 +``` + +2. Deploy: + +```bash +kubectl apply -f production-tls.yaml +``` + +## Optional Components + +### Reranker Service + +The Reranker service improves RAG result relevance. Add it to your deployment: + +```yaml expandable +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: reranker + namespace: tyk-ai-studio +spec: + replicas: 1 + selector: + matchLabels: + app: reranker + template: + metadata: + labels: + app: reranker + spec: + containers: + - name: reranker + image: tykio/reranker_cpu:latest + ports: + - containerPort: 8080 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: reranker + namespace: tyk-ai-studio +spec: + selector: + app: reranker + ports: + - port: 8080 + targetPort: 8080 +``` + +### Transformer Server + +The Transformer Server handles embedding generation and model inference. Add it to your deployment: + +```yaml expandable +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: transformer-server + namespace: tyk-ai-studio +spec: + replicas: 1 + selector: + matchLabels: + app: transformer-server + template: + metadata: + labels: + app: transformer-server + spec: + containers: + - name: transformer-server + image: tykio/transformer_server_cpu:latest + ports: + - containerPort: 8080 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: transformer-server + namespace: tyk-ai-studio +spec: + selector: + app: transformer-server + ports: + - port: 8080 + targetPort: 8080 +``` + +## Database Options + +### Using Internal PostgreSQL + +For development or small deployments, you can deploy PostgreSQL within your cluster: + +```yaml expandable +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: postgres-pvc + namespace: tyk-ai-studio +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: standard +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: tyk-ai-studio +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + containers: + - name: postgres + image: postgres:13 + env: + - name: POSTGRES_DB + value: "tyk-ai-studio" + - name: POSTGRES_USER + value: "postgres" + - name: POSTGRES_PASSWORD + value: "secure-password" + ports: + - containerPort: 5432 + volumeMounts: + - name: postgres-storage + mountPath: /var/lib/postgresql/data + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 500m + memory: 1Gi + volumes: + - name: postgres-storage + persistentVolumeClaim: + claimName: postgres-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: tyk-ai-studio +spec: + selector: + app: postgres + ports: + - port: 5432 + targetPort: 5432 +``` + +### Using External Database + +For production environments, configure your external database connection in the Secret: + +```yaml expandable +apiVersion: v1 +kind: Secret +metadata: + name: tyk-ai-config + namespace: tyk-ai-studio +type: Opaque +stringData: + DATABASE_URL: "postgres://user:password@your-db-host:5432/tyk-ai-studio" + # ... other config values +``` + +## Maintenance + +### Upgrading + +To upgrade an existing installation: + +```bash +# Update the deployment with new configuration +kubectl apply -f your-deployment.yaml + +# Or update just the image +kubectl set image deployment/tyk-ai-studio ai-studio=tykio/ai-studio:new-version -n tyk-ai-studio +``` + +### Uninstalling + +To remove the deployment: + +```bash +# Delete all resources in the namespace +kubectl delete namespace tyk-ai-studio + +# Or delete specific resources +kubectl delete -f your-deployment.yaml +``` + +### Viewing Logs + +```bash expandable +# Main application logs +kubectl logs -l app.kubernetes.io/name=tyk-ai-studio + +# Database logs (if using internal database) +kubectl logs -l app=postgres + +# Optional component logs +kubectl logs -l app=reranker +kubectl logs -l app=transformer +``` + +## Troubleshooting + +1. Check pod status: +```bash +kubectl get pods +``` + +2. Check ingress configuration: +```bash +kubectl get ingress +``` + +3. View pod details: +```bash +kubectl describe pod +``` + +4. Common issues: +- Database connection failures: Check credentials and network access +- Ingress not working: Verify DNS records and TLS configuration +- Resource constraints: Check pod resource limits and node capacity + +## Next Steps + +Once deployed, proceed to the [Initial Configuration](/ai-management/ai-studio/configuration) guide to set up Tyk AI Studio. diff --git a/ai-management/ai-studio/filters.mdx b/ai-management/ai-studio/filters.mdx new file mode 100644 index 00000000..9477673a --- /dev/null +++ b/ai-management/ai-studio/filters.mdx @@ -0,0 +1,284 @@ +--- +title: "Filters and Middleware" +'og:description': "How to use Filters and Middleware in Tyk AI Studio?" +tags: ['AI Studio', 'AI Management', 'Filters', 'Middleware'] +sidebarTitle: "Filters & Policies" +--- + +The **Filters List View** allows administrators to manage filters and middleware applied to prompts or data sent to Large Language Models (LLMs) via the AI Gateway or Chat Rooms. Filters and middleware ensure data governance, compliance, and security by processing or controlling the flow of information. Below is an enhanced description with the distinction between **Filters** and **Middleware**: + +--- + +#### **Filters vs. Middleware** + +1. **Filters**: + - **Purpose**: Filters act as governance blocks that either approve or deny a prompt before it reaches the upstream LLM. + - **Behavior**: + - Filters **do not modify the prompt**. + - They analyze the contents of the input prompt to decide if it complies with organizational policies or contains restricted content. + - Example: A PII detector that blocks prompts containing sensitive information. + +2. **Middleware**: + - **Purpose**: Middleware processes prompts or outputs generated by tools, modifying them before they are passed on to the LLM. + - **Behavior**: + - Middleware **modifies the prompt or output** to enhance security, anonymize data, or perform other transformations. + - Middleware only works with tools (e.g., API-based services) and is not used directly with raw input prompts. + - Example: An anonymizer that removes Personally Identifiable Information (PII) from tool outputs. + +--- + +#### **Table Overview** + +1. **Name**: + - The name of the filter or middleware (e.g., `Anonymize PII (LLM)`, `Fixed PII Filter`). + +2. **Description**: + - A brief summary of the filter or middleware's functionality (e.g., "Uses Regex to remove obvious PII"). + +3. **Actions**: + - A menu (three-dot icon) that allows administrators to: + - Edit the filter or middleware. + - Delete the filter or middleware. + +--- + +#### **Features** + +1. **Add Filter Button**: + - A green button labeled **+ ADD FILTER**, located in the top-right corner. Clicking this button opens a form to create a new filter or middleware. + +2. **Pagination Dropdown**: + - Located at the bottom-left corner, this control allows administrators to adjust the number of entries displayed per page. + +--- + +#### **Examples of Filters and Middleware** + +- **Filters**: + - **PII Detector**: A regex-based filter that blocks prompts containing sensitive PII. + - **JIRA Field Analysis**: Ensures no PII is included in data retrieved from JIRA fields before passing to the LLM. + +- **Middleware**: + - **Anonymize PII (LLM)**: Uses an LLM to anonymize sensitive data before sending it downstream. + - **NER Service Filter**: A Named Entity Recognition (NER) microservice that modifies outputs to remove identified entities. + +--- + +#### **Use Cases** + +1. **Prompt Validation with Filters**: + - Ensures that only compliant and secure prompts are sent to LLMs. + - Example: Blocking a prompt with sensitive data that should not be processed by an unapproved vendor. + +2. **Data Preprocessing with Middleware**: + - Prepares data from tools or external sources for safe interaction with LLMs by modifying or anonymizing content. + - Example: Removing sensitive ticket details from a JIRA query before sending to an LLM. + +3. **Organizational Security**: + - Both filters and middleware ensure sensitive information is protected and handled in line with organizational governance policies. + +4. **Enhanced Tool Interactions**: + - Middleware supports tools by transforming their outputs, enabling richer and safer LLM interactions. + +--- + +#### **Key Benefits** + +1. **Improved Data Governance**: + - Filters and middleware work together to enforce strict controls over data flow, protecting sensitive information. + +2. **Flexibility**: + - Middleware allows for data transformation, enhancing interoperability between tools and LLMs. + - Filters ensure compliance without altering user-provided prompts. + +3. **Compliance and Security**: + - Prevent unauthorized or sensitive data from reaching unapproved vendors, ensuring regulatory compliance. + +This detailed structure for **Filters and Middleware** provides organizations with robust governance tools to secure and optimize data workflows in the Tyk AI Studio. + +### Filter Edit View (and example Filter) + +The **Filter Edit View** enables administrators to create or modify filters using the **Tengo scripting language**. Filters serve as governance tools that analyze input data (e.g., prompts or files) and decide whether the content is permitted to pass to the upstream LLM. In this example, the filter uses regular expressions (regex) to detect Personally Identifiable Information (PII) and blocks the prompt if any matches are found. + +--- + +#### **Form Sections and Fields** + +1. **Name** *(Required)*: + - Specifies the name of the filter (e.g., `PII Detector`). + +2. **Description** *(Optional)*: + - A brief summary of the filter's purpose and functionality (e.g., "Simple Regex-based PII detector to prevent the wrong data being sent to LLMs"). + +3. **Script** *(Required)*: + - A **Tengo script** that defines the logic of the filter. The script evaluates input data and determines whether the filter approves or blocks it. + - The example script detects PII using a collection of regex patterns and blocks the data if a match is found. + +--- + +#### **Example Script** + +This script demonstrates a regex-based PII detection filter: + +```tengo expandable +text := import("text") + +// regexes for various PII +patterns := { + "email": `[\w\.-]+@[\w\.-]+\.\w+`, + "phone": `\+?\d{1,3}?[-.\s]?\(?\d{1,4}?\)?[-.\s]?\d{1,4}[-.\s]?\d{1,9}`, + "ssn": `\b\d{3}-\d{2}-\d{4}\b`, + "credit_card": `\b(?:\d[ -]*?){13,16}\b`, + "ipv4": `\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b`, + "ipv6": `\b([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}\b`, + "dob": `\b\d{1,2}[/-]\d{1,2}[/-]\d{2,4}\b`, + "address": `\d+\s[A-Za-z]+\s[A-Za-z]+`, + "passport": `\b[A-PR-WYa-pr-wy][1-9]\d\s?\d{4}[1-9]\b`, + "drivers_license": `\b[A-Z]{1,2}\d{1,7}\b`, + "bank_account": `\b\d{8,17}\b` +} + +/* +payload := ` +John Doe, born on 12/25/1980, +resides at 1234 Elm Street, Springfield, IL 62704. +You can contact him via email at john.doe@example.com or +by phone at +1-555-123-4567. His Social Security Number is 123-45-6789, +and his U.S. passport number is 987654321. John's driver's license +number is D1234567, and his bank account number is 123456789012. +He often uses the IP address 192.168.1.1 to access his online banking. +` +*/ + +filter := func(payload) { + for key, pattern in patterns { + found := text.re_match(pattern, payload) + if found { + return false + } + } + + return true +} + +result := filter(payload) +``` + +--- + +#### **Key Features of the Script** + +1. **Patterns Dictionary**: + - Defines regex patterns for detecting specific PII types (e.g., email, phone, SSN, IP addresses, etc.). + +2. **Filter Function**: + - Iterates through the patterns and checks if the input payload matches any of them. + - If a match is found, the filter blocks the input (`return false`). + +3. **Usage Context**: + - This filter can be applied to prompts, files, or any other input to ensure that sensitive information is not unintentionally shared with an LLM. + +--- + +#### **Action Buttons** +1. **Update Filter / Create Filter**: + - Saves the filter configuration, making it active for future data processing. + +2. **Back to Filters**: + - Returns to the Filters List View without saving changes. + +--- + +#### **Purpose and Benefits** + +1. **Data Governance**: + - Enforces strict control over what data can be sent to LLMs, ensuring compliance with privacy regulations. + +2. **Flexibility**: + - Filters can be tailored to specific organizational needs using custom scripts. + +3. **Security**: + - Prevents sensitive or unauthorized data from leaking to unapproved vendors or external systems. + +This **Filter Edit View** provides a robust and customizable interface for creating scripts to enforce data governance and security in the Tyk AI Studio. + +### Example Middleware for Tools + +Middleware filters in the Tyk AI Studio modify data coming from tools before passing it to the LLM. These filters are applied to sanitize, anonymize, or enhance the data to ensure it complies with organizational standards and privacy regulations. Below is an example of a middleware filter that sanitizes Personally Identifiable Information (PII), specifically email addresses, from the tool's output. + +--- + +#### **Middleware Script: Email Redaction Example** + +```tengo expandable +// Import the 'text' module for regular expression operations +text := import("text") + +// Define regular expression patterns for various PII +email_pattern := `[\w\.-]+@[\w\.-]+\.\w+` + +// Define the function to sanitize PII in the input string +filter := func(input) { + // Replace email addresses + input = text.re_replace(email_pattern, input, "[REDACTED EMAIL]") + + return input +} + +// Process the input payload +result := filter(payload) +``` + +--- + +#### **Explanation of the Script** + +1. **Module Import**: + - The `text` module is imported to enable regular expression operations (`text.re_replace`). + +2. **Regex Pattern**: + - A regex pattern is defined to detect email addresses: + - Example pattern: `[\w\.-]+@[\w\.-]+\.\w+` + - This pattern matches standard email formats. + +3. **Filter Function**: + - The `filter` function accepts an input string (e.g., tool output) and: + - Uses `text.re_replace` to identify email addresses. + - Replaces detected email addresses with `[REDACTED EMAIL]`. + +4. **Return Processed Output**: + - The sanitized output is returned, ensuring that sensitive information like email addresses is redacted before reaching the LLM. + +--- + +#### **Use Case for Middleware** + +**Tool Example**: +Imagine a tool, such as `Support Ticket Viewer`, which retrieves user tickets from a system. These tickets often contain email addresses. Middleware ensures that no sensitive email information is included in the output sent to the LLM. + +- **Input Payload Example**: + ```text + User email: john.doe@example.com has reported an issue with their account. + ``` + +- **Sanitized Output**: + ```text + User email: [REDACTED EMAIL] has reported an issue with their account. + ``` + +--- + +#### **Benefits of Middleware** + +1. **Data Privacy**: + - Protects sensitive user information by ensuring it is sanitized before being sent to external systems. + +2. **Compliance**: + - Ensures organizational adherence to privacy laws like GDPR or HIPAA. + +3. **Enhanced Security**: + - Prevents accidental sharing of PII with external vendors or LLMs. + +--- + +This middleware example demonstrates how flexible and powerful Tyk's scripting capabilities are, enabling administrators to enforce strict data governance policies while supporting advanced LLM and tool integration workflows. diff --git a/ai-management/ai-studio/llm-management.mdx b/ai-management/ai-studio/llm-management.mdx new file mode 100644 index 00000000..82e1de44 --- /dev/null +++ b/ai-management/ai-studio/llm-management.mdx @@ -0,0 +1,89 @@ +--- +title: "LLM Management" +'og:description': "How to manage LLMs in Tyk AI Studio?" +tags: ['AI Studio', 'AI Management', 'LLM Management'] +sidebarTitle: "LLM Management" +--- + +Tyk AI Studio provides a centralized system for managing Large Language Model (LLM) providers, models, associated costs, and usage budgets. This allows administrators to control which models are available, how they are used, and track associated expenses. + +## Overview + +The LLM Management system allows you to: + +* **Configure LLM Providers:** Connect to various LLM vendors (OpenAI, Anthropic, Azure OpenAI, Google Vertex AI, etc.). +* **Manage Models:** Specify which models from a provider are available for use within Tyk AI Studio. +* **Define Pricing:** Set input and output token costs for each model to enable accurate cost tracking. +* **Set Budgets:** Establish monthly spending limits for LLM usage, either globally for a model or per Application. +* **Control Access:** Determine which teams can access specific LLM configurations (via associated Apps). + +## Configuring LLM Providers + +Administrators can configure connections to different LLM providers through the UI or API. + +1. **Navigate:** Go to the LLM Configuration section in the Admin UI. +2. **Add New LLM:** Click "Add LLM Configuration". +3. **Provider Details:** + * **Name:** A user-friendly name for this configuration (e.g., "OpenAI GPT-4 Turbo"). + * **Vendor:** Select the LLM vendor (e.g., `openai`, `anthropic`, `azure`, `vertex`). + * **API Key/Credentials:** Securely provide the necessary authentication credentials. Use the **Secrets Management** system (`$SECRET/YourSecretName`) for best practice. + * **Base URL (Optional):** Override the default API endpoint if needed (e.g., for Azure OpenAI). + * **API Version (Optional):** Specify the API version for certain providers like Azure. + + LLM Provider Config + +4. **Model Selection:** + * **Allowed Models:** Specify the exact model names from the vendor that can be used via this configuration (e.g., `gpt-4-turbo`, `claude-3-opus-20240229`). + * **Default Model:** The model used if a request doesn't specify one. + +5. **Route ID:** A unique identifier used in API paths (e.g., `/proxy/{routeId}/...` or `/openai/{routeId}/...`) to target this specific LLM configuration. + +6. **Privacy Level:** Assign a privacy level to the LLM. This interacts with the Tool system, preventing tools with higher privacy levels from being used with LLMs having lower levels. + + Privacy levels define how data is protected by controlling LLM access based on its sensitivity: + - Public – Safe to share (e.g., blogs, press releases). + - Internal – Company-only info (e.g., reports, policies). + - Confidential – Sensitive business data (e.g., financials, strategies). + - Restricted (PII) – Personal data (e.g., names, emails, customer info). + +7. **Save:** Save the configuration. + +## Model Pricing + +To enable cost tracking in the Analytics system, you need to define the price per token for each model. + +1. **Navigate:** Go to the Model Prices section in the Admin UI. +2. **Add Price:** Define prices for specific models. + * **Vendor:** Select the vendor. + * **Model Name:** Enter the exact model name. + * **Input Token Price:** Cost per input token (usually stored as integer * 10000 for precision). + * **Output Token Price:** Cost per output token (usually stored as integer * 10000 for precision). + + Model Price Config + +3. **Save:** Save the pricing information. + +The Analytics system uses these prices along with token counts from LLM interactions (recorded by the Proxy and Chat systems) to calculate usage costs. + +## Budget Control + +Tyk AI Studio allows setting monthly spending limits to control AI costs. + +* **LLM Budget:** A global monthly budget can be set directly on an LLM configuration. This limits the total spending across *all* applications using that specific LLM configuration. +* **Application Budget:** A monthly budget can be set on an Application (`Apps` section). This limits the spending *for that specific application*, potentially across multiple LLM configurations it might use. + +**How it Works:** + +1. Budgets are checked *before* an LLM request is forwarded by the Proxy. +2. The system calculates the current monthly spending for the relevant entity (LLM or App) based on data from the Analytics system. +3. If the current spending plus the estimated cost of the *incoming* request (if calculable, otherwise based on past usage) exceeds the budget, the request is blocked (e.g., 429 Too Many Requests). +4. The **Notification System** can be configured to send alerts when budget thresholds (e.g., 80%, 100%) are reached. + +**Configuration:** + +* **LLM Budget:** Set the `MonthlyBudget` field when creating/editing an LLM configuration. +* **App Budget:** Set the `MonthlyBudget` field when creating/editing an App configuration. + + Budget Config + +By combining LLM configuration, pricing, and budgeting, administrators gain granular control over AI model access and expenditure within Tyk AI Studio. diff --git a/ai-management/ai-studio/llms.mdx b/ai-management/ai-studio/llms.mdx new file mode 100644 index 00000000..97f56afe --- /dev/null +++ b/ai-management/ai-studio/llms.mdx @@ -0,0 +1,132 @@ +--- +title: "Large Language Models (LLMs)" +'og:description': "How to configure LLMs in AI Studio?" +tags: ['AI Studio', 'AI Management', 'LLMs', 'Large Language Models'] +--- + +The **LLMs View** provides administrators with an overview of the Large Language Model (LLM) vendors integrated into the portal. This section allows for managing the LLMs available for use in Chat Room features and the AI Gateway. Below is a breakdown of the features and data displayed: + +--- + +#### **Table Overview** +The table displays the following columns: + +1. **Name**: + The name of the LLM vendor or model (e.g., Anthropic, OpenAI GPT-4o, VLLM). + +2. **Short Description**: + A brief overview of the LLM, highlighting its features, strengths, and recommended use cases. + - Example: "Anthropic's flagship LLM: Claude is known for its excellent support for code-related tasks and code generation." + +3. **Vendor**: + The name of the LLM vendor providing the model (e.g., Anthropic, OpenAI). + +4. **Privacy Level**: + Indicates the model's privacy capability. Privacy levels define how data is protected by controlling LLM access based on its sensitivity. LLM providers with lower privacy levels can't access higher-level data sources and tools, ensuring secure and appropriate data handling. + + The system works with 4 privacy levels from low to high: + - Public – Safe to share (e.g., blogs, press releases). + - Internal – Company-only info (e.g., reports, policies). + - Confidential – Sensitive business data (e.g., financials, strategies). + - Restricted (PII) – Personal data (e.g., names, emails, customer info). + +5. **Proxied**: + A status indicator showing whether the LLM is proxied through the AI Gateway for added security and control. + - **Green dot**: Proxied. + - **Red dot**: Not proxied. + +6. **Actions**: + A menu (three-dot icon) that allows administrators to perform specific actions on the LLM, such as editing its details, managing configurations, or removing it from the portal. + +--- + +#### **Features** +1. **Add LLM Button**: + A green button in the top-right corner labeled **+ ADD LLM**. Clicking this button opens a form to integrate a new LLM vendor or model into the portal. + +2. **Pagination Dropdown**: + Found at the bottom-left corner, this dropdown allows users to control how many LLMs are displayed per page. + +--- + +#### **Use Cases** +- **Chat Room Features**: + LLMs integrated into the Chat Room provide users with easy access to conversational AI for various purposes, such as general inquiries or task automation. + +- **AI Gateway**: + LLMs configured for the AI Gateway are used to route requests securely, offering fine-grained access control and privacy protections for API calls. + +--- + +### Edit/Create LLM Vendor View + +The **Edit/Create LLM Vendor View** allows administrators to configure or update the details of a Large Language Model (LLM) vendor. This form supports adding new LLMs or modifying existing ones for use in the Chat Room features and AI Gateway. Below is a detailed breakdown of the form fields and their functionality: + +--- + +#### **Form Sections and Fields** + +##### **LLM Description** +1. **Name** *(Required)*: + The name of the LLM vendor (e.g., "Anthropic"). This field identifies the LLM within the portal. + +2. **Short Description** *(Optional)*: + A brief summary of the LLM's capabilities, highlighting its key features (e.g., "Claude is known for its excellent support for code-related tasks and code generation."). + +3. **Long Description** *(Optional)*: + A detailed explanation of the LLM, including its history, technical details, and specific use cases. This provides deeper insight into the model for administrators and users. + +4. **Vendor** *(Dropdown)*: + The name of the vendor offering the LLM. This dropdown lists pre-configured vendors (e.g., Anthropic, OpenAI). + +5. **Default Model** *(Required)*: + The specific model to use by default for this LLM (e.g., "claude-3.5-sonnet-20240620"). Administrators can specify the exact model version. + +6. **Privacy Level** *(Optional)*: + The privacy capability of the LLM. Select from four levels (Public, Internal, Confidential, Restricted) to indicate what sensitivity of data the LLM can handle. + +--- + +##### **Access Details** +This section defines the API-related details required for integrating the LLM. + +1. **API Endpoint** *(Required for Gateway Integration)*: + The URL used to send requests to the LLM. This is necessary for enabling the LLM in the AI Gateway. + +2. **API Key** *(Optional)*: + A secure key for authenticating with the LLM's API. + - **View/Hide Toggle**: Allows administrators to toggle between showing and hiding the key for security purposes. + +--- + +##### **Portal Display Information** +Settings that determine how the LLM appears in the portal for end-users and developers. + +1. **Logo URL** *(Optional)*: + A link to an image that represents the LLM vendor. This logo is displayed in the portal's user interface. + +2. **Enabled in Proxy** *(Toggle)*: + Determines whether the LLM is proxied through the AI Gateway. + - **Enabled**: Routes traffic securely through the gateway. + - **Disabled**: Direct access to the LLM's API. + +--- + +##### **Filters** +- **Filters** *(Optional)*: + A field to add or manage filters that are executed in the AI Gateway when a request flows through the REST endpoint. + - Example: Adding preprocessing steps to modify user queries before they reach the LLM. + +--- + +#### **Action Buttons** +1. **Update LLM / Create LLM**: + A button at the bottom of the form that saves the changes or creates the new LLM vendor. This button is active only when all required fields are completed. + +2. **Back to LLMs**: + A link at the top-right corner that navigates back to the LLMs List View without saving changes. + +--- + +#### **Purpose** +This view serves as a centralized interface for managing LLM integrations. It ensures administrators have the flexibility to configure details, enable secure access, and define how LLMs are presented and used within the portal. diff --git a/ai-management/ai-studio/model-prices.mdx b/ai-management/ai-studio/model-prices.mdx new file mode 100644 index 00000000..344d6fb8 --- /dev/null +++ b/ai-management/ai-studio/model-prices.mdx @@ -0,0 +1,109 @@ +--- +title: "Model Prices View for Tyk AI Studio" +'og:description': "How to manage model prices in AI Studio?" +tags: ['AI Studio', 'AI Management', 'Model Price'] +--- + +The **Model Prices View** allows administrators to manage and track the cost associated with Large Language Model (LLM) usage. This section is essential for monitoring expenses and setting pricing for LLM use within the AI Gateway and Chat Room features. Below is a detailed breakdown of the table and its features: + +--- + +#### **Table Overview** +The table displays the following columns: + +1. **Model Name**: + - The name of the LLM model for which pricing is defined (e.g., `claude-3.5-sonnet-20240620`, `gpt-4o`). + +2. **Vendor**: + - The organization providing the LLM (e.g., Anthropic, OpenAI). + +3. **Cost per Input Token**: + - The price charged for each input token processed by the LLM. + - Example: A value of `0.000001` represents the cost in the specified currency per token. + +4. **Cost per Output Token**: + - The price charged for each output token generated by the LLM. + - Example: A value of `0.000015` reflects the token generation cost in the specified currency. + +5. **Currency**: + - The currency in which the model's pricing is defined (e.g., USD). + +6. **Actions**: + - A menu (three-dot icon) with quick actions for each model, allowing administrators to: + - Edit the pricing. + - Delete the pricing configuration. + +--- + +#### **Features** + +1. **Add Model Price Button**: + - A green button labeled **+ ADD MODEL PRICE**, located in the top-right corner. Clicking this button opens a form to configure the pricing for a new LLM model. + +2. **Pagination Dropdown**: + - Located at the bottom-left corner, this control allows users to adjust how many pricing entries are displayed per page. + +--- + +#### **Use Cases** +- **AI Gateway Tracking**: + - Ensures that token-based costs for API calls through the AI Gateway are accurately logged and monitored. + +- **Chat Room Cost Analysis**: + - Tracks and evaluates expenses associated with user interactions in the Chat Room feature. + +--- + +#### **Quick Insights** +This section simplifies the process of tracking and updating LLM pricing, ensuring transparency and control over usage costs. The ability to edit or add prices directly from this interface provides flexibility for managing vendor costs dynamically as new models or pricing structures are introduced. + +### Edit/Create Model Prices + +The **Edit/Create Model Price Form** allows administrators to define or update the cost structure for a specific Large Language Model (LLM). The pricing information configured here is used for cost tracking and billing purposes and must align with the model names used in API calls or the model settings screen. + +--- + +#### **Form Fields and Descriptions** + +1. **Model Name** *(Required)*: + - The exact name of the model this price configuration applies to (e.g., `claude-3.5-sonnet-20240620`). + - **Important**: This name must match the name used in client API calls or the LLM settings in the portal for correct mapping. + +2. **Vendor** *(Dropdown)*: + - The name of the LLM provider (e.g., Anthropic, OpenAI). + - Selectable from pre-configured vendors in the portal. + +3. **Cost per Input Token** *(Decimal, Required)*: + - The price charged per input token sent to the LLM. + - Example: A value of `0.000001` indicates the cost per input token in the specified currency. + +4. **Cost per Output Token** *(Decimal, Required)*: + - The price charged per output token generated by the LLM. + - Example: A value of `0.000015` specifies the output token cost in the specified currency. + +5. **Currency** *(Text, Required)*: + - The currency in which the pricing is defined (e.g., USD). + - Example: Use "USD" for United States Dollar pricing. + +--- + +#### **Action Buttons** +1. **Update Model Price / Create Model Price**: + - A green button at the bottom of the form that saves the price configuration. It updates an existing price or creates a new one depending on the context. + +2. **Back to Model Prices**: + - A link in the top-right corner to return to the **Model Prices View** without saving changes. + +--- + +#### **Usage and Purpose** +- **Price Tracking**: + - Ensures token-level costs are recorded for API calls and Chat Room interactions. + - Provides transparency for usage billing. + +- **Integration Consistency**: + - Ensures that the configured pricing aligns with the exact model names used in client interactions or other system settings to avoid mismatches. + +--- + +This form is essential for cost management and aligns LLM usage with its associated financial metrics, providing a streamlined approach to managing expenses in the portal. diff --git a/ai-management/ai-studio/notifications.mdx b/ai-management/ai-studio/notifications.mdx new file mode 100644 index 00000000..b71be3a9 --- /dev/null +++ b/ai-management/ai-studio/notifications.mdx @@ -0,0 +1,66 @@ +--- +title: "Notification" +'og:description': "How to configure notifications in Tyk AI Studio?" +tags: ['AI Studio', 'AI Management', 'Notifications'] +sidebarTitle: "Notifications" +--- + +Tyk AI Studio includes a centralized Notification System responsible for generating and delivering alerts and messages to users and administrators based on specific system events. + +## Purpose + +The Notification System aims to: + +* **Inform Stakeholders:** Keep users and administrators aware of important events or required actions. +* **Enable Proactive Management:** Alert administrators to potential issues or thresholds being reached (e.g., budget limits). +* **Improve User Experience:** Provide timely feedback on asynchronous processes or user-related events. + +## Key Features + +* **Event-Driven:** Notifications are triggered by specific occurrences within the Tyk AI Studio platform. +* **Configurable Channels:** Supports multiple delivery methods, primarily: + * **Email:** Sending notifications to registered user email addresses. + * **In-App Notifications:** Displaying messages directly within the Tyk AI Studio UI. +* **User Preferences:** Allows users (and potentially administrators) to configure which notifications they wish to receive and via which channels (where applicable). +* **Centralized Logic:** Provides a single system for managing notification templates and delivery rules. + +## Common Notification Triggers + +Examples of events that might trigger notifications include: + +* **[Budget Control](/ai-management/ai-studio/llm-management):** + * Approaching spending limit threshold (e.g., 80% of budget). + * Reaching or exceeding spending limit. +* **[User Management](/ai-management/ai-studio/user-management):** + * New user registration/invitation. + * Password reset request. + * Changes in user roles or group memberships. +* **System Health & Errors:** + * Significant system errors or failures. + * Service degradation alerts. +* **Security Events:** + * Suspicious login activity (if monitored). + * Changes to critical security settings. + +## Configuration + +* **System-Level (Admin):** Administrators typically configure the core settings for the notification system, such as: + * Email server (SMTP) details for sending emails. + * Default notification templates. + * Enabling/disabling specific system-wide notification types. +* **User-Level:** Users can often manage their notification preferences in their profile settings: + * Opt-in/opt-out of specific notification categories. + * Choose preferred delivery channels (e.g., receive budget alerts via email). + + Notification Prefs UI + +## Integration + +The Notification System integrates with various other Tyk AI Studio components that generate relevant events, including: + +* Budget Control System +* User Management System +* Analytics System (potentially for performance alerts) +* Proxy/Gateway (for error or security event alerts) + +This system ensures timely communication, helping users and administrators stay informed about the status and activity within the Tyk AI Studio platform. diff --git a/ai-management/ai-studio/overview.mdx b/ai-management/ai-studio/overview.mdx new file mode 100644 index 00000000..5f827921 --- /dev/null +++ b/ai-management/ai-studio/overview.mdx @@ -0,0 +1,113 @@ +--- +title: "AI Studio" +'og:description': "AI Management for Platform Teams with Tyk AI Studio, a comprehensive platform for managing and deploying AI LLMs and chats" +tags: ['AI Management', 'Platform Teams', 'Tyk AI Studio'] +sidebarTitle: "Overview" +--- + +import AIStudioCards from '/snippets/AIStudioCards.mdx'; +import { ButtonLeft } from '/snippets/ButtonLeft.mdx'; + +Tyk AI Studio is a comprehensive platform that enables platform teams to manage and deploy AI applications with enterprise-grade governance, security, and control. + +## Prerequisites + +Before getting started with Tyk AI Studio, you need to obtain a license from Tyk Technologies. Contact support@tyk.io or your account manager to request your AI Studio license. + +## Key Features + + + +Tyk AI Studio enables platform teams to: +- Centralise access credentials to AI vendors, including commercial and in-house offerings +- Log and measure AI usage across the organization +- Build and release Chatbots for internal collaboration +- Ingest data into a knowledge corpus with siloed access settings +- Create AI functions as a service for common automations +- Script complex multi-modal assistants that intelligently select which AI vendor, model, and data set to use +- Implement role-based access control to democratise LLM access while maintaining security + +## Enterprise AI Challenges + +Organizations implementing AI face several key challenges: + +* **Shadow AI**: Unauthorized tools running without governance or oversight +* **Data privacy and compliance**: Meeting regulatory requirements while enabling innovation +* **Security and access control**: Implementing proper authentication and authorization +* **Cost management**: Controlling expenses from unmonitored AI usage + +## Integrate AI with Confidence + +Tyk AI Studio helps organizations harness AI's potential while ensuring proper governance, security, compliance, and control. Purpose-built for enterprises, this AI gateway and management solution enables seamless governance that overcomes the risks and challenges of AI adoption. + + + +## Solution Components + +Tyk AI Studio provides a comprehensive suite of capabilities to manage, govern, and interact with AI across your organization: + +### Centralized AI management + +Unify and control AI usage across your organization: +- Govern AI with role-based access control, rate limiting and audit logging +- Monitor usage, costs, budgets, and performance in real time +- Manage how LLMs are accessed and used, with an AI gateway as a single point of control +- Ensure compliance with global privacy regulations through customizable data flow management + +### AI Gateway + +Seamlessly connect to AI tools and models: +- Proxy to large language models (LLMs) and integrate custom data models and tools +- Use the [AI Gateway](/ai-management/ai-studio/proxy) to enable secure, scalable access to AI services across teams +- Track usage statistics, cost breakdowns, and tool utilization to optimize resources + +### AI Portal + +Empower developers with a curated AI service catalog: +- Simplify access to AI tools and services through a unified portal +- Enable seamless integration with internal systems and external workflows +- Accelerate innovation by providing developers with the tools they need to build faster + +### AI Chat + +Bring AI-powered collaboration to every user: +- Deliver intuitive chat interfaces for direct interaction with AI tools and data sources +- Enable teams to access AI-driven insights through a unified, secure chat experience +- Foster collaboration and innovation across your organization + +## Benefits + +Tyk AI Studio empowers organizations to adopt AI securely and efficiently, delivering: + +- **Centralized governance and control:** Consistency at the core of your business for enhanced security, compliance, troubleshooting and auditing +- **Strengthened security:** Peace of mind from strict access controls, secure interactions and region-specific compliance +- **Simplified workflows:** Reduced complexity and enhanced efficiency supporting developers and less technical users to work with multiple LLMs and tools +- **Trusted data privacy:** Rigorous compliance with data protection standards, reducing risk of reputational, operational and financial damage +- **Seamless integration:** Enhanced workflows in customer support, development, and marketing with trusted AI tools +- **Cost optimization:** Control over expenses and accountability, enabling smarter budgets + +## Use Cases + +Proxying LLM traffic through the AI Gateway delivers control, visibility, and scalability across various scenarios: + +- **Interact with your APIs:** Connect your API management to enable API interaction for wider teams +- **Banking and financial services:** Ensure only anonymized customer data is sent to LLMs, tracking usage by department to manage costs +- **Software development:** Leverage AI for code suggestions and issue tracking in Jira +- **Data governance:** Audit and secure AI interactions to meet regulatory standards +- **Healthcare:** Route LLM traffic through an AI gateway to comply with HIPAA, protecting patient data while enabling AI-driven insights +- **E-commerce:** Integrate LLMs with product catalogs, allowing employees to query inventory or sales data through a chat interface + +## MCP servers in AI Studio + +AI Studio provides comprehensive [MCP (Model Context Protocol) capabilities](/ai-management/mcps/overview#mcp-for-enterprise-use) including: + +- **Remote MCP catalogues and server support** – Expose internal APIs and tools to AI assistants securely without requiring local installations +- **Secure local MCP server deployment** – Deploy MCP servers within controlled environments, integrated with Tyk AI Gateway for monitoring and governance +- **Ready-to-use MCP integrations** – Including API to MCP conversion, Dashboard API access, and searchable documentation access + +For more details about Model Context Protocol (MCP) integration, please visit the [Tyk MCPs overview](/ai-management/mcps/overview) page. + +
+ + + diff --git a/ai-management/ai-studio/proxy.mdx b/ai-management/ai-studio/proxy.mdx new file mode 100644 index 00000000..60831ea4 --- /dev/null +++ b/ai-management/ai-studio/proxy.mdx @@ -0,0 +1,71 @@ +--- +title: "Proxy & API Gateway" +'og:description': "How AI Gateway works?" +tags: ['AI Studio', 'AI Management', 'AI Gateway', 'AI Proxy'] +sidebarTitle: "AI Gateway" +--- + +The Tyk AI Studio Proxy is the central gateway for all Large Language Model (LLM) interactions within the platform. It acts as a secure, observable, and policy-driven entry point, managing requests from client applications to the configured backend LLM services. + +## Purpose + +The Proxy serves several critical functions: + +* **Unified Access Point:** Provides a single, consistent endpoint for applications to interact with various LLMs. +* **Security Enforcement:** Handles authentication, authorization, and applies security policies. +* **Policy Management:** Enforces rules related to budget limits, model access, and applies custom [Filters](/ai-management/ai-studio/filters). +* **Observability:** Logs detailed analytics data for each request, feeding the [Analytics & Monitoring](/ai-management/ai-studio/analytics) system. +* **Vendor Abstraction:** Hides the complexities of different LLM provider APIs, especially through the OpenAI-compatible endpoint. + +## Core Functions + +1. **Request Routing:** Incoming requests include a `routeId` in their path (e.g., `/proxy/{routeId}/...` or `/openai/{routeId}/...`). The Proxy uses this `routeId` to identify the target [LLM Configuration](/ai-management/ai-studio/llm-management) and route the request accordingly. + +2. **Authentication & Authorization:** + * Validates the API key provided by the client application. + * Identifies the associated Application and User. + * Checks if the Application/User group has permission to access the requested LLM Configuration based on [RBAC rules](/ai-management/ai-studio/user-management). + +3. **Policy Enforcement:** Before forwarding the request to the backend LLM, the Proxy enforces policies defined in the LLM Configuration or globally: + * **Budget Checks:** Verifies if the estimated cost exceeds the configured [Budgets](/ai-management/ai-studio/llm-management) for the App or LLM. + * **Model Access:** Ensures the requested model is allowed for the specific LLM configuration. + * **Filters:** Applies configured request [Filters](/ai-management/ai-studio/filters) to modify the incoming request payload. + +4. **Analytics Logging:** After receiving the response from the backend LLM (and potentially applying response Filters), the Proxy logs detailed information about the interaction (user, app, model, tokens used, cost, latency, etc.) to the [Analytics](/ai-management/ai-studio/analytics) database. + +## Endpoints + +Tyk AI Studio typically exposes two primary types of proxy endpoints: + +### 1. OpenAI-Compatible Endpoint (`/openai/{routeId}/v1/...`) + +* **Purpose:** This endpoint mimics the official OpenAI API structure. It allows developers to use standard OpenAI SDKs (Python, Node.js, etc.) to interact with *any* LLM configured in Tyk AI Studio, regardless of the actual backend vendor (Anthropic, Google Vertex AI, etc.). +* **Translation:** Tyk AI Studio includes a translation layer (using libraries like `langchaingo`) that converts standard OpenAI API requests into the format required by the target backend LLM (defined in the `{routeId}` configuration) and translates the backend LLM's response back into the standard OpenAI format. +* **Benefits:** Simplifies integration significantly, allowing developers to write code once and target multiple LLM backends managed by Tyk AI Studio. + + ```bash + # Example using OpenAI Python SDK + import openai + + client = openai.OpenAI( + base_url="https://your-ai-studio-host/openai/my-anthropic-config/v1", + api_key="YOUR_AI_STUDIO_APP_API_KEY" + ) + + response = client.chat.completions.create( + model="claude-3-opus-20240229", # Model allowed in 'my-anthropic-config' + messages=[{"role": "user", "content": "Hello!"}] + ) + print(response.choices[0].message.content) + ``` + +### 2. Direct Proxy Endpoint (`/proxy/{routeId}/...`) + +* **Purpose:** Provides a more direct pass-through to the backend LLM, potentially with less translation than the OpenAI-compatible endpoint. The exact request/response format expected at this endpoint might depend more heavily on the specific backend LLM vendor configured for the `{routeId}`. +* **Usage:** Might be used for accessing vendor-specific features not covered by the OpenAI API standard or in scenarios where the OpenAI translation layer is not desired. + +## Configuration & Security + +The behavior of the Proxy for a specific route is determined by the corresponding [LLM Configuration](/ai-management/ai-studio/llm-management), which includes details about the backend vendor, model access, budget limits, and associated filters. + +By centralizing LLM access through the Proxy, Tyk AI Studio provides a robust layer for security, control, and observability over AI interactions. diff --git a/ai-management/ai-studio/quickstart.mdx b/ai-management/ai-studio/quickstart.mdx new file mode 100644 index 00000000..b00fac53 --- /dev/null +++ b/ai-management/ai-studio/quickstart.mdx @@ -0,0 +1,210 @@ +--- +title: "Quickstart" +'og:description': "AI Studio Quickstart" +tags: ['AI Studio', 'AI Management', 'Quickstart'] +sidebarTitle: "Quickstart" +--- + +This guide will help you get started with the Tyk AI Studio using Docker Compose or package installation. + +## Prerequisites + +### License Requirement +- A valid Tyk AI Studio license from Tyk Technologies. Contact support@tyk.io or your account manager to obtain your license. + +### For Docker Compose Installation +- Docker and Docker Compose installed on your system +- PostgreSQL database (recommended for production) - if not provided, SQLite will be used as fallback + +### For Package Installation +- Linux system with systemd +- PostgreSQL database (strongly recommended) - if not configured, SQLite will be used as fallback + +## Installation Methods + +### Method 1: Docker Compose Installation + +1. Create a new directory for your project: + ```bash expandable + mkdir tyk-ai-studio && cd tyk-ai-studio + ``` + +2. Create a `compose` directory and add the following Docker Compose file: + ```bash + mkdir compose && cd compose + ``` + +3. Create a `compose.yaml` file with the following content: + ```yaml + version: "3" + services: + ai-studio: + image: tykio/tyk-ai-studio:latest + volumes: + - ./confs/.env:/app/.env + environment: + - DATABASE_URL=postgres://postgres:postgres@postgres:5432/postgres + - DATABASE_TYPE=postgres + depends_on: + postgres: + condition: service_healthy + ports: + - 8080:8080 # Main application port + - 9090:9090 # Gateway server port + + postgres: + image: postgres:latest + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=postgres + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 5s + timeout: 5s + retries: 5 + ``` + +4. Create a configuration directory and environment file: + ```bash + mkdir -p confs + touch confs/.env + ``` + +5. Add your configuration to the `.env` file (example): + + **For PostgreSQL (recommended):** + ```env + ALLOW_REGISTRATIONS=true + ADMIN_EMAIL=you@tyk.io + SITE_URL=http://localhost:8080 + FROM_EMAIL=noreply@tyk.io + DATABASE_URL=postgres://postgres:postgres@postgres:5432/postgres + DATABASE_TYPE=postgres + TYK_AI_SECRET_KEY=a35b3f7b0fb4dd3a048ba4fc6e9fe0a8cb804d7884c62b6b2ea09c99612c4405 + FILTER_SIGNUP_DOMAINS=tyk.io + TYK_AI_LICENSE=XXXX + # Optional SMTP settings + # SMTP_SERVER=smtp.sendgrid.net + # SMTP_PORT=587 + # SMTP_USER=apikey + # SMTP_PASS= + ``` + + **For SQLite (development only):** + ```env + ALLOW_REGISTRATIONS=true + ADMIN_EMAIL=you@tyk.io + SITE_URL=http://localhost:8080 + FROM_EMAIL=noreply@tyk.io + DATABASE_URL=tyk-ai-studio.db + DATABASE_TYPE=sqlite + TYK_AI_SECRET_KEY=a35b3f7b0fb4dd3a048ba4fc6e9fe0a8cb804d7884c62b6b2ea09c99612c4405 + FILTER_SIGNUP_DOMAINS=tyk.io + TYK_AI_LICENSE=XXXX + ``` + + > **Note:** PostgreSQL is strongly recommended for production use. SQLite is only suitable for development and testing. + +#### Starting the Service + +1. Start the services using Docker Compose: + ```bash + docker compose up -d + ``` + +2. Verify that the services are running: + ```bash + docker compose ps + ``` + +#### Accessing the Portal + +Once the services are running: + +- Access the AI Portal interface at: `http://localhost:8080` +- Access the Gateway at: `http://localhost:9090` + +#### Monitoring Logs + +To view the logs from the services: +```bash +docker compose logs -f +``` + +#### Stopping the Service + +To stop and remove the containers: +```bash +docker compose down +``` + +### Method 2: Package Installation + +1. Add the Tyk package repository: + ```bash + # For Ubuntu/Debian systems + curl -fsSL https://packagecloud.io/tyk/tyk-ee/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/tyk-ee-archive-keyring.gpg + echo "deb [signed-by=/usr/share/keyrings/tyk-ee-archive-keyring.gpg] https://packagecloud.io/tyk/tyk-ee/ubuntu/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/tyk-ee.list + + # For RHEL/CentOS systems + curl -s https://packagecloud.io/install/repositories/tyk/tyk-ee/script.rpm.sh | sudo bash + ``` + +2. Install the package: + ```bash + # For Ubuntu/Debian + sudo apt update + sudo apt install tyk-ai-studio + + # For RHEL/CentOS + sudo yum install tyk-ai-studio + ``` + +3. Configure the application: + ```bash + sudo nano /etc/tyk-ai-studio/.env + ``` + + Add your configuration (similar to Docker Compose example above). Ensure you configure PostgreSQL for production: + ```env + DATABASE_URL=postgres://username:password@localhost:5432/tyk_ai_studio + DATABASE_TYPE=postgres + TYK_AI_LICENSE=your-license-key-here + # ... other configuration options + ``` + + > **Note:** The `TYK_AI_LICENSE` environment variable is required for the service to start. Contact support@tyk.io or your account manager if you need to obtain a license. + +4. Start the service: + ```bash + sudo systemctl enable tyk-ai-studio + sudo systemctl start tyk-ai-studio + ``` + +5. Check service status: + ```bash + sudo systemctl status tyk-ai-studio + ``` + +## Service Components + +The Docker Compose setup includes: + +- **Tyk AI Studio Service**: The main AI Portal application + - Runs on ports 8080 (web interface) and 9090 (gateway server) + - Connects to PostgreSQL for data storage + - Uses environment variables for configuration + +- **PostgreSQL Database**: + - Stores application data + - Uses default credentials (configurable via environment variables) + +## Troubleshooting + +If you encounter issues: + +1. Check that all required ports (8080, 9090) are available +2. Ensure your `.env` file contains valid API keys +3. Verify that Docker and Docker Compose are properly installed +4. Check the logs for any error messages: `docker compose logs -f` \ No newline at end of file diff --git a/ai-management/ai-studio/secrets.mdx b/ai-management/ai-studio/secrets.mdx new file mode 100644 index 00000000..3f0ae0d2 --- /dev/null +++ b/ai-management/ai-studio/secrets.mdx @@ -0,0 +1,67 @@ +--- +title: "Secrets Management" +'og:description': "How to configure secret management in Tyk AI Studio?" +tags: ['AI Studio', 'AI Management', 'Secret Management'] +sidebarTitle: "Secrets Management" +--- + +Tyk AI Studio provides a secure mechanism for storing and managing sensitive information, such as API keys, passwords, authentication tokens, or other credentials required by various platform configurations. + +## Purpose + +The Secrets Management system aims to: + +* **Prevent Exposure:** Avoid hardcoding sensitive values directly in configuration files or UI fields. +* **Centralize Management:** Provide a single place to manage and update credentials. +* **Enhance Security:** Store sensitive data encrypted at rest. + +## Core Concepts + +* **Secret:** A named key-value pair where the 'key' is the **Variable Name** used for reference, and the 'value' is the actual sensitive data (e.g., an API key string). + +* **Encryption:** + * Secret values are **encrypted at rest** within Tyk AI Studio's storage. + * The encryption and decryption process relies on a key derived from the **`TYK_AI_SECRET_KEY` environment variable** set when running the Tyk AI Studio instance. + * **CRITICAL:** The `TYK_AI_SECRET_KEY` must be kept confidential and managed securely. Loss of this key will render existing secrets unusable. Changing the key will require re-entering all secrets. + +* **Reference Syntax:** Secrets are referenced within configuration fields (like API key fields in LLM or Tool setups) using a specific syntax: + ``` + $SECRET/VariableName + ``` + Replace `VariableName` with the exact name given to the secret when it was created (e.g., `$SECRET/OPENAI_API_KEY`, `$SECRET/JIRA_AUTH_TOKEN`). + +* **Runtime Resolution:** When a configuration uses a field containing a secret reference (e.g., `$SECRET/MY_KEY`): + 1. The configuration itself stores the `$SECRET/MY_KEY` string, *not* the actual secret value. + 2. Only when the system needs the actual value at runtime (e.g., the [Proxy](/ai-management/ai-studio/proxy) preparing a request for an LLM, or a [Tool](/ai-management/ai-studio/tools) calling its external API), Tyk AI Studio retrieves the encrypted secret value, decrypts it using the `TYK_AI_SECRET_KEY`, and injects the plain text value into the operation. + 3. The decrypted value is typically used immediately and not persisted further. + +## Creating & Managing Secrets (Admin) + +Administrators manage secrets via the Tyk AI Studio UI or API: + +1. Navigate to the Secrets management section. +2. Create a new secret by providing: + * **Variable Name:** A unique identifier (letters, numbers, underscores) used in the `$SECRET/VariableName` reference. + * **Secret Value:** The actual sensitive string (e.g., `sk-abc123xyz...`). +3. Save the secret. It is immediately encrypted and stored. + + Secrets UI + +Secrets can be updated or deleted as needed. Updating a secret value will automatically apply the new value wherever the `$SECRET/VariableName` reference is used, without needing to modify the configurations themselves. + +## Usage Examples + +Secrets are commonly used in: + +* **[LLM Configurations](/ai-management/ai-studio/llm-management):** Storing API keys for providers like OpenAI, Anthropic, Google Vertex AI, etc. + * *Example:* In the API Key field for an OpenAI configuration: `$SECRET/OPENAI_API_KEY` +* **[Tool Configurations](/ai-management/ai-studio/tools):** Storing API keys, authentication tokens (Bearer, Basic Auth), or other credentials needed to interact with the external API the tool represents. + * *Example:* In a field for a custom header for a JIRA tool: `Authorization: Basic $SECRET/JIRA_BASIC_AUTH_TOKEN` +* **[Data Source Configurations](/ai-management/ai-studio/datasources-rag):** Storing API keys or connection credentials for vector databases (e.g., Pinecone, Milvus) or embedding service providers. + * *Example:* In the API Key field for a Pinecone vector store: `$SECRET/PINECONE_API_KEY` + +## Security Considerations + +* **Protect `TYK_AI_SECRET_KEY`:** This is the master key for secrets. Treat it with the same level of security as database passwords or root credentials. Use environment variable management best practices. +* **Principle of Least Privilege:** Grant administrative access (which includes secrets management) only to trusted users. +* **Regular Rotation:** Consider policies for regularly rotating sensitive credentials by updating the Secret Value in Tyk AI Studio. diff --git a/ai-management/ai-studio/sso.mdx b/ai-management/ai-studio/sso.mdx new file mode 100644 index 00000000..6e1e4e07 --- /dev/null +++ b/ai-management/ai-studio/sso.mdx @@ -0,0 +1,71 @@ +--- +title: "SSO Integration" +'og:description': "How to configure SSO in Tyk AI Studio?" +tags: ['AI Studio', 'AI Management', 'Single Sign On'] +sidebarTitle: "SSO Integration" +--- + +Tyk AI Studio supports Single Sign-On (SSO) integration, allowing users to authenticate using their existing credentials from external Identity Providers (IdPs). This simplifies login, enhances security, and centralizes user management. + +## Purpose + +The SSO integration aims to: + +* Allow users to log in to Tyk AI Studio using their familiar corporate or social identity credentials. +* Eliminate the need for separate Tyk AI Studio-specific passwords. +* Improve security by leveraging the organization's existing IdP infrastructure and policies (e.g., MFA). +* Streamline user provisioning and de-provisioning (depending on IdP capabilities and configuration). + + + + +## Technology: Tyk Identity Broker (TIB) + +Tyk AI Studio leverages the embedded **Tyk Identity Broker (TIB)** component to handle SSO integrations. TIB acts as a bridge between Tyk AI Studio (the Service Provider or SP) and various external Identity Providers (IdPs). + +## Supported Protocols & Providers + +TIB enables Tyk AI Studio to integrate with IdPs supporting standard protocols, including: + +* **OpenID Connect (OIDC):** Commonly used by providers like Google, Microsoft Entra ID (Azure AD), Okta, Auth0. +* **SAML 2.0:** Widely used in enterprise environments (e.g., Okta, Ping Identity, ADFS). +* **LDAP:** For integration with traditional directory services like Active Directory. +* **Social Logins:** Providers like GitHub, GitLab, etc. (often via OIDC). + +## Configuration (Admin) + +Administrators configure SSO providers within the Tyk AI Studio administration interface (likely via TIB's configuration settings exposed through Tyk AI Studio): + +1. **Select Protocol:** Choose the appropriate protocol (OIDC, SAML, etc.). +2. **Provider Details:** Enter the specific configuration details required by the chosen protocol and IdP. + * **OIDC Example:** Client ID, Client Secret, Issuer URL, Discovery Endpoint. + * **SAML Example:** IdP SSO URL, IdP Issuer/Entity ID, IdP Public Certificate, SP Entity ID (Tyk AI Studio's identifier). +3. **Profile Mapping:** Configure how attributes received from the IdP (e.g., email, name, group memberships) map to Tyk AI Studio user profiles. + * Identify which IdP attribute contains the unique user identifier (e.g., `email`, `sub`, `preferred_username`). + * Map IdP attributes to Tyk AI Studio user fields (e.g., `given_name` -> First Name, `family_name` -> Last Name). +4. **Group Mapping (Optional but Recommended):** Configure rules to automatically assign users to Tyk AI Studio [Groups](/ai-management/ai-studio/user-management) based on group information received from the IdP. + * *Example:* If the IdP sends a `groups` claim containing "Tyk AI Studio Admins", map this to automatically add the user to the "Administrators" group in Tyk AI Studio. +5. **Enable Provider:** Activate the configured IdP for user login. + + SSO Config UI + +## Login Flow + +When SSO is enabled: + +1. User navigates to the Tyk AI Studio login page. +2. User clicks a button like "Login with [Your IdP Name]" (e.g., "Login with Google", "Login with Okta"). +3. User is redirected to the external IdP's login page. +4. User authenticates with the IdP (using their corporate password, MFA, etc.). +5. Upon successful authentication, the IdP redirects the user back to Tyk AI Studio (via TIB) with an authentication assertion (e.g., OIDC ID token, SAML response). +6. TIB validates the assertion and extracts user profile information. +7. Tyk AI Studio finds an existing user matching the unique identifier or provisions a new user account based on the received profile information (Just-In-Time Provisioning). +8. Group memberships may be updated based on configured mapping rules. +9. The user is logged into Tyk AI Studio. + +## Benefits + +* **Improved User Experience:** One less password to remember. +* **Enhanced Security:** Leverages established IdP security policies. +* **Centralized Control:** User access can often be managed centrally via the IdP. +* **Simplified Onboarding/Offboarding:** User access to Tyk AI Studio can be tied to their status in the central IdP. diff --git a/ai-management/ai-studio/teams.mdx b/ai-management/ai-studio/teams.mdx new file mode 100644 index 00000000..2972cd21 --- /dev/null +++ b/ai-management/ai-studio/teams.mdx @@ -0,0 +1,125 @@ +--- +title: "Teams View for Tyk AI Studio" +'og:description': "Overview of teams in AI Studio?" +tags: ['AI Studio', 'AI Management', 'Teams'] +--- + +# Teams View for Tyk AI Studio + +The Teams View allows administrators to manage role-based access control by organizing users into teams. Teams define permissions and access levels across the portal, enabling streamlined user management. + +--- + +#### **Table Overview** +The teams are displayed in a tabular format with the following columns: + +1. **ID**: + A unique identifier assigned to each team for easy reference. + +2. **Name**: + The name of the team, describing its role or purpose (e.g., "Solutions Architects," "Customer Support"). + +3. **Actions**: + A menu (represented by three dots) allowing administrators to perform additional actions on a team, such as editing its details, managing permissions, or deleting it. + +--- + +#### **Features** +1. **Add Team Button**: + Located in the top-right corner of the view, this green button allows administrators to create a new team. Clicking the button opens a form to configure the team's name, permissions, and members. + +2. **Pagination Dropdown**: + Found at the bottom-left corner of the table, this dropdown allows administrators to select how many teams are displayed per page (e.g., 10, 20, or more teams). + +--- + +#### **Role-Based Access Control** +Each team represents a set of users with shared permissions. Teams help to: +- Grant or restrict access to specific features or sections of the portal. +- Streamline permission management by assigning roles at the team level instead of individually for each user. +- Enhance security by ensuring users only have access to the resources they need. + +--- + +The Teams View is a critical tool for managing access control efficiently, ensuring that users have the appropriate permissions based on their roles within the organization. + +### Teams Quick Actions in Tyk AI Studio + +The Teams View includes a set of quick actions accessible via the **Actions** menu (three-dot icon) for each team. These actions allow administrators to make modifications on the fly without navigating to separate pages. + +--- + +#### **Quick Actions Overview** + +1. **Add Catalogue to Team**: + Associates a general catalogue of resources with the team. Catalogues are bundles of LLMs, tools, and data sources that the team can access. + +2. **Add Data Catalogue to Team**: + Specifically links a data catalogue to the team. This grants access to specific datasets and data resources. + +3. **Add Tool Catalogue to Team**: + Assigns a tool catalogue to the team, enabling access to specific tools and utilities defined for the team. + +4. **Add User to Team**: + Opens an interface to add a new user to the selected team. This action facilitates user-role assignment directly from the Teams View. + +5. **Edit Team**: + Redirects to an editing interface where the team's name, description, and permissions can be modified. + +6. **Delete Team**: + Permanently removes the team from the portal. This action may require confirmation to prevent accidental deletions. + +--- + +#### **Efficiency in Team Management** +These quick actions streamline team management by allowing administrators to update access, assign resources, or modify user roles without navigating away from the Teams View. This improves workflow efficiency, especially in environments with frequent updates or large user bases. + +### Team Details View for Tyk AI Studio + +The **Team Details View** allows administrators to review and modify the access credentials and object ownership of a specific team. This includes managing users and assigning catalogues for various resources. Below is an explanation of the elements and actions available: + +--- + +#### **Team Information** +- **Name**: + Displays the name of the selected team (e.g., "Solutions Architects"). This provides context for the resources and users associated with the team. + +--- + +#### **Users in Team** +- **List of Users**: + Displays the names of all users currently assigned to the team (e.g., Martin, Leonid, Ahmet). + - Each user entry includes a **delete icon** (trash bin) for removing the user from the team. + +- **Add User Button**: + A green button labeled **+ ADD USER** that allows administrators to add a new user to the team. Clicking this opens a user selection interface. + +--- + +#### **Catalogues in Team** +Catalogues grant access to resources, tools, or data collections that are assigned to the team. + +1. **Catalogues in Team**: + - Displays a list of general catalogues assigned to the team. + - Includes a **+ ADD CATALOGUE** button to add new catalogues. + +2. **Data Catalogues in Team**: + - Lists all data catalogues assigned to the team. + - Includes a **+ ADD DATA CATALOGUE** button for adding new data catalogues. + +3. **Tool Catalogues in Team**: + - Lists all tool catalogues associated with the team (e.g., "Solution Architects"). + - Includes a **+ ADD TOOL CATALOGUE** button to add additional tool catalogues. + +- **Delete Icons**: + Each catalogue entry includes a delete icon (trash bin) for removing the catalogue from the team. + +--- + +#### **Navigation** +- **Back to Teams**: + A link in the top-right corner that returns the administrator to the Teams List View without saving any changes made in the current view. + +--- + +This Team Details View provides a centralized interface for managing team resources and user roles. It ensures that administrators can efficiently update permissions and access rights, maintaining the organization's security and productivity standards. diff --git a/ai-management/ai-studio/tools.mdx b/ai-management/ai-studio/tools.mdx new file mode 100644 index 00000000..1a911233 --- /dev/null +++ b/ai-management/ai-studio/tools.mdx @@ -0,0 +1,109 @@ +--- +title: "Interact with Tools" +'og:description': "How to integrate tools in Tyk AI Studio?" +tags: ['AI Studio', 'AI Management', 'Tools'] +sidebarTitle: "Tools & Extensibility" +--- + +Tyk AI Studio's Tool System allows Large Language Models (LLMs) to interact with external APIs and services, dramatically extending their capabilities beyond simple text generation. This enables LLMs to perform actions, retrieve real-time data, and integrate with other systems. + +## Purpose + +Tools bridge the gap between conversational AI and external functionalities. By defining tools, you allow LLMs interacting via the [Chat Interface](/ai-management/ai-studio/chat-interface) or API to: + +* Access real-time information (e.g., weather, stock prices, database records). +* Interact with other software (e.g., search JIRA tickets, update CRM records, trigger webhooks). +* Perform complex calculations or data manipulations using specialized services. + +## Core Concepts + +* **Tool Definition:** A Tool in Tyk AI Studio is essentially a wrapper around an external API. Its structure and available operations are defined using an **OpenAPI Specification (OAS)** (v3.x, JSON or YAML). +* **Allowed Operations:** From the provided OAS, administrators select the specific `operationIds` that the LLM is permitted to invoke. This provides granular control over which parts of an API are exposed. +* **Authentication:** Tools often require authentication to access the target API. Tyk AI Studio handles this securely by integrating with [Secrets Management](/ai-management/ai-studio/secrets). You configure the authentication method (e.g., Bearer Token, Basic Auth) defined in the OAS and reference a stored Secret containing the actual credentials. +* **Privacy Levels:** Each Tool is assigned a privacy level. This level is compared against the privacy level of the [LLM Configuration](/ai-management/ai-studio/llm-management) being used. A Tool can only be used if its privacy level is less than or equal to the LLM's level, preventing sensitive tools from being used with potentially less secure or external LLMs. + + Privacy levels define how data is protected by controlling LLM access based on its sensitivity: + - Public – Safe to share (e.g., blogs, press releases). + - Internal – Company-only info (e.g., reports, policies). + - Confidential – Sensitive business data (e.g., financials, strategies). + - Restricted (PII) – Personal data (e.g., names, emails, customer info). +* **Tool Catalogues:** Tools are grouped into logical collections called Catalogues. This simplifies management and access control. +* **Filters:** Optional [Filters](/ai-management/ai-studio/filters) can be applied to tool interactions to pre-process requests sent to the tool or post-process responses received from it (e.g., for data sanitization). +* **Documentation:** Administrators can provide additional natural language documentation or instructions specifically for the LLM, guiding it on how and when to use the tool effectively. +* **Dependencies:** Tools can declare dependencies on other tools, although the exact usage pattern might vary. + +## How it Works + +When a user interacts with an LLM via the [Chat Interface](/ai-management/ai-studio/chat-interface): + +1. The LLM receives the user prompt and the definitions of available tools (based on user group permissions and Chat Experience configuration). +2. If the LLM determines that using one or more tools is necessary to answer the prompt, it generates a request to invoke the specific tool operation(s) with the required parameters. +3. Tyk AI Studio intercepts this request. +4. It validates the request, checks permissions, and retrieves necessary secrets for authentication. +5. Tyk AI Studio applies any configured request Filters. +6. It calls the external API defined by the Tool. +7. It receives the response from the external API. +8. Tyk AI Studio applies any configured response Filters. +9. It sends the tool's response back to the LLM. +10. The LLM uses the tool's response to formulate its final answer to the user. + +## Creating & Managing Tools (Admin) + +Administrators define and manage Tools via the UI or API: + +1. **Define Tool:** Provide a name, description, and privacy level. +2. **Upload OpenAPI Spec:** Provide the OAS document (JSON/YAML). +3. **Select Operations:** Choose the specific `operationIds` the LLM can use. +4. **Configure Authentication:** Select the OAS security scheme and link to a stored [Secret](/ai-management/ai-studio/secrets) for credentials. +5. **Add Documentation:** Provide natural language instructions for the LLM. +6. **Assign Filters (Optional):** Add request/response filters. + + Tool Config + +## Organizing & Assigning Tools (Admin) + +* **Create Catalogues:** Group related tools into Tool Catalogues (e.g., "CRM Tools", "Search Tools"). +* **Assign to Groups:** Assign Tool Catalogues to specific Teams. This grants users in those groups *potential* access to the tools within the catalogue. + + Catalogue Config + +## Using Tools (User) + +Tools become available to end-users through multiple access methods: + +### Chat Interface Access + +Tools become available within the [Chat Interface](/ai-management/ai-studio/chat-interface) if: + +1. The specific Chat Experience configuration includes the relevant Tool Catalogue. +2. The user belongs to a Team that has been assigned that Tool Catalogue. +3. The Tool's privacy level is compatible with the LLM being used in the Chat Experience. + +The LLM will then automatically decide when to use these available tools based on the conversation. + +### AI Portal Tool Catalogue + +The [AI Portal](/ai-management/ai-studio/ai-portal) provides a dedicated **Tool Catalogue** where users can: + +* **Browse Available Tools:** View all tools they have access to, similar to how they can browse LLMs and Data Sources. +* **Subscribe to Tools:** Add tools to their applications as part of the app creation process. +* **Access Documentation:** View tool-specific documentation and usage guidelines. +* **Manage Subscriptions:** Control which tools are included in their various applications. + +### API Access + +Developers can access tools programmatically through the Tyk AI Studio APIs: + +* **Tool Discovery API:** Retrieve lists of available tools and their specifications. +* **Tool Invocation API:** Execute tool operations directly via REST endpoints. +* **Application Integration:** Include tools in [Apps](/ai-management/ai-studio/apps) for streamlined API access. + +### MCP (Model Context Protocol) Access + +Tools can also be accessed through the **Model Context Protocol (MCP)**, providing: + +* **Standardized Interface:** Use MCP-compatible clients to interact with tools in a vendor-neutral way. +* **Enhanced Integration:** Connect tools to MCP-enabled applications and AI frameworks. +* **Protocol Compliance:** Leverage the growing ecosystem of MCP-compatible tools and clients. + +This multi-access approach ensures that tools can be utilized across different interfaces and integration patterns, from simple chat interactions to complex programmatic integrations. diff --git a/ai-management/ai-studio/user-management.mdx b/ai-management/ai-studio/user-management.mdx new file mode 100644 index 00000000..d19ef114 --- /dev/null +++ b/ai-management/ai-studio/user-management.mdx @@ -0,0 +1,83 @@ +--- +title: "User Management & RBAC" +'og:description': "How to configure user management in Tyk AI Studio?" +tags: ['AI Studio', 'AI Management', 'User Management', 'RBAC'] +sidebarTitle: "User Management & RBAC" +--- + +Tyk AI Studio includes a comprehensive system for managing users, their authentication methods, and controlling their access to platform resources using Teams and Role-Based Access Control (RBAC). + +## Purpose + +The User Management & RBAC system provides administrators with the tools to: + +* Manage the lifecycle of user accounts. +* Define how users authenticate (UI sessions, API keys). +* Organize users into logical teams. +* Grant fine-grained access to Tyk AI Studio resources (LLMs, Tools, Data Sources, Chat Experiences) based on team membership. +* Assign platform-level permissions using roles. + +## Core Concepts + +* **User:** Represents an individual interacting with Tyk AI Studio. Users are typically identified by an email address or username and can be created manually by administrators, via invitation, self-registration (if enabled), or provisioned through [SSO Integration](/ai-management/ai-studio/sso). +* **Authentication:** The process of verifying a user's identity. + * **Session-based:** For users logging into the Tyk AI Studio UI (using username/password or SSO). + * **API Key:** For applications or scripts interacting with Tyk AI Studio APIs (like the [Proxy](/ai-management/ai-studio/proxy)). +* **API Key:** A unique, long-lived token generated by a user. Applications use this key (typically in an `Authorization: Bearer ` header) to authenticate requests on behalf of the user who generated it. +* **Team:** A collection of users. Teams are the primary mechanism for assigning access rights to resources. A user can belong to multiple teams. +* **Resource:** Any entity within Tyk AI Studio whose access needs to be controlled. This includes: + * [LLM Configurations](/ai-management/ai-studio/llm-management) + * Tool Catalogues (collections of [Tools](/ai-management/ai-studio/tools)) + * Data Source Catalogues (collections of [Data Sources](/ai-management/ai-studio/datasources-rag)) + * Chat Experiences (configurations for the [Chat Interface](/ai-management/ai-studio/chat-interface)) +* **Role:** Defines a set of broad, platform-level permissions. Common roles include: + * **Admin:** Full access to configure and manage the Tyk AI Studio platform. + * **Standard User:** Access to use assigned resources (e.g., chat, query LLMs) but limited or no administrative capabilities. +* **RBAC (Role-Based Access Control):** Tyk AI Studio's access control model. Access is granted primarily by assigning resource access to **Teams**, and then adding **Users** to those Teams. **Roles** provide overarching platform permissions. +* **User Entitlements:** The complete set of permissions a specific user has at any given time. This is calculated based on their assigned Role and the combined permissions granted through all the Teams they belong to. Systems like the Proxy check these entitlements before allowing an action. + +## User Lifecycle Management (Admin) + +Administrators manage users via the UI or API: + +* **Creation:** Create user accounts manually, send invitations, or manage users provisioned via SSO. +* **Team Assignment:** Add or remove users from various Teams. +* **Role Assignment:** Assign a primary Role (e.g., Admin, Standard) to each user. +* **Status Management:** Activate or deactivate user accounts. +* **API Key Management:** Admins may have visibility into user API keys (though users typically generate their own). + + User Management UI + +## Team Management (Admin) + +Teams are central to managing permissions: + +* **Creation/Deletion:** Create and manage teams (e.g., "Developers", "Sales Team", "Product Docs Users"). +* **User Assignment:** Add/remove users from teams. +* **Resource Assignment:** Grant access to specific LLM Configurations, Tool Catalogues, or Data Source Catalogues *to the team*. Any user in that team inherits this access. + + Group Management UI + +## Authentication Methods + +* **UI Login:** Users access the web interface by logging in with their credentials (username/password) or via a configured [SSO Provider](/ai-management/ai-studio/sso). This establishes a browser session. +* **API Key Authentication:** + 1. A user generates an API Key via their profile settings in the UI. + 2. The user securely provides this key to their application or script. + 3. The application includes the key in the `Authorization` header for requests to Tyk AI Studio APIs: + ``` + Authorization: Bearer + ``` + 4. Tyk AI Studio validates the key and associates the request with the user who generated it. + +## Access Control Flow Example (API Request) + +When an application makes a request to the [Proxy](/ai-management/ai-studio/proxy) using an API Key: + +1. **Key Validation:** Tyk AI Studio validates the API Key. +2. **User Identification:** The system identifies the User associated with the key. +3. **Team Membership:** The system determines all Teams the User belongs to. +4. **Resource Check:** The request targets a specific resource (e.g., an LLM Configuration via its `routeId`). +5. **Permission Verification:** Tyk AI Studio checks if *any* of the user's Teams have been granted access to the requested resource. +6. **Entitlement Check:** Additional checks based on the user's Role and specific entitlements might occur (e.g., budget checks, model restrictions). +7. **Access Granted/Denied:** If all checks pass, the request proceeds; otherwise, it's denied (e.g., 401 Unauthorized or 403 Forbidden). diff --git a/ai-management/ai-studio/users.mdx b/ai-management/ai-studio/users.mdx new file mode 100644 index 00000000..bf311e1b --- /dev/null +++ b/ai-management/ai-studio/users.mdx @@ -0,0 +1,92 @@ +--- +title: "Users List View for Tyk AI Studio" +'og:description': "How to configure Users in AI Studio?" +tags: ['AI Studio', 'AI Management', 'Users'] +--- + +The Users List View provides administrators with an overview of all registered users on the platform. This interface allows for managing users' access and permissions. Below is a detailed explanation of its components: + +--- + +#### **Table Overview** +The table contains user-specific data displayed in rows and organized into the following columns: + +- **ID**: + A unique identifier assigned to each user for easy referencing. + +- **Name**: + The full name of the user as registered in the system. + +- **Email**: + The user's email address, which serves as their primary contact and login credential. + +- **Is Admin**: + Indicates whether the user has administrative privileges. + - **Yes**: The user has admin access and can perform higher-level management tasks. + - **No**: The user does not have admin privileges and is limited to standard user capabilities. + +- **Actions**: + A menu (represented by three dots) that allows administrators to perform additional actions for each user, such as editing user details, managing permissions, or removing a user from the system. + +--- + +#### **Features** +- **Add User Button**: + Located in the top-right corner of the view, this green button allows administrators to add a new user to the system. Upon clicking, a form or modal is expected to open for entering the new user's details. + +- **Pagination Dropdown**: + Found at the bottom-left corner of the table, this dropdown allows administrators to select how many users are displayed per page (e.g., 10, 20, or more users). + +--- + +### Add or Edit Users + +The User Form allows administrators to create new user accounts on the platform. Below is a detailed explanation of the fields and options available in this form: + +--- + +#### **Form Fields** +1. **Name** *(Required)*: + A text field where the administrator enters the full name of the new user. + +2. **Email** *(Required)*: + A text field for the user's email address, which serves as their primary login identifier. + +3. **Password** *(Required)*: + A password field where the administrator sets an initial password for the user. + +--- + +#### **User Role and Permissions** +The form provides toggles to configure the user's role and access permissions: + +1. **Admin User**: + A toggle switch to grant the user administrative privileges. + - **Enabled**: The user becomes an admin with elevated permissions. + - **Disabled**: The user remains a standard user. + +2. **Show Portal**: + A toggle switch to grant or restrict access to the portal interface. + - **Enabled**: The user can access portal features. + - **Disabled**: Portal access is restricted. + +3. **Show Chat**: + A toggle switch to enable or disable the user's access to chat functionality. + - **Enabled**: The user can utilize chat features. + - **Disabled**: Chat features are hidden from the user. + +--- + +#### **Action Button** +- **Add User**: + A button located at the bottom of the form that finalizes the creation of the user account. This button becomes active only after all required fields are completed. + +--- + +#### **Navigation** +- **Back to Users**: + A link in the top-right corner that navigates the administrator back to the Users List View without saving changes. + +--- + +This form provides a streamlined way for administrators to add and configure new user accounts, ensuring flexibility in assigning roles and managing access permissions. diff --git a/ai-management/mcps/api-to-mcp.mdx b/ai-management/mcps/api-to-mcp.mdx new file mode 100644 index 00000000..2e7fdbdf --- /dev/null +++ b/ai-management/mcps/api-to-mcp.mdx @@ -0,0 +1,233 @@ +--- +title: "Natural-language interaction with your APIs (API to MCP)" +'og:description': "Enable AI assistants to safely and dynamically interact with your existing APIs using Tyk's API to MCP tooling." +tags: ['AI MCP', 'API-to-MCP', 'Tyk AI MCP'] +sidebarTitle: "API to MCP" +--- + +## Overview + +**API to MCP** enables AI assistants to safely and dynamically interact with your existing APIs. It allows non-technical users to access API functionality through natural language, while developers retain full control over what endpoints are exposed and how they are accessed. + +This allows AI tools to interpret, invoke, and structure API operations without requiring any backend modifications. + +**Use this tool to:** +- Expose your APIs for AI interaction +- Allow AI assistants to understand and call API operations +- Configure basic access controls (e.g., filtering operations and setting headers) to manage how AI tools interact with your APIs + +If you're looking for quick setup, [jump to Quick Start](#quick-start). For deeper understanding, see [How It Works](#how-it-works) and [Use Cases](#use-cases). + +```mermaid +graph LR + A["Your OpenAPI"] --> B["Tyk API-to-MCP Tool"] + B --> C["Tyk MCP Server"] + D["AI Assistant"] <--> C + C <--> E["Your API"] + + style A fill:#ffffff,stroke:#D1D1E0,stroke-width:1px,font-size:18px + style B fill:#d5f5e3,stroke:#D1D1E0,stroke-width:1px,font-size:18px + style C fill:#d5f5e3,stroke:#D1D1E0,stroke-width:1px,font-size:18px + style D fill:#eeeeee,stroke:#D1D1E0,stroke-width:1px,font-size:18px + style E fill:#ffffff,stroke:#D1D1E0,stroke-width:1px,font-size:18px + + linkStyle default stroke-width:2px +``` + +## Key Features +- **Dynamic OpenAPI Loading:** Load specifications from local files or HTTP/HTTPS URLs +- **OpenAPI Overlay Support:** Apply overlays to customize specifications +- **Flexible Operation Filtering:** Include/exclude specific operations using glob patterns +- **Comprehensive Parameter Handling:** Preserves formats and includes metadata +- **Built-in Access Control & Security:** Control which endpoints are exposed, enforce authentication (API keys, OAuth, etc.), and add custom headers to all API requests, for secure AI access +- **Authentication Support:** Handles API keys, OAuth tokens, and other security schemes +- **MCP Extensions:** Support for custom x-mcp extensions to override tool names and descriptions +- **Multiple Integration Options:** Works with Claude Desktop, Cursor, Vercel AI SDK, and other MCP-compatible environments + +Check the [complete features list](https://github.com/TykTechnologies/api-to-mcp#features) is available in Tyk's *api-to-mcp* GitHub repository. + +

Quick Start

+To get started quickly, the primary way to use it is by configuring your AI assistant to run it directly as an MCP tool. + +### Requirements +- [Node.js v18+](https://nodejs.org/en/download) installed +- An accessible OpenAPI specification, e.g. `https://petstore3.swagger.io/api/v3/openapi.json` (could be a local file as well) +- Claude Desktop (which we show in this example) or other MCP-compatible AI assistant that supports connecting to external MCP-compatible tool servers (e.g. Cursor, Vercel AI SDK, Cline extension in VS Code etc.) + +### Configure your AI Assistant + +To connect the tool with Claude Desktop or other MCP-compatible assistants, you need to register it as an MCP server. Most AI assistance share similar MCP server definition. This is the definition for *api-to-mcp* with petstore as the OpenAPI: + +```json expandable +{ + "mcpServers": { + "api-tools": { + "command": "npx", + "args": [ + "-y", + "@tyk-technologies/api-to-mcp@latest", + "--spec", + "https://petstore3.swagger.io/api/v3/openapi.json" + ], + "enabled": true + } + } +} +``` + +**Step 1.** +To enable the tool, paste the above configuration into your AI assistant’s MCP config file + +- **Claude Desktop**: For MacOS, you need to update `~/Library/Application Support/Claude/claude_desktop_config.json`. See the [Claude Desktop setup instructions](https://github.com/TykTechnologies/api-to-mcp?tab=readme-ov-file#setting-up-in-claude-desktop) for Windows OS and more customization options. +- **Cursor**: See the [Cursor setup guide](https://github.com/TykTechnologies/api-to-mcp#cursor) for instruction on setting it with Cursor. + +**Step 2.** +Once connected, ask the AI to perform an operation (e.g., "List all pets" or "Create a new user"). + +

How It Works

+### User flow in *API to MCP** user flow + +1. **Input**: Your OpenAPI specification (required) and optional overlays +2. **Processing**: The API to MCP tool loads the spec, applies any overlays, and transforms API operations into MCP tools +3. **Runtime**: The MCP server exposes these tools to AI assistants, which are now discoverable to the AI assistant +4. **Execution Flow**: When you ask the AI assistant a question, it calls the tool (via the MCP server), which translates the request, forwards it to your API, and returns a formatted response. + + +```mermaid +flowchart LR + subgraph "Input" + A["OpenAPI Specification"] + B["Optional Overlays"] + end + + subgraph "API to MCP Tool" + C["1. Load & Parse
OpenAPI Spec"] + D["2. Apply Overlays
(if provided)"] + E["3. Transform API Operations
into MCP Tools"] + F["MCP Server"] + end + + subgraph "Runtime" + G["4. AI Assistant
Discovers Tools"] + H["AI Assistant
Calls Tools"] + I["Your API"] + end + + A -->|YAML/JSON| C + B -->|Optional| D + C --> D + D --> E + E -->|Register Tools| F + F -->|Expose Tools| G + G --> H + H -->|Request| F + F -->|Translate & Forward| I + I -->|Response| F + F -->|Format & Return| H + + classDef input fill:#f9f,stroke:#333,stroke-width:2px,font-size:18px; + classDef tool fill:#bbf,stroke:#333,stroke-width:2px,font-size:18px; + classDef runtime fill:#bfb,stroke:#333,stroke-width:2px,font-size:18px; + + class A,B input; + class C,D,E,F tool; + class G,H,I runtime; + +linkStyle default stroke-width:2px +``` + +### Request lifecycle: how an AI assistant invokes an API tool + +The following diagram illustrates the flow of a request through the system at runtime: + +```mermaid +sequenceDiagram + participant AI as "AI Client" + participant MCP as "MCP Server" + participant Tool as "Tool Handler" + participant API as "API Client" + participant Target as "Target API" + + AI->>MCP: Invoke Tool + MCP->>Tool: Execute Tool Handler + Tool->>Tool: Extract Parameters + Tool->>Tool: Validate Input + Tool->>API: executeApiCall() + API->>API: Apply Security + API->>API: Construct Request + API->>Target: Make HTTP Request + Target-->>API: HTTP Response + + alt Successful Response + API-->>Tool: Success Response + Tool->>MCP: Format MCP Result + MCP-->>AI: Tool Execution Success + else Error Response + API-->>Tool: Error Response + Tool->>MCP: Map to MCP Error + MCP-->>AI: Tool Execution Error + end + +``` + +API to MCP can be found in [api-to-mcp GitHub repository](https://github.com/TykTechnologies/api-to-mcp) + +

Use cases

+Use **API to MCP** when you need to: + +- **Connect AI Assistants to Existing APIs** - Let AI tools understand and call your existing API operations using natural language β€” no code changes needed, just [configuration](https://github.com/TykTechnologies/api-to-mcp/#configuration). + +- **Create a Unified Interface for AI Systems** - Standardize how APIs are accessed by AI across your organization with a consistent protocol (MCP). + +- **Control API Access for AI** - Filter which operations are available to AI, apply authentication, and monitor usage securely. + +- **Improve API Discoverability** - Enable AI systems to automatically list available endpoints, input parameters, and expected responses. + +- **Test APIs Using AI** - Use AI assistants to generate test inputs, invoke endpoints, and validate responses in a conversational way. Natural-language test generation and feedback. + +- **Auto-docs & validation** - Use AI to test, describe, or troubleshoot APIs in a conversational way. Natural-language test generation and feedback. + +- **Workflow Automation** - Connect APIs and AI logic in real time to automate workflows and streamline processes. + +--- + +## Best Practices + +- **Start small**: Only expose safe, limited endpoints +- **Use filters**: allow list or block list endpoints as needed +- **Secure your APIs**: Pass tokens, headers, or keys securely +- **Track usage**: Monitor tool access and patterns +- **Version specs**: Maintain OpenAPI version control +- **Use env vars**: Don't hardcode secrets in CLI or config +- **Validate safely**: Test in staging before going live + +--- + +## Customize your own version of the api-to-mcp tool + +If you'd like to share your MCP with a predefined OpenAPI spec and configuration, you can customize this tool to fit your needs. Useful for sharing pre-configured setups with others. + +By creating a customized version, others can use the tool with minimal configuration --- no need to manually specify specs or overlays. + +Refer to the [customization and publishing guide](https://github.com/TykTechnologies/api-to-mcp?tab=readme-ov-file#customizing-and-publishing-your-own-version) in the *api-to-mcp* repository for step-by-step instructions. + +--- + +## FAQs + +**Does this work with GraphQL?** +Not currently β€” OpenAPI REST APIs only. + +**How do I secure my API requests?** +Use `--headers`, environment variables. Check the [configuration section](https://github.com/TykTechnologies/api-to-mcp/tree/main#configuration) for more details. + +**Can I hide or rename tools?** +Yes β€” use `x-mcp` extensions and filters. + +**What AI tools are supported?** +Any tool that supports MCP: Claude, Cursor, VS Code, and more. + + +## Summary + +API to MCP transforms OpenAPI specs into AI-compatible tools using the MCP standard. It enables your AI stack to dynamically understand, test, and invoke your existing APIs securely β€” without modifying your existing backend. diff --git a/ai-management/mcps/dashboard-api-to-mcp.mdx b/ai-management/mcps/dashboard-api-to-mcp.mdx new file mode 100644 index 00000000..57ddcb75 --- /dev/null +++ b/ai-management/mcps/dashboard-api-to-mcp.mdx @@ -0,0 +1,105 @@ +--- +title: "Natural-language interaction with Tyk Dashboard (API to MCP)" +'og:description': "Talk to Tyk Dashboard like a person using AI tools" +tags: ['AI MCP', 'Dashboard API-to-MCP', 'Tyk Dashboard API MCP', 'Dashboard API', 'Talk to Tyk Dashboard', 'AI Management'] +sidebarTitle: "Dashboard API to MCP" +--- + +## Overview + +Use `tyk-dashboard-mcp` to expose your **Tyk Dashboard API** to AI assistants like Claude, Cursor, or VS Code extensions β€” enabling natural-language interaction with your Tyk Dashboard. + +This tool is a preconfigured fork of [api-to-mcp GitHub repository](https://github.com/TykTechnologies/api-to-mcp), designed specifically for the *Tyk Dashboard* API. It comes bundled with a predefined OpenAPI spec and overlays, so you don’t need to configure much manually. + +Explore the core functionality in the [API to MCP guide](/ai-management/mcps/api-to-mcp). + + +## Use Cases + +Once connected, you, with your AI assistants, can perform helpful actions on your Tyk Dashboard using natural language. For example: +- **Explore your API landscape** - List APIs, describe endpoints in plain English, review policies +- **Query Dashboard settings for audits or support tasks** - List users and keys +- **Automate admin tasks** - Create or update API definitions (e.g., OAS-based APIs) through AI-driven flows, reducing manual clicks (please note that we haven't documented this just yet) +- **Power AI developer tools** - Use this as a backend for developer assistants like Claude, Cursor, or VS Code extensions to guide devs while on boarding and using Tyk Dashboard on daily basis. Ideal for internal use cases like AI-driven dashboards, documentation bots, or dev portals powered by LLMs. +- **Build internal chatbots** - Create internal tools that let team members ask questions like "What APIs are active?" or "What's the global rate limit defined API X?" + + +## Setup Instructions + +**Step 1.** Use the following MCP server config for Claude Desktop, Cursor, Cline or any other MCP-compatible tool: + +```json expandable +{ + "mcpServers": { + "tyk-dashboard-api": { + "command": "npx", + "args": [ + "-y", + "@tyk-technologies/tyk-dashboard-mcp", + "--target", + "https://your-dashboard-domain.com", + "--headers", + "Authorization: $TYK_API_KEY" + ], + "enabled": true + } + } +} +``` + +Refer to your assistant’s docs for where to place this config β€” e.g. +- `claude_desktop_config.json` for [Claude configuration](https://modelcontextprotocol.io/quickstart/user#2-add-the-filesystem-mcp-server) +- `.cursor-config.json` for [Cursor configuration](https://docs.cursor.com/context/model-context-protocol#configuring-mcp-servers) +- `cline_mcp_settings.json` for [Cline configuration](https://docs.roocode.com/features/mcp/using-mcp-in-roo#configuring-mcp-servers) (as a VS Code extension). + +**Step 2.** +Once connected, ask your AI assistant to perform an operation (e.g., "List all apis" or "Create a new user"). + +## Examples + +Here you can see the response of asking the *Cline* in VS Code: + +1. Task: *Show me the Tyk dashboard api endpoint to create apis* + +Screenshot of the response to request of AI to create a new user + +
+ +2. Task: *Please create a new user in tyk dashboard* + +Screenshot of the response to request of AI to create a new user + +## Tips + +- You don’t need to manually define an OpenAPI spec β€” this tool includes the official Tyk Dashboard OpenAPI spec. +- You can fork or extend the tool if you want to include additional internal APIs alongside the dashboard. +- It's an open source and you can find it in [tyk-dashboard-mcp GitHub repository](https://github.com/TykTechnologies/tyk-dashboard-mcp) + +## FAQs + +**How is this different from `api-to-mcp`?** +`tyk-dashboard-mcp` is a customised version which is preconfigured for the Tyk Dashboard API. No need to specify your own spec. + +**Does this expose all dashboard functionality?** +Only the operations defined in the OpenAPI spec. You can customize the access list to show/hide more. In the following MCP server config the `--whitelist` to only allow access to getAPIs operation and to only allow to create Tyk OAS definitions: + +```json expandable +{ + "mcpServers": { + "tyk-dashboard-api": { + "command": "npx", + "args": [ + "-y", + "@tyk-technologies/tyk-dashboard-mcp", + "--target", + "https://your-dashboard-domain.com", + "--headers", + "Authorization: $TYK_API_KEY", + "--whitelist", + "getApis*,POST:/api/apis/oas", + ], + "enabled": true + } + } +} +``` diff --git a/ai-management/mcps/overview.mdx b/ai-management/mcps/overview.mdx new file mode 100644 index 00000000..b6a3dc44 --- /dev/null +++ b/ai-management/mcps/overview.mdx @@ -0,0 +1,51 @@ +--- +title: "Tyk MCPs" +description: "A comprehensive guide to Model Context Protocol (MCP) servers in Tyk and how they extend AI capabilities." +sidebarTitle: "Overview" +keywords: ['AI MCP', 'MCPs in Tyk', 'Model Context Protocol'] +tags: ['AI MCP', 'MCPs in Tyk', 'Model Context Protocol'] +--- + +## MCP capabilities + +[Model Context Protocol (MCP)](https://modelcontextprotocol.io/introduction) servers help AI systems securely interact with external services and tools. They establish structured, governed connections that integrate seamlessly with your Tyk environment. + +## What are MCPs? + +Model Context Protocol (MCP) servers extend AI systems by exposing external services, tools, and resources in a standardised way. They act as bridges between AI applications and external systems, securely managing authentication, access, and execution. + +With Tyk MCPs, your AI agents can: + +- Access external data sources and APIs +- Execute specialised tools and functions +- Interact with system resources +- Retrieve contextual information + +MCPs use a defined protocol to connect AI agents with external systems, expanding AI capabilities while maintaining governance and control. + +## Why standardisation matters + +The MCP specification standardises how AI agents discover and interact with external capabilities. This helps: + +- **Simplify integration** across diverse systems +- **Enhance security** through consistent architecture +- **Promote interoperability** with different vendor solutions +- **Improve governance** when managing AI systems at scale + +

MCP for Enterprise use

+Tyk extends the MCP model for enterprise deployments with the following capabilities: + +- **Remote MCP catalogues and server support** – Expose internal APIs and tools to AI assistants securely without requiring local installations. +- **Secure local MCP server deployment** – Deploy MCP servers within controlled environments, integrated with Tyk AI Gateway for monitoring and governance. +- **Standardised protocols** – Maintain interoperability standards for seamless integration into existing workflows. + +These features enable enterprises to scale AI integrations securely while ensuring compliance and operational control. + +

Ready-to-use MCP options

+Tyk offers several ready-to-use MCP integrations: + +- **[API to MCP](/ai-management/mcps/api-to-mcp)** – Convert existing APIs (via OpenAPI/Swagger specs) into MCP-accessible tools. +- **[Dashboard API to MCP](/ai-management/mcps/dashboard-api-to-mcp)** – Expose the Tyk Dashboard API for management and monitoring. +- **[Tyk Docs MCP](/ai-management/mcps/tyk-docs-mcp)** – Provide AI access to searchable Tyk documentation. + +For more information on implementing MCPs, [contact the Tyk team](https://tyk.io/contact/) do discuss your specific use cases. diff --git a/ai-management/mcps/tyk-docs-mcp.mdx b/ai-management/mcps/tyk-docs-mcp.mdx new file mode 100644 index 00000000..e876e8d3 --- /dev/null +++ b/ai-management/mcps/tyk-docs-mcp.mdx @@ -0,0 +1,90 @@ +--- +title: "Natural-language interaction with Tyk Docs (MCP)" +'og:description': "Talk to Tyk documentation like a person using AI tools. Use Docs MCP to enable AI assistants to search and retrieve information from Tyk documentation." +tags: ['AI MCP', 'Tyk Docs', 'AI Documentation Search', 'Talk to Tyk Docs'] +sidebarTitle: "Tyk Docs MCP" +--- + +## Overview + +*Tyk Docs MCP* is a tool [MCP server](https://github.com/modelcontext/spec) exposing the Tyk documentation to connected AI assistants. It enables the users of AI assistants "talk to" Tyk's documentation. Instead of searching manually, users can ask natural-language questions β€” and get answers backed by Tyk official docs. The tool makes AI-assisted support, troubleshooting, and documentation exploration fast and reliable. + + +Here you can see the AI assistant chooses to use Tyk Docs MCP (*Cline* in VS Code) while answering the query *How do I set a rate limit for a Tyk API?*: + +Screenshot of the response to request of AI to create a new user + + +Screenshot of the response to request of AI to create a new user + + +## Key Features + +- **Semantic search** β€” finds the most relevant content, not just keyword matches +- **Contextual results** β€” includes sections around your result for better understanding +- **Product filters** β€” limit results by product (Gateway, Dashboard, etc.) +- **Includes links** β€” jump straight into the relevant section of the docs +- **Answer snippets** β€” shows concise answers when possible +- **Always up to date** β€” syncs with the latest Tyk documentation + + +

Use Cases

+Let AI do the digging β€” here’s how teams use Tyk Docs MCP: + +- **First-line support** - *How do I set a rate limit for a Tyk API?* +
AI can help answer questions about Tyk products, features, and usage cases. + +- **Feature implementation help** - *How do I enable JWT authentication in Tyk Gateway?* +
AI can help developers use Tyk's features by providing ad hoc step-by-step instructions and examples. + +- **Troubleshooting guidance** - *I'm seeing 'Auth field missing' error. What does that mean?* +
AI can help identify issues and provide guidance on how to resolve them. + +- **Fast API reference** - *What fields are in the /apis response?* +
AI can help developers quickly find the information they need to implement Tyk's features. + +- **Discover what’s possible** - *What analytics tools does Tyk include?* +
AI can help developers discover new ways to use Tyk's features and capabilities. + + +## Quick Start + +To get started quickly, the primary way to use it is by configuring your AI assistant to run it directly as an MCP tool. + +### Requirements +- [Node.js v18+](https://nodejs.org/en/download) installed +- Internet access to reach Tyk documentation index +- MCP-compatible AI assistant that supports connecting to external MCP-compatible tool servers (e.g. Claude, Cursor, Vercel AI SDK, Cline extension in VS Code etc.), + +### Configure your AI Assistant + +To connect the tool with MCP-compatible assistants, you need to register it as an MCP server. Most AI assistance share similar MCP server definition. This is the definition for *docs-mcp*: + +```json expandable +{ + "mcpServers": { + "tyk-docs-search": { + "command": "npx", + "args": [ + "-y", + "@tyk-technologies/docs-mcp@latest" + ], + "enabled": true + } + } +} +``` + +**Step 1.** +To enable the tool, paste the above configuration into your AI assistant’s MCP config file + +- **Claude Desktop**: For MacOS, you need to update `~/Library/Application Support/Claude/claude_desktop_config.json`. See the [Claude Desktop setup instructions](https://github.com/TykTechnologies/api-to-mcp?tab=readme-ov-file#setting-up-in-claude-desktop) for Windows OS and more customization options. +- **Cursor**: See the [Cursor setup guide](https://github.com/TykTechnologies/api-to-mcp#cursor) for instruction on setting it with Cursor. + +**Step 2.** +Once connected, ask the AI to perform an operation as suggested in the [use cases above](#use-cases) + + +## How It Works under the hood + +Tyk Docs MCP tool is built on top of the [@buger/probe-docs-mcp project](https://github.com/buger/docs-mcp). This package comes with Tyk's documentation already bundled, offering a plug-and-play experience for users who want to integrate Tyk documentation into their AI assistants. diff --git a/ai-management/overview.mdx b/ai-management/overview.mdx new file mode 100644 index 00000000..0a4f6def --- /dev/null +++ b/ai-management/overview.mdx @@ -0,0 +1,86 @@ +--- +title: "AI management" +'og:description': "Tyk AI Management landing page. This page provides an overview of Tyk's AI management solutions including AI Studio and MCPs." +tags: ['Tyk AI management', 'AI Studio', 'Tyk MCPs'] +sidebarTitle: "Overview" +--- + +As artificial intelligence becomes increasingly integrated into enterprise systems, organisations need structured, secure, and governed approaches to manage AI capabilities effectively. Tyk's AI management solutions are designed to help enterprises integrate, control, and scale AI applications while maintaining compliance and security. + +## Secure AI for the enterprise + +Tyk's AI management solutions address key challenges in AI governance, security, and integration. They enable organisations to deploy AI capabilities while maintaining oversight, managing risks, and meeting enterprise standards. + +## AI integration architecture and its importance + +Integrating AI into existing systems requires a structured architecture that connects models, APIs, and specialised tools securely and efficiently. + +A managed AI integration architecture provides: + +- **Standardisation** to ensure interoperability across AI components +- **Security** across AI workflows and data interactions +- **Governance** to monitor and control AI usage and data +- **Scalability** for enterprise-wide deployment and increasing complexity +- **Interoperability** across vendors and services + +Without a structured approach, organisations risk fragmented solutions, security gaps, and unmanaged AI usage ("shadow AI"). By integrating AI into existing systems, enterprises can achieve a more secure and efficient approach to AI management. + +## Tyk’s AI management capabilities + +Tyk provides two key solutions for AI management: + +### [AI Studio](/ai-management/ai-studio/overview) + +Tyk AI Studio is a platform for managing and deploying AI applications securely and at scale. It provides: + +- **Centralised governance** with role-based access control and compliance tracking +- **Cost management** through usage monitoring and budgeting tools +- **Security features** including unified access controls and credential management +- **Developer enablement** via curated AI service catalogues +- **Collaboration tools** through intuitive AI interfaces + +AI Studio supports enterprises in reducing unauthorised AI usage by providing central management across all AI interactions. + +[Explore AI Studio](/ai-management/ai-studio/overview) + +### [Tyk MCPs](/ai-management/mcps/overview) + +The [Model Context Protocol (MCP)](https://modelcontextprotocol.io/introduction) provides a standardised method for AI components to interact with external resources. + +With MCPs, organisations can: + +- **Integrate securely** with external AI providers and services +- **Build custom tools** for AI assistants and workflows +- **Access resources** such as files, APIs, and databases +- **Enhance AI workflows** with contextual information + +MCPs help expand AI system functionality by enabling secure, standardised interactions between services. + +[Explore Tyk MCPs](/ai-management/mcps/overview) + +### How AI Studio and MCPs work together + +Tyk AI Studio and Tyk MCPs are complementary: + +- **AI Studio** offers governance, monitoring, and development tooling for AI management. +- **Tyk MCPs** provide secure connections to external services and APIs. + +Together, they create a flexible, governed framework for managing AI applications at scale. + +## Next steps + +To start using Tyk's AI management capabilities: + +1. Explore the [AI Studio documentation](/ai-management/ai-studio/overview) +2. Review [Tyk MCPs](/ai-management/mcps/overview) and how they extend AI systems +3. [Request a demo](https://tyk.io/ai-demo/) to see the platform in action. + +## Key outcomes + +Tyk's AI management solutions are designed to: + +- **Reduce risk** through centralised access and monitoring +- **Improve efficiency** across AI development workflows +- **Enhance cost control** with usage and budgeting insights +- **Support compliance** with data protection and security standards +- **Enable scalable architectures** based on open protocols diff --git a/api-management/api-versioning.mdx b/api-management/api-versioning.mdx new file mode 100644 index 00000000..96c6aeb8 --- /dev/null +++ b/api-management/api-versioning.mdx @@ -0,0 +1,625 @@ +--- +title: "API Versioning" +'og:description': "Create and manage multiple versions of an API" +tags: ['API versioning', 'version', 'Tyk Classic', 'Tyk OAS', 'API', 'versioning'] +sidebarTitle: "API Versioning" +--- + +## Introduction + +API versioning is a crucial practice in API development and management that allows you to evolve your API over time while maintaining backward compatibility for existing clients. As your API grows and changes, versioning provides a structured way to introduce new features, modify existing functionality, or deprecate outdated elements without breaking integrations for users who rely on previous versions. + +API versioning is important for several reasons: +- **Flexibility**: It allows you to improve and expand your API without disrupting existing users. +- **Stability**: Clients can continue using a specific version of the API, ensuring their applications remain functional. +- **Transition management**: You can gradually phase out older versions while giving clients time to migrate to newer ones. +- **Documentation**: Each version can have its own documentation, making it easier for developers to understand the specific capabilities and limitations of the version they're using. + +--- + +## When to use API versioning + +There are many occasions when you might use versioning with your APIs, here are just a few examples. + +### Adding new features + +Imagine you're running an e-commerce API, and you want to introduce a new recommendation engine. Instead of modifying the existing endpoint and potentially breaking current integrations, you could create a new version of the API that includes this feature. This allows you to roll out the enhancement to interested clients while others continue using the previous version without disruption. + +### Changing response formats + +Let's say you have a weather API that currently returns temperatures in Fahrenheit. You decide to switch to Celsius for international standardization. By creating a new API version with this change, you can transition to the new format without affecting existing users who expect Fahrenheit readings. This gives clients time to adapt their applications to handle the new response format. + +### Deprecating outdated functionality + +If your financial API includes a legacy payment processing method that you plan to phase out, versioning allows you to create a new version without this feature. You can then encourage users to migrate to the new version over time, eventually deprecating the old version containing the outdated functionality. + +### Optimizing performance + +You might discover a more efficient way to structure your API requests and responses. By introducing these optimizations in a new version, you can offer improved performance to clients who are ready to upgrade, while maintaining the existing version for those who aren't prepared to make changes yet. + +## Sunsetting API versions + +API sunsetting is the process of phasing out or retiring an older version of an API or an entire API. It's a planned, gradual approach to ending support for an API or API version. To aid with the automation of this process, all Tyk API versions can be configured with an optional expiry date `expiration` (`expires` for Tyk Classic APIs), after which the API will no longer be available. If this is left blank then the API version will never expire. This is configured in standard ISO 8601 format. + +When sunsetting API versions, you may have endpoints that become deprecated between versions. It can be more user friendly to retain those endpoints but return a helpful error, instead of just returning `HTTP 404 Not Found`. + +This is easy to do with Tyk. You could, for example, include the deprecated endpoint in the new version of the API and configure the [mock response](/api-management/traffic-transformation/mock-response) middleware to provide your clients with relevant information and instruction. Alternatively, you could return a `HTTP 302 Found` header and redirect the user to the new endpoint. + +--- + +## How API versioning works with Tyk + +API versioning is supported for all APIs that can be deployed on Tyk, both Tyk OAS APIs (for REST and Streaming services) and Tyk Classic APIs (used for GraphQL and TCP services). There are differences in the approach and options available when using the two API formats: + +- With Tyk OAS APIs you're essentially creating distinct iterations of your API, each with its own API definition file, allowing almost complete differentiation of configuration between your API versions. +- With Tyk Classic APIs all versions of an API are configured from a single API definition. This means that they share many features with only a subset available to be configured differently between versions. + +For more details, see [this comparison](#comparison-between-tyk-oas-and-tyk-classic-api-versioning). + +Some key concepts that are important to understand when versioning your APIs are: + +- [Version identifiers](/api-management/api-versioning#version-identifiers) +- [Default version](/api-management/api-versioning#default-version) +- [Base and child versions](/api-management/api-versioning#base-and-child-apis) +- [Controlling access to versions](/api-management/api-versioning#controlling-access-to-versioned-apis) + +### Version Identifiers + +The version identifier is the method by which the API client specifies which version of an API it is addressing with each request. Tyk supports multiple [locations](/api-management/api-versioning#version-identifier-location) within the request where this identifier can be placed. Typically the value assigned in the version identifier will be matched to the list of versions defined for the API and, assuming that the client is [authorized to access](/api-management/api-versioning#controlling-access-to-versioned-apis) that version, Tyk will apply the version specific processing to the request. + +#### Version identifier location + +Tyk supports three different locations where the client can indicate which version of an API they wish to invoke with their request: + +- [Request URL (path)](/api-management/api-versioning#request-url-path) +- [Query parameter](/api-management/api-versioning#query-parameter) +- [Request header](/api-management/api-versioning#request-header) + +When choosing a version identifier location, consider your API design philosophy, infrastructure requirements, client needs, caching strategy, and backward compatibility concerns. Whichever method you choose, aim for consistency across your API portfolio to provide a uniform experience for your API consumers. + +##### Request URL (path) + +Including the version identifier in the path (for example `/my-api/v1/users`) is a widely used approach recognized in many API designs. The version identifier is clearly visible in the request and, with the unique URL, can simplify documentation of the different versions. Tyk can support the version identifier as the **first URL fragment** after the listen path, such that the request will take the form `//`. + +##### Query parameter + +Defining a query parameter that must be provided with the request (for example `/my-api/users?version=v1`) is easy to implement and understand. The version identifier is clearly visible in the request and can be easily omitted to target a default version. Many analytics tools can parse query parameters, making this a very analytics-friendly approach to versioning. + +##### Request header + +Defining a specific header that must be provided with the request (for example `x-api-version:v1`) keeps the URL *clean*, which can be aesthetically pleasing and easier to read. It works well with RESTful design principles, treating the version as metadata about the request and allows for flexibility and the ability to make changes to the versioning scheme without modifying the URL structure. Headers are less visible to users than the request path and parameters, providing some security advantage. Be aware that other proxies or caches might not consider headers for routing, which could bring issues with this method. + +#### Stripping version identifier + +Typically Tyk will pass all request headers and parameters to the upstream service when proxying the request. For a versioned API, the version identifier (which may be in the form of a header, path parameter or URL fragment) will be included in this scope and passed to the upstream. + +The upstream (target) URL will be constructed by combining the configured `upstream.url` (`target_url` for Tyk Classic APIs) with the full request path unless configured otherwise (for example, by using the [strip listen path](/api-management/gateway-config-tyk-oas#listenpath) feature). + +If the version identifier is in the request URL then it will be included in the upstream (target) URL. If you don't want to include this identifier, then you can set `stripVersioningData` (`strip_versioning_data` for Tyk Classic APIs) and Tyk will remove it prior to proxying the request. + +#### Version identifier pattern + +When using the [Request URL](/api-management/api-versioning#request-url-path) for the versioning identifier, if Tyk is configured to strip the versioning identifier then the first URL fragment after the `listenPath` (`listen_path` for Tyk Classic APIs) will be deleted prior to creating the proxy URL. If the request does not include a versioning identifier and Tyk is configured to [fallback to default](/api-management/api-versioning#fallback-to-default), this may lead to undesired behaviour as the first URL fragment of the endpoint will be deleted. + +In Tyk 5.5.0 we implemented a new configuration option `urlVersioningPattern` (`url_versioning_pattern` for Tyk Classic APIs) to the API definition where you can set a regex that Tyk will use to determine whether the first URL fragment after the `listenPath` is a version identifier. If the first URL fragment does not match the regex, it will not be stripped and the unaltered URL will be used to create the upstream URL. + +### Default version + +When multiple versions are defined for an API, one must be declared as the **default version**. If a request is made to the API without providing the version identifier, then this will automatically be treated as a request to the *default* version. This has been implemented to support future versioning of an originally unversioned API, as you can continue to support legacy clients with the default version. + +Tyk makes it easy for you to specify - and change - the *default* version for your APIs. + +#### Fallback to default + +The standard behaviour of Tyk, if an invalid version is requested in the version identifier, is to reject the request returning `HTTP 404 This API version does not seem to exist`. Optionally, Tyk can be configured to redirect these requests to the *default* version by configuring the `fallbackToDefault` option in the API definition (`fallback_to_default` for Tyk Classic APIs). + + +### Base and child APIs + +Tyk OAS introduces the concept of a **Base API**, which acts as a *parent* that routes requests to the different *child* versions of the API. The Base API stores the information required for Tyk Gateway to locate and route requests to the appropriate *child* APIs. + +The *child* APIs do not have any reference back to the *parent* and so can operate completely independently if required. Typically, and we recommend, the *child* versions should be configured as Internal APIs that are not directly reachable by clients outside Tyk. + +The Base API is a working version of the API and is usually the only one configured as an *External API*, so that client requests are handled (and routed) according to the configuration set in the Base API (via the version identifier included in the header, url or query parameter). + +You can configure a Tyk OAS API as a *Base* API by adding the `versioning` object to the `info` section in the Tyk Vendor Extension. This is where you will configure all the settings for the versioned API. The *child* APIs do not contain this information. + +Note that any version (*child* or *Base*) can be set as the [default version](/api-management/api-versioning#default-version). + +Tyk Classic APIs do not have the base and child concept because all versions share an API definition. + +### Controlling access to versioned APIs + +Tyk's access control model supports very granular permissions to versioned APIs using the `access_rights` assigned in the [access keys/tokens](/api-management/policies#what-is-a-session-object) or [security policies](/api-management/policies#what-is-a-security-policy) that are applied to keys. + +This means that you could restrict client access to only the [Base API](/api-management/api-versioning#base-and-child-apis), while allowing developers to create and test new versions independently. These will only be added to the "routing table" in the Base API when the API owner is ready and access keys could then be updated to grant access to the new version(s). + +Note that an access key will only have access to the `default` version if it explicitly has access to that version (e.g. if `v2` is set as default, a key must have access to `v2` to be able to [fallback to the default](/api-management/api-versioning#fallback-to-default) if the versioning identifier is not correctly provided in the request. + +When using Tyk OAS APIs each version of the API has a unique [API Id](/api-management/gateway-config-tyk-oas#info), so you simply need to identify the specific versions in the `access_rights` list in the key in the same way that you would add multiple different APIs to a single key. + + + +Creating a new version of a Tyk OAS API will not affect its API Id, so any access keys that grant access to the API will continue to do so, however they will not automatically be granted access to the new version (which will have a new API Id). + + + +When using Tyk Classic APIs you can explicitly grant access to specific versions of an API by specifying only those versions in the `versions` list in the key within the single entry for the API in the `access_rights` list. + +### Comparison between Tyk OAS and Tyk Classic API versioning + +As explained, there are differences between the way that versioning works for Tyk OAS and Tyk Classic APIs. + +These are largely due to the fact that a separate API definition is generated for each version of a Tyk OAS API, with one designated as the [base](/api-management/api-versioning#base-and-child-apis) version (which should be exposed on Tyk Gateway with the other (child) versions set to [internal](/api-management/traffic-transformation/internal-endpoint) visibility) whereas all versions of a Tyk Classic API are described by a single API definition. + +The Tyk Classic approach limits the range of features that can differ between versions. + +This table gives an indication of some of the features that can be configured per-version (βœ…) or only per-API (❌️) for Tyk OAS and Tyk Classic APIs. + +| Feature | Configurable in Tyk OAS versioning | Configurable in Tyk Classic versioning | +| :--------- | :------------------------------------ | :---------------------------------------- | +| Client-Gateway security | βœ… | ❌️ | +| Request authentication method | βœ… | ❌️ | +| API-level header transform | βœ… | βœ… | +| API-level request size limit | βœ… | βœ… | +| API-level rate limiting | βœ… | ❌️ | +| API-level caching | βœ… | ❌️ | +| Endpoints (method and path) | βœ… | βœ… | +| Per-endpoint middleware | βœ… | βœ… | +| Context and config data for middleware | βœ… | ❌️ | +| Custom plugin bundle | βœ… | ❌️ | +| Upstream target URL | βœ… | βœ… | +| Gateway-Upstream security | βœ… | ❌️ | +| Traffic log config | βœ… | ❌️ | +| API segment tags | βœ… | ❌️ | + +--- + +## Configuring API versioning in the API definition + +You can configure a Tyk OAS API as a [Base API](/api-management/api-versioning#base-and-child-apis) by adding the `info.versioning` [object](/api-management/gateway-config-tyk-oas#versioning) to the [Tyk Vendor Extension](/api-management/gateway-config-tyk-oas#tyk-vendor-extension). + +Some notes on this: + +- if the *base* version is to be used as the *default* then you can use the value `self` as the identifier in the `default` field +- in the `versions` field you must provide a list of key-value pairs containing details of the *child* versions: + - `id`: the unique API Id (`x-tyk-api-gateway.info.id`) assigned to the API (either automatically by Tyk or user-defined during API creation) + - `name`: an identifier for this version of the API, for example `v2` + +The *child API* does not require any modification to its API definition. The important thing is that its API Id must be added to the `versions` list in the *base API* definition. We strongly recommend, however, that you configure `info.state.internal` to `true` for all child APIs so that they can only be accessed via the *base API*. + + + +If you are using Tyk Classic APIs, please see [this section](/api-management/api-versioning#versioning-with-tyk-classic-apis). + + + +### Example Tyk OAS Base API + +In the following example, we configure a *Base API*: + +```json {hl_lines=["11-27"],linenos=true, linenostart=1} expandable +{ + "info": { + "title": "example-base-api", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": {}, + "components": {}, + "x-tyk-api-gateway": { + "info": { + "versioning": { + "default": "v1", + "enabled": true, + "key": "x-api-version", + "location": "header", + "name": "v1", + "versions": [ + { + "id": "", + "name": "v2" + } + ], + "fallbackToDefault": true, + "stripVersioningData": false, + "urlVersioningPattern": "" + }, + "expiration": "2030-01-01 00:00", + "name": "example-base-api", + "state": { + "active": true, + "internal": false + } + }, + "server": { + "listenPath": { + "strip": true, + "value": "/example-base-api/" + } + }, + "upstream": { + "url": "http://httpbin.org/" + } + } +} +``` + +This API definition will configure Tyk Gateway to expect the `x-api-version` header to be provided and will invoke a version of the API as follows: +- if the header key has the value `v1` then the Base API will be processed +- if it is `v2` then the request will be forwarded internally to the API with API Id `` +- if any other value is provided in the header, then the `default` version will be used (in this instance, the Base API) because `fallbackToDefault` has been configured +- if the header is not provided, then the request will be handled by the `default` version (in this instance the Base API) + +This API version will automatically expire on the 1st January 2030 and stop accepting requests. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out API versioning - though it requires a valid API Id to be used in place of ``. + +--- + +## API versioning in the Tyk Dashboard API Designer + +You can use the API Designer in the Tyk Dashboard to add and manage versions for your APIs. + +### Create a new version of an API + +Starting from a Tyk OAS API (either newly created or an existing non-versioned API) you can easily add a new version by following these steps. + +1. **Create the new version** + + Select **Create a new version** from the **Actions** menu. + + Creating a new version of a Tyk OAS API from the Actions menu + + This will bring up the *Create new API version* pop-up. + + Choosing the version name for a new version of a Tyk OAS API + + Assign your existing version a *name* (e.g. v1), then assign a *name* for the new version (e.g. v2). You should then select one of these to use as the *default* version. + + Select **Create Version** + +2. **Configure the new version** + + You'll be taken to the API designer view for the new version - remember that this will be a completely separate Tyk OAS API Definition for the new version. By default, Tyk Dashboard will have given this API the same name as the original, appending the *version name* that you assigned in **step 1**. + + Tyk will have set the *Access* to *Internal*, so that this version cannot be directly accessed by clients, only via the original (base) API. + + New version of Tyk OAS API has been created + + Configure the new version of your API as required. + + By default, Tyk will have configured the [versioning identifier location](/api-management/api-versioning#version-identifier-location) as Request Header, with the key `x-api-version`. You can modify this if required, but if you're happy with this then that's it - your new API version is ready for use. + + Don't forget to select **Save API**. + +### Working with versioned APIs + +When you have a versioned API the *Base API* will appear in the **Created APIs** list, with an expansion icon that you can select to reveal the versions. + +Versioned API shows in the Created APIs list + +You can see that the *base* and *default* versions are indicated graphically. You can reach the API Designer for each version in the usual way, by selecting the API name in the list (or from the **Actions** menu for the API). + +#### Switch between API versions + +When you are in the API Designer for a versioned API, you can switch between versions using the drop-down next to the API name. + +Choose between API versions in the API designer + +#### Manage version settings + +When you are in the API Designer for a versioned API, there will be a new option in the **Actions** menu: **Manage Versions**. + +New menu option to manage versions + +This takes you to the version management screen for the API. + +Manage the different versions of your API + +You can, from the **Actions** menu: +- delete an API version +- select a new *default* version + +If you select **Edit Settings** you will open a pop-up that allows you to: +- change the versioning identifier (location and/or key) +- set the [versioning identifier pattern](/api-management/api-versioning#version-identifier-pattern) (optional, if using URL path location) +- configure [fallback to default behaviour](/api-management/api-versioning#fallback-to-default) +- configure [version identifier stripping](/api-management/api-versioning#stripping-version-identifier) + +In this example, the version identifier location is set to *Header* allowing configuration of the *Key name*: + +Configure the versioning settings for your API + +--- + +## Versioning with Tyk Classic APIs + +All configuration for versioning of Tyk Classic APIs is documented [here](/api-management/gateway-config-tyk-classic#tyk-classic-api-versioning). + +### Example versioned Tyk Classic API + +Here's an example of the minimal configuration that would need to be added to the API definition for a Tyk Classic API with two versions (`v1` and `v2`): + +```json {linenos=true, linenostart=1} expandable +{ + "version_data": { + "not_versioned": false, + "default_version": "v1", + "versions": { + "v1": { + "name": "v1", + "expires": "", + "paths": { + "ignored": [], + "white_list": [], + "black_list": [] + }, + "use_extended_paths": true, + "extended_paths": { + "ignored": [], + "white_list": [], + "black_list": [], + "transform": [], + "transform_response": [], + "transform_jq": [], + "transform_jq_response": [], + "transform_headers": [], + "transform_response_headers": [], + "hard_timeouts": [], + "circuit_breakers": [], + "url_rewrites": [], + "virtual": [], + "size_limits": [], + "method_transforms": [], + "track_endpoints": [], + "do_not_track_endpoints": [], + "validate_json": [], + "internal": [], + "persist_graphql": [] + }, + "global_headers": {}, + "global_headers_remove": [], + "global_headers_disabled": false, + "global_response_headers": {}, + "global_response_headers_remove": [], + "global_response_headers_disabled": false, + "ignore_endpoint_case": false, + "global_size_limit": 0, + "override_target": "" + }, + "v2": { + "name": "v2", + "expires": "", + "paths": { + "ignored": [], + "white_list": [], + "black_list": [] + }, + "use_extended_paths": true, + "extended_paths": { + "ignored": [], + "white_list": [], + "black_list": [], + "transform": [], + "transform_response": [], + "transform_jq": [], + "transform_jq_response": [], + "transform_headers": [], + "transform_response_headers": [], + "hard_timeouts": [], + "circuit_breakers": [], + "url_rewrites": [], + "virtual": [], + "size_limits": [], + "method_transforms": [], + "track_endpoints": [], + "do_not_track_endpoints": [], + "validate_json": [], + "internal": [], + "persist_graphql": [] + }, + "global_headers": {}, + "global_headers_remove": [], + "global_headers_disabled": false, + "global_response_headers": {}, + "global_response_headers_remove": [], + "global_response_headers_disabled": false, + "ignore_endpoint_case": false, + "global_size_limit": 0, + "override_target": "http://httpbin.org/ip" + } + } + }, + "definition": { + "location": "header", + "key": "x-api-version", + "strip_versioning_data": false, + "fallback_to_default": true, + "url_versioning_pattern": "" + } +} +``` + +In this example, there are two versions of the API +- the version identifier is expected in a request header `x-api-version` +- the versions are named `v1` and `v2` +- the only difference between `v1` and `v2` is that `v2` will proxy the request to a different upstream via the configured `override_target` +- the default version (`default_version`) is `v1` +- if the request header contains an invalid version named (e.g. `v3`), it will be directed to the default (`fallback_to_default:true`) + +### Tyk Classic API versioning in the API Designer + +You can use the API Designer in the Tyk Dashboard to add and manage versions for your Tyk Classic APIs. + +#### Create a versioned API + +1. **Enable versioning** + + In the API Designer, navigate to the **Versions** tab. + + Enabling versioning for a Tyk Classic API + + Deselect the **Do not use versioning** checkbox to enable versioning and display the options. + +2. **Configure the versioning identifier** + + Choose from the drop-down where the version identifier will be located and, if applicable, provide the key name (for query parameter or request header locations). + + Configuring the versioning identifier + +3. **Add a new version** + + You will see the existing (`Default`) version of your API in the **Versions List**. You can add a new version by providing a version name (which will be the value your clients will need to provide in the version location when calling the API). + + You can optionally configure an **Override target host** that will replace the target path that was set in the base configuration for the version. Note that this is not compatible with Service Discovery or Load Balanced settings. + + Select **Add** to create this new version for your API. + + Adding a new version to your API + +4. **Set the default version** + + You can choose any of your API versions to act as the [default](/api-management/api-versioning#default-version). + + Choosing the default version for your API + + Select **Update** to save the changes to your API. + +#### Switch between versions of a Tyk Classic API + +When you are in the API Designer for a versioned Tyk Classic API, you can switch between versions from the **Edit Version** dropdown in the **Endpoint Designer** tab. + +Choosing the API version for which to configure endpoint middleware + +Remember to select **Update** to save the changes to your API. + +

Configuring Tyk Classic API versioning in Tyk Operator

+When using Tyk Operator, you can configure versioning for a Tyk Classic API within `spec.definition` and `spec.version_data`. + +In the following example: + +- the version identifier is a header with the name `x-api-version` (comments demonstrate how to configure the alternative version identifier locations) +- the API has one version with the name `v1` +- the default version is set to `v1` +- an allow list, block list and ignore authentication middleware have been configured for version `v1` +- an alternative upstream URL (`override_target`) is configured for `v1` to send requests to `http://test.org` + +```yaml {linenos=table,hl_lines=["14-17", "25-27", "29-82"], linenostart=1} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: versioned-api +spec: + name: Versioned API + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://version-api.example.com + listen_path: /version-api + strip_listen_path: true + definition: + # Tyk should find version data in Header + location: header + key: x-api-version + + # Tyk should find version data in First URL Element + #location: url + + # Tyk should find version data in URL/Form Parameter + #location: url-param + #key: api-version + version_data: + default_version: v1 + not_versioned: false + versions: + v1: + name: v1 + expires: "" + override_target: "http://test.org" + use_extended_paths: true + extended_paths: + ignored: + - path: /v1/ignored/noregex + method_actions: + GET: + action: no_action + code: 200 + data: "" + headers: + x-tyk-override-test: tyk-override + x-tyk-override-test-2: tyk-override-2 + white_list: + - path: v1/allowed/allowlist/literal + method_actions: + GET: + action: no_action + code: 200 + data: "" + headers: + x-tyk-override-test: tyk-override + x-tyk-override-test-2: tyk-override-2 + - path: v1/allowed/allowlist/reply/{id} + method_actions: + GET: + action: reply + code: 200 + data: flump + headers: + x-tyk-override-test: tyk-override + x-tyk-override-test-2: tyk-override-2 + - path: v1/allowed/allowlist/{id} + method_actions: + GET: + action: no_action + code: 200 + data: "" + headers: + x-tyk-override-test: tyk-override + x-tyk-override-test-2: tyk-override-2 + black_list: + - path: v1/disallowed/blocklist/literal + method_actions: + GET: + action: no_action + code: 200 + data: "" + headers: + x-tyk-override-test: tyk-override + x-tyk-override-test-2: tyk-override-2 +``` + +## Creating Versioned APIs via the Tyk Dashboard and Gateway APIs + +As explained, you can directly [configure the version settings](/api-management/api-versioning#configuring-api-versioning-in-the-api-definition) within the Tyk OAS API definition using the `info.versioning` section of the Tyk Vendor Extension. + +Alternatively, Tyk can look after the linking of base and child versions if you are using the Tyk Dashboard API or Tyk Gateway API to manage your Tyk OAS APIs. + +If you are using Tyk Classic, then you should configure versioning within the API definition prior to creating the API. + +### Creating Base Version + +When creating the [base version](/api-management/api-versioning#base-and-child-apis) of your API, you do not need to do anything special - the version details will be added when you later create the first child version. + +### Creating Child Versions + +When you want to create a [child version](/api-management/api-versioning#base-and-child-apis) for an existing API using the Tyk Dashboard API or Tyk Gateway API, you must provide additional query parameters to link the child and base APIs. + +These parameters are common to the `POST /api/apis/oas` and `POST /tyk/apis/oas` endpoints: + +- `base_api_id`: The API ID of the Base API to which the new version will be linked. +- `base_api_version_name`: The version name of the base API while creating the first version. This doesn't have to be sent for the next versions but if it is set, it will override the base API version name. +- `new_version_name`: The version name of the created version. +- `set_default`: If true, the new version is set as default version. + +These options are also available when [updating an existing API definition](/api-management/gateway-config-managing-oas#updating-an-api) using the `PATCH /api/apis/oas` or `PATCH /tyk/apis/oas` endpoints. + +#### Version Settings + +When using the Tyk Gateway API or Tyk Dashboard API to create new child versions, the default versioning settings will be: + +- versioning identifier location: header +- versioning identifier key: `x-tyk-version` + +If you need to change these, you should do so within the `info.versioning` section of the API definition for the base version as explained [previously](/api-management/api-versioning#configuring-api-versioning-in-the-api-definition). diff --git a/api-management/authentication/basic-authentication.mdx b/api-management/authentication/basic-authentication.mdx new file mode 100644 index 00000000..c604d565 --- /dev/null +++ b/api-management/authentication/basic-authentication.mdx @@ -0,0 +1,194 @@ +--- +title: "Basic Authentication" +'og:description': "How to configure basic authentication in Tyk?" +sidebarTitle: "Basic Authentication" +tags: ['Authentication', 'Authorization', 'Tyk Authentication', 'Tyk Authorization', 'Secure APIs', 'Basic Authentication'] +--- + +## What is Basic Authentication? + +Basic Authentication is a straightforward authentication method where the user's credentials (username and password) are sent to the server, usually in a standard HTTP header. + +## How does Basic Authentication Work? + +The user credentials are combined and encoded in this form: + +``` +Basic base64Encode(username:password) +``` + +A real request could look something like: + +``` +GET /api/widgets/12345 HTTP/1.1 +Host: localhost:8080 +Authorization: Basic am9obkBzbWl0aC5jb206MTIzNDU2Nw== +Cache-Control: no-cache +``` + +In this example the username is `john@smith.com` and the password is `1234567` (see [base64encode.org](https://www.base64encode.org)) + +### The Problem with Basic Authentication + +With Basic Authentication, the authentication credentials are transferred from client to server as encoded plain text. This is not a particularly secure way to transfer the credentials as it is highly susceptible to intercept; as the security of user authentication is usually of critical importance to API owners, Tyk recommends that Basic Authentication should only ever be used in conjunction with additional measures, such as [mTLS](/basic-config-and-security/security/mutual-tls/client-mtls#why-use-mutual-tls). + +## Configuring your API to use Basic Authentication + +The OpenAPI Specification indicates the use of [Basic Authentication](https://swagger.io/docs/specification/v3_0/authentication/basic-authentication/) in the `components.securitySchemes` object using the `type: http` and `scheme: basic`: + +```yaml expandable +components: + securitySchemes: + myAuthScheme: + type: http + scheme: basic + +security: + - myAuthScheme: [] +``` + +With this configuration provided by the OpenAPI description, all that is left to be configured in the Tyk Vendor Extension is to enable authentication, to select this security scheme and to indicate where Tyk should look for the credentials. Usually the credentials will be provided in the `Authorization` header, but Tyk is configurable, via the Tyk Vendor Extension, to support custom header keys and credential passing via query parameter or cookie. + +```yaml expandable +x-tyk-api-gateway: + server: + authentication: + enabled: true + securitySchemes: + myAuthScheme: + enabled: true + header: + enabled: true + name: Authorization +``` + +Note that URL query parameter keys and cookie names are case sensitive, whereas header names are case insensitive. + +You can optionally [strip the user credentials](/api-management/client-authentication#managing-authorization-data) from the request prior to proxying to the upstream using the `authentication.stripAuthorizationData` field (Tyk Classic: `strip_auth_data`). + +### Multiple User Credential Locations + +The OpenAPI Specification's `securitySchemes` mechanism allows only one location for the user credentials, but in some scenarios an API might need to support multiple potential locations to support different clients. + +The Tyk Vendor Extension supports this by allowing configuration of alternative locations in the basic auth entry in `server.authentication.securitySchemes`. Building on the previous example, we can add optional query and cookie locations as follows: + +```yaml expandable +x-tyk-api-gateway: + server: + authentication: + enabled: true + securitySchemes: + myAuthScheme: + enabled: true + header: + enabled: true + name: Authorization + query: + enabled: true + name: query-auth + cookie: + enabled: true + name: cookie-auth +``` + +### Extract Credentials from the Request Payload + +In some cases, for example when dealing with SOAP, user credentials can be passed within the request body rather in the standard Basic Authentication format. You can configure Tyk to handle this situation by extracting the username and password from the body using regular expression matching (regexps). + +You must instruct Tyk to check the request body by adding the `extractCredentialsFromBody` field to the basic auth entry in `server.authentication.securitySchemes`, for example: + +```yaml expandable +x-tyk-api-gateway: + server: + authentication: + enabled: true + securitySchemes: + myAuthScheme: + enabled: true + extractCredentialsFromBody: + enabled: true + userRegexp: '(.*)' + passwordRegexp: '(.*)' +``` + +Note that each regexp should contain only one match group, which must point to the actual values of the user credentials. + +### Caching User Credentials + +The default behaviour of Tyk's Basic Authentication middleware is to cache user credentials, improving the performance of the authentication step when a client makes frequent requests on behalf of the same user. + +When a request is received, it presents credentials which are checked against the users registered in Tyk. When a match occurs and the request is authorized, the matching credentials are stored in a cache with a configurable refresh period. When future requests are received, Tyk will check the presented credentials against those in the cache first, before checking the full list of registered users. + +The cache will refresh after `cacheTTL` seconds (Tyk Classic: `basic_auth.cache_ttl`). + +If you do not want to cache user credentials, you can turn this off using `disableCaching` in the basic auth entry in `server.authentication.securitySchemes` (Tyk Classic: `basic_auth.disable_caching`). + +### Using Tyk Classic APIs + +As noted in the Tyk Classic API [documentation](/api-management/gateway-config-tyk-classic#configuring-authentication-for-tyk-classic-apis), you can select Basic Authentication using the `use_basic_auth` option. This will default to expect the user credentials in the `Authorization` header. + + +## Using Tyk Dashboard to Configure Basic Authentication + +Using the Tyk Dashboard, you can configure the Basic Authentication method from the Server section in the API Designer by enabling **Authentication** and selecting **Basic Authentication** from the drop-down: + +Target Details: Basic Auth + +- select the location(s) where Tyk should look for the token +- provide the key name for each location (we prefill the default `Authorization` for the *header* location, but you can replace this if required) +- optionally select [strip authorization data](/api-management/client-authentication#managing-authorization-data) to remove the auth token locations from the request prior to proxying to the upstream +- optionally configure the [basic authentication cache](/api-management/authentication/basic-authentication#caching-user-credentials) +- optionally configure [extraction of credentials from the request body](#extract-credentials-from-the-request-payload) + +## Registering Basic Authentication User Credentials with Tyk + +When using Basic Authentication, the API key used to access the API is not generated by the Tyk system, instead you need to create and register the credentials of your users with Tyk. Tyk will compare the credentials provided in the request against the list of users you have created. + +The way that this is implemented is through the creation of a key that grants access to the API (as you would for an API protected by [auth token](/api-management/authentication/bearer-token)), however for this key you will provide a username and password. + +When calling the API, users would never use the key itself as a token, instead their client must provide the Basic Auth credentials formed from the registered username and password, as [described previously](#how-does-basic-authentication-work). + + +### Using Tyk Dashboard UI + +You can use the Tyk Dashboard to register a user's Basic Authentication credentials that can then be used to access your API. + +Navigate to the **Keys** screen and select **Add Key**. + +Follow the instructions in the [access key guide](/api-management/gateway-config-managing-classic#access-an-api) and you'll notice that, when you select the Basic Auth protected API, a new **Authentication** tab appears: + +Note that the **Authentication** tab will also be displayed if you create a key from a policy that grants access to a Basic Auth protected API. + +Complete the user's credentials on this tab and create the key as normal. The key that is created in Tyk Dashboard is not in itself an access token (that is, it cannot be used directly to gain access to the API) but is used by Tyk to validate the credentials provided in the request and to determine the appropriate authorization, including expiry of authorization. + +### Using the Tyk Dashboard API + +You can register user credentials using the `POST /api/apis/keys/basic/{username}` endpoint in the [Tyk Dashboard API](/tyk-dashboard-api). The request payload is a [Tyk Session Object](/api-management/policies#what-is-a-session-object) (access key). + +- the user's *username* is provided as a path parameter +- the user's *password* is provided as `basic_auth_data.password` within the request payload + +You use the `POST` method to create a new user and `PUT` to update an existing entry. + + + +Be careful to ensure that the `org_id` is set correctly and consistently so that the Basic Authentication user is created in the correct organization. + + + +### Using the Tyk Gateway API + +You can register user credentials using the `POST /tyk/keys/{username}` endpoint in the [Tyk Dashboard API](/tyk-dashboard-api). The request payload is a [Tyk Session Object](/api-management/policies#what-is-a-session-object) (access key). + +- the user's *username* is provided as a path parameter +- the user's *password* is provided as `basic_auth_data.password` within the request payload + +You use the `POST` method to create a new user and `PUT` to update an existing entry. + + + +Be careful to ensure that the `org_id` is set correctly and consistently so that the Basic Authentication user is created in the correct organization. + + + + diff --git a/api-management/authentication/bearer-token.mdx b/api-management/authentication/bearer-token.mdx new file mode 100644 index 00000000..28cb6af4 --- /dev/null +++ b/api-management/authentication/bearer-token.mdx @@ -0,0 +1,105 @@ +--- +title: "Bearer Tokens" +'og:description': "How to configure bearer tokens in Tyk?" +sidebarTitle: "Bearer Tokens" +tags: ['Authentication', 'Authorization', 'Tyk Authentication', 'Tyk Authorization', 'Secure APIs', 'Bearer Tokens'] +--- + +## What is a bearer token ? + +> Any party in possession of an auth (or bearer) token (a "bearer") can use it to get access to the associated resources (without demonstrating possession of a cryptographic key). To prevent misuse, auth tokens need to be protected from disclosure in storage and in transport. + +Tyk provides auth (bearer) token access as one of the most convenient building blocks for managing security to your API. Tokens are added to a request as a header or as a query parameter. If added as a header, they may be preceded by the word "Bearer" to indicate their type, though this is optional. Usually these tokens are provided in the `Authorization` header, however Tyk can be configured to accept the token in a different header, as a query parameter or in a cookie. + +## Configuring your API to use Auth Token + +The OpenAPI Specification indicates the use of [Auth Token](https://swagger.io/docs/specification/v3_0/authentication/bearer-authentication/) in the `components.securitySchemes` object using `type: apiKey`. It also includes specification of the location (`in`) and key (`name`) that are to be used when providing the token to the API, for example: + +```yaml expandable +components: + securitySchemes: + myAuthScheme: + type: apiKey + in: header + name: Authorization + +security: + - myAuthScheme: [] +``` + +With this configuration provided by the OpenAPI description, all that is left to be configured in the Tyk Vendor Extension is to enable authentication and to select this security scheme. + +```yaml expandable +x-tyk-api-gateway: + server: + authentication: + enabled: true + securitySchemes: + myAuthScheme: + enabled: true +``` + +Note that URL query parameter keys and cookie names are case sensitive, whereas header names are case insensitive. + +You can optionally [strip the auth token](/api-management/client-authentication#managing-authorization-data) from the request prior to proxying to the upstream using the `authentication.stripAuthorizationData` field (Tyk Classic: `strip_auth_data`). + +## Multiple Auth Token Locations + +The OpenAPI Specification's `securitySchemes` mechanism allows only one location for the auth token, but in some scenarios an API might need to support multiple potential locations to support different clients. + +The Tyk Vendor Extension supports this by allowing configuration of alternative locations in the auth token entry in `server.authentication.securitySchemes`. Building on the previous example, we can add optional query and cookie locations as follows: + +```yaml expandable +x-tyk-api-gateway: + server: + authentication: + enabled: true + securitySchemes: + myAuthScheme: + enabled: true + query: + enabled: true + name: query-auth + cookie: + enabled: true + name: cookie-auth +``` + +## Dynamic mTLS with Auth Token +The Auth Token method can support [Dynamic mTLS](/basic-config-and-security/security/mutual-tls/client-mtls#dynamic-mtls) where the client can provide a TLS certificate in lieu of a standard Auth Token. This can be configured for an API using the [enableClientCertificate](/api-management/gateway-config-tyk-oas#token) option (Tyk Classic: `auth.use_certificate`). + +## Auth Token with Signature + +If you are migrating from platforms like Mashery, which use request signing, you can enable signature validation alongside auth token by configuring the additional [signatureValidation](/api-management/gateway-config-tyk-oas#token) field (Tyk Classic: `auth.signature`). + +You can configure: + +- the location of the signature +- the algorithm used to create the signature (`MasherySHA256` or `MasheryMD5`) +- secret used during signature +- an allowable clock skew + +## Using Custom Auth Tokens + +If you have your own identity provider you may want to use that to generate and manage the access tokens, rather than having Tyk generate the tokens. You can use the `POST /tyk/keys/{keyID}` endpoint in the [Tyk Gateway API](/tyk-gateway-api) to import those tokens to Tyk, off-loading access control, quotas and rate limiting from your own application. + +## Using Tyk Dashboard to Configure Auth Token + +Using the Tyk Dashboard, you can configure the Auth Token authentication method from the Server section in the API Designer by enabling **Authentication** and selecting **Auth Token** from the drop-down: + +Configuring the Auth Token method + +- select the location(s) where Tyk should look for the token +- provide the key name for each location (we prefill the default `Authorization` for the *header* location, but you can replace this if required) +- select **Strip authorization data** to remove the auth token locations from the request prior to proxying to the upstream, as described [here](/api-management/client-authentication#managing-authorization-data) +- optionally select **Enable client certificate** to enable [Dynamic mTLS](/basic-config-and-security/security/mutual-tls/client-mtls#dynamic-mtls) for the API, so the client can provide a certificate in place of the token + +Note that the [auth token + signature](/api-management/authentication/bearer-token#auth-token-with-signature) option is not available in the Tyk Dashboard API Designer. + + +## Using Tyk Classic APIs + +As noted in the Tyk Classic API [documentation](/api-management/gateway-config-tyk-classic#configuring-authentication-for-tyk-classic-apis), a new Tyk Classic API will use the auth (bearer) token method by default with the token expected in the `Authorization` header, so configuration is slightly different as there is no need to `enable` this method. You should configure the `auth` object for any non-default settings, such as a different token location or Dynamic mTLS. + + + diff --git a/api-management/authentication/custom-auth.mdx b/api-management/authentication/custom-auth.mdx new file mode 100644 index 00000000..07bb7daa --- /dev/null +++ b/api-management/authentication/custom-auth.mdx @@ -0,0 +1,19 @@ +--- +title: "Custom Authentication" +'og:description': "How to implement custom authentication in Tyk using Go plugins, Python CoProcess, and JSVM plugins." +sidebarTitle: "Custom Authentication" +tags: ['Authentication', 'Authorization', 'Tyk Authentication', 'Tyk Authorization', 'Go Plugins', 'Python CoProcess', 'JSVM Plugin'] +--- + +## Go Plugins + +Go Plugin Authentication allows you to implement custom authentication logic using the Go programming language. This method is useful for scenarios where you need to implement specialized authentication mechanisms that are not natively supported by Tyk. +To learn more about using Tyk Golang Plugins, go [here](/api-management/plugins/golang) + +## Use Python CoProcess and JSVM Plugin Authentication + +Tyk allows for custom authentication logic using Python and JavaScript Virtual Machine (JSVM) plugins. This method is useful for implementing unique authentication mechanisms that are tailored to your specific requirements. + +* See [Custom Authentication with a Python plugin](/api-management/plugins/rich-plugins#custom-authentication-plugin-tutorial) for a detailed example of a custom Python plugin. +* See [JavaScript Middleware](/api-management/plugins/javascript#) for more details on using JavaScript Middleware. + diff --git a/api-management/authentication/oauth-2.mdx b/api-management/authentication/oauth-2.mdx new file mode 100644 index 00000000..91bec464 --- /dev/null +++ b/api-management/authentication/oauth-2.mdx @@ -0,0 +1,639 @@ +--- +title: "OAuth 2.0" +'og:description': "How to configure OAuth 2.0 in Tyk?" +sidebarTitle: "OAuth 2.0" +tags: ['Authentication', 'Authorization', 'Tyk Authentication', 'Tyk Authorization', 'Secure APIs', 'OAuth 2.0'] +--- + +## Use Tyk as an OAuth 2.0 Authorization Server + +Tyk can act as an OAuth 2.0 *authorization server*, performing token generation and management for *clients* accessing APIs deployed on Tyk. There are many great resources on the Internet that will help you to understand the OAuth 2.0 Authorization Framework, which we won't attempt to duplicate here. We will provide a basic introduction to the [concepts and terminology](#oauth-20-core-concepts) before we dive into the details of using Tyk as your *auth server*. + +Tyk offers some great features when used as the *authorization server* including: + +- **Fine-Grained Access Control:** Manage access using Tyk's built-in access controls, including versioning and named API IDs +- **Usage Analytics:** Leverage Tyk's analytics capabilities to monitor OAuth 2.0 usage effectively, grouping data by Client Id +- **Multi-API Access**: Enable access to multiple APIs using a single OAuth token; configure one API for OAuth 2.0 token issuance and the other APIs with the [Auth Token](/api-management/authentication/bearer-token) method, linking them through a common policy + +*Tyk as OAuth authorization server* supports the following *grant types*: + +- [Authorization Code Grant](#using-the-authorization-code-grant): the *client* is redirected to an *identity server* where the *user* must approve access before an *access token* will be issued +- [Client Credentials Grant](#using-the-client-credentials-grant): used for machine-to-machine access, authentication is performed using only the *client Id* and *client secret* +- [Resource Owner Password Grant](#using-the-resource-owner-password-grant) (a.k.a. Password Grant): only for use where the *client* is highly trusted, as the *client* must provide the *Resource Owner*'s own credentials during authentication + + + + + **Tyk does not recommend the use of Resource Owner Password Grant**. This method is considered unsafe and is prohibited in the [OAuth 2.0 Security Best Practice](https://datatracker.ietf.org/doc/html/draft-ietf-oauth-security-topics-13#section-3.4") but is supported for use with legacy clients. + + + +To make use of this, you'll need to: + +- understand how to integrate your *client* (and, for Authorization Code grant, your *identity server*) according to the OAuth grant type +- [register a client app](#client-app-registration) for each client that needs to access the API +- [configure your API proxy](#configuring-your-api-proxy) to use the *Tyk OAuth 2.0* authentication method + +{/* TODO: This video probably needs to be re-recorded with Tyk OAS, so not publishing for now: */} + + + +## OAuth 2.0 Core Concepts + +**OAuth 2.0** (Open Authorization 2.0) is a widely adopted authorization protocol that allows third-party applications to access user resources securely, without needing to expose sensitive credentials such as user passwords. It is an industry-standard framework that enables a delegated approach to securing access to APIs and services. The [IETF OAuth 2.0 specification](https://datatracker.ietf.org/doc/html/rfc6749) outlines the standard for OAuth 2.0. + +> "The OAuth 2.0 authorization framework enables a third-party application to obtain limited access to an HTTP service, either on behalf of a resource owner by orchestrating an approval interaction between the resource owner and the HTTP service, or by allowing the third-party application to obtain access on its own behalf." β€” [RFC 6749](https://datatracker.ietf.org/doc/html/rfc6749) + +OAuth 2.0 provides a mechanism for **client applications** to request limited access to resources hosted by a **resource server**, on behalf of a **resource owner** (typically a user), without exposing the resource owner's credentials. This allows secure sharing of data between applicationsβ€”for example, allowing a calendar app to access a user's contacts to automatically find available time slots for meetings. + +OAuth 2.0 has many variations and flows suited for different use cases, this section will provide an overview of the core principles, terminology, and key concepts, specifically focusing on how you can implement OAuth 2.0 with Tyk. + +### Terminology + +- **Protected Resource**: The service or data that is protected by OAuth (e.g. an API endpoint) and requires authorization to access. +- **Resource Owner**: The **user** or system that owns the *Protected Resource* and has the ability to grant or deny access to it. +- **Client**: The application or system that seeks access to the *Protected Resource*. It acts on behalf of the *Resource Owner*. +- **Access Token**: A short-lived piece of data that grants the *Client* access to the *Protected Resource*. The token proves that the *Client* has been authorized by the *Resource Owner*. +- **Authorization Server**: The server that issues *Access Tokens* to the *Client* after validating the *Client*'s identity and obtaining consent from the *Resource Owner*. +- **Client Application**: The application that requests authorization from the *Authorization Server*. This application must first be registered with the *Authorization Server* to obtain credentials (*Client Id* and *Client Secret*). +- **Resource Server**: The server that hosts the *Protected Resource*. It receives access requests from *Clients*, which must include a valid *Access Token*. +- **Identity Server**: A server that authenticates the *Resource Owner*, offering the facility to log in and authorize *Client* access to *Protected Resources*. +- **Scope**: Defines the specific permissions or access levels being requested by the *Client* (e.g. read, write, delete). +- **Grant Type**: The method by which the *Client* obtains an *Access Token*, based on the OAuth flow being used (e.g. Authorization Code, Client Credentials, Resource Owner Password Credentials). + +### Access Tokens + +In OAuth 2.0, **access tokens** are used to represent the authorization granted to the *client* by the *resource owner*. These tokens are typically small, opaque data objects that are passed along with each API request to authenticate the *client*. While the OAuth 2.0 specification does not mandate a specific format, **JSON Web Tokens (JWTs)** are commonly used as they can encode metadata, such as the *user*'s identity, permissions, and token expiry time. + +Tokens usually come with an expiration date to limit the time they are valid and minimize the risk of abuse. *Access tokens* can often be refreshed via a **refresh token** if they expire, allowing for long-lived access without requiring the *user* (*resource owner*) to reauthorize the *application* (*client*). + +### Client Application + +For a *client* to request an *Access Token* from the *Authorization Server*, it must first authenticate itself. This ensures that the *Resource Owner* can confidently delegate access to the requested resources. + +To do this, the *client* is registered with the *Authorization Server* as a **Client Application**, which requires the following elements: + +- **Client Id**: A unique, public identifier for the *client application* (e.g., a username or application name). +- **Client Secret**: A confidential string (like a password) that is shared between the *client* and the *Authorization Server*. The *client secret* is never exposed to the *Resource Owner*. +- **Redirect URI**: The URL to which the *client* will be redirected after the authorization process is complete (either granted or denied). + +The *client* sends the *client Id* and *client secret* during the authorization request to prove its identity and authenticate its request for an *access token*. Depending on the OAuth *grant type* being used (e.g. Authorization Code Flow, Client Credentials Flow), the *Authorization Server* will authenticate the *client* and, if successful, issue an *Access Token*. + + +## Manage Client Access Policies + +The *access tokens* issued to clients by *Tyk Authorization Server* are the same as other [session objects](/api-management/policies#what-is-a-session-object) and can be associated with [access security policies](/api-management/policies#what-is-a-security-policy) at the point of creation. These allow the application of quotas, rate limits and access rights in the normal manner. + +Security policies can be assigned to *client apps* and will be applied to all access tokens issued for that *client app*. + + +## Client App Registration + +For all grant types, the first common step is the registration of the *client* with Tyk Dashboard by creation of a *Client App*. This will allocate a *client Id* and *client secret* that must be provided in future authentication requests by the *client*. + +### Using the Tyk Dashboard UI + +1. *Client apps* are registered per-API, so the first step is to [configure Tyk OAuth 2.0](#configuring-your-api-proxy) as the security method to be used for the API. With this done, you can navigate to the OAuth Client management screen for the API from the **Actions** menu on the **Created APIs** screen: + +Accessing the list of OAuth Clients for an API + +2. You will now be prompted to register a *client app* that will be granted access to the API configuring: + +- redirect URI +- [optional] [security policies](#manage-client-access-policies) to be applied to access tokens generated for the client +- [optional] [metadata](/api-management/policies#what-is-a-session-metadata) to be added to the access tokens + +Add New OAuth Client + +**Note**: when using *Authorization Code grant* the *redirect uri* configured for the *client app* must be the same as that configured in the API definition. + +Select the **Create** button to register the *client app*. + +3. In the OAuth Client management screen, you will see a list of *client apps* registered with the API (as identified by their *client Id*). By clicking on the list item, or from the **Actions** menu's **Edit** option you will be taken to the *Edit Client app* screen, where you can see the *client secret* and make any modifications you need. There is also the option to [revoke tokens](#revoking-access-tokens) that have been issued for this *client app*. + +View client Id and client secret + +### Using the Tyk Dashboard API + +The Tyk Dashboard API contains several endpoints that are provided to manage *client apps*. *Client apps* are registered per-API, so each takes as an input the *API Id* for the API: + +| Action | Endpoint | Reference | +| :--- | :--- | :--- | +| Register a new client app | `POST /api/apis/oauth/{{api-id}}` | [link](/api-management/dashboard-configuration#create-a-new-oauth20-client) | +| Get a list of registered client apps | `GET /api/apis/oauth/{{api-id}}` | [link](/api-management/dashboard-configuration#list-oauth-clients) | +| Get the details of a client app | `GET /api/apis/oauth/{{api-id}}/{{client_id}}` | [link](/api-management/dashboard-configuration#get-an-oauth20-client) | +| Delete a client app | `DELETE /api/apis/oauth/{{api-id}}/{{client_id}}` | [link](/api-management/dashboard-configuration#delete-oauth-client) | + + +## Using the Authorization Code Grant + +When using Tyk as the Authorization Server with the Authorization Code grant, the following steps are followed after [registering the Client App](#client-app-registration): + +Authorization grant type flow + +**Explanatory notes:** + +(1) *client* makes a request to the [authorization endpoint](#authorization-request) on the *Auth Server* + +(2) The *Auth Server* notes the request parameters and returns `HTTP 307 Temporary Redirect`, redirecting the user to an *Identity Server* + +(5) the *user* must log in on the *Identity Server* and authorize the *client* + +(6) when the *user* successfully authenticates and authorizes the request, the *Identity Server* must request an [Authorization Code](#authorization-code-request) from the *Auth Server* + +(8) The *Identity Server* provides the *Authorization Code* to the *client* + +(9) The *client* exchanges the *Authorization Code* for an [Access Token](#exchange-the-authorization-code-for-an-access-token) from the *Auth Server* + +(10) The *client* uses the *Access Token* to authenticate with the protected API using the [Auth Token](/api-management/authentication/bearer-token) method + +### Integration with Identity Server + +Whilst Tyk can provide the *authorization server* functionality, issuing and managing access and authorization tokens, the *identity server* functions (authenticating users (resource owners) and allowing them to authorize client access) must be performed by a separate Identity Provider (IdP). + +The identity server will need access to the Tyk Dashboard API to [obtain an Authorization Code](/api-management/dashboard-configuration#oauth20-authorization-code). + +### Authorization Request + +The authorization endpoint for an API proxy on Tyk is a special endpoint automatically added to the proxy definition, accessible from `POST //oauth/authorize` + +The following parameters are required in a request to this endpoint: + +| Parameter | Value | +| :--------------- | :-------------------------- | +| `response_type` | `code` | +| `client_id` | client Id | +| `redirect_uri` | Redirect URI (URL encoded) | + +For example: + +```bash +curl -X POST https://tyk.cloud.tyk.io/my-api/oauth/authorize/ \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "response_type=code&client_id=my-client-id&redirect_uri=http%3A%2F%2Fidentityserver.com%2Fclient-redirect-uri" +``` + +This command, issued by the *client* is the first step of requesting access to the `/my-api` proxy deployed on a Tyk Gateway at `https://tyk.cloud.tyk.io`. + +If the *client Id* (`my-client-id`) is valid, the response will be `HTTP 307 Temporary Redirect` with the redirect URI (`http://identityserver.com/client-redirect-uri`) in the `location` header. + +### Authorization Code Request + +The *Identity Server* requests an *Authorization Code* from the *Authentication Server*. Tyk's *authorization code* endpoint is hosted in the [Tyk Dashboard API](/api-management/dashboard-configuration#oauth20-authorization-code), accessible from `POST /api/apis/{api_id}/authorize-client`. The same `redirect_uri` as provided in the original request must be provided alongside the `client_id` as a security feature to verify the client identity. + +This endpoint is protected using the Dashboard API secret assigned to the *Identity Server*, which must be provided in the `Authorization` header. + +The following parameters are required in a `POST` request to this endpoint: + +| Parameter | Value | +| :--------------- | :-------------------------- | +| `response_type` | `code` | +| `client_id` | client Id | +| `redirect_uri` | Redirect URI (URL encoded) | + +For example: + +```bash +curl -X POST \ + https://admin.cloud.tyk.io/api/apis/oauth/{my-api-id}/authorize-client/ \ + -H "Authorization: " \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "response_type=code&client_id=my-client-id&redirect_uri=http%3A%2F%2Fidentityserver.com%2Fclient-redirect-uri" +``` + +This command, issued by the *identity server* requests an *authorization code* from the Tyk Dashboard at `https://admin.cloud.tyk.io` to access the proxy with API Id `my-api-id`. + +If the *client Id* (`my-client-id`) is valid and `redirect_uri` matches the one provided in the initial request, an *authorization code* will be provided in the response payload, for example: + +```json +{ + "code": "EaG1MK7LS8GbbwCAUwDo6Q", + "redirect_to": "http://example.com/client-redirect-uri?code=EaG1MK7LS8GbbwCAUwDo6Q" +} +``` + +### Exchange the Authorization Code for an Access Token + +Once the *client* has the *authorization code*, it can exchange this for an *access token*, which is used to access the protected API. The token exchange endpoint for an API proxy on Tyk is a special endpoint automatically added to the proxy definition, accessible from `POST //oauth/token`. + +This endpoint is protected using [Basic Authentication](/api-management/authentication/basic-authentication) where the username is the *client Id* and the password is the *client secret*. + +The following parameters are required in the request: + +| Parameter | Value | +| :--------------- | :-------------------------- | +| `grant_type` | `authorization_code` | +| `client_id` | client Id | +| `code` | Authorization Code | +| `redirect_uri` | Redirect URI (URL encoded) | + +For example: + +```bash +curl -X POST \ + https://tyk.cloud.tyk.io/my-api/oauth/token/ \ + -H "Authorization: Basic bXktY2xpZW50LWlkOm15LWNsaWVudC1zZWNyZXQ=" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=authorization_code&client_id=my-client-id&code=EaG1MK7LS8GbbwCAUwDo6Q&redirect_uri=http%3A%2F%2Fidentityserver.com%2Fclient-redirect-uri" +``` + +This command, issued by the *client* is the final step to obtain an access token for the `/my-api` proxy deployed on a Tyk Gateway at `https://tyk.cloud.tyk.io`. The basic auth key is the base64 encoded representation of `my-client-id:my-client-secret` The `client_id` and `redirect_uri` match those provided in the initial [authorization request](#authorization-request). The `code` is the *authorization code* provided to the *identity server* in the [authorization code request](#authorization-code-request). + +The response payload contains: +- `access_token`: the token which can be used by the *client* to access the protected API +- `expires_in`: the expiration date/time of the access token +- `token_type`: set to `bearer` indicating that the access token should be provided in an [Auth Token](/api-management/authentication/bearer-token) request to the protected API +- `refresh_token`: [optional] a special token that can be used in the [Refresh Token](#using-refresh-tokens) flow + +For example: + +```json expandable +{ + "access_token": "580defdbe1d21e0001c67e5c2a0a6c98ba8b4a059dc5825388501573", + "expires_in": 3600, + "refresh_token": "NWQzNGVhMTItMDE4Ny00MDFkLTljOWItNGE4NzI1ZGI1NGU2", + "token_type": "bearer" +} +``` + + + +## Using the Client Credentials Grant +When using Tyk as the *authorization server* with the Client Credentials grant, the *client* accesses resources on behalf of itself rather than on behalf of a *user*, so there is no user login/authorization step (as seen with [Authorization Code grant](#using-the-authorization-code-grant)). This flow is ideal for server-to-server interactions. + +After [registering the Client App](#client-app-registration), the *client* simply requests an access token directly from the authorization server: + +Client Credentials grant type flow + +### Access Token Request + +The *client* obtains an access token for an API proxy on Tyk from a special endpoint automatically added to the proxy definition, accessible from `POST //oauth/token`. + +This endpoint is protected using Basic Authentication where the username is the client Id and the password is the client secret. + +The following parameters are required in the request: + +| Parameter | Value | +| :--------------- | :-------------------------- | +| `grant_type` | `client_credentials` | +| `client_id` | client Id | +| `secret` | client secret | + +For example: + +```bash +curl -X POST \ + https://tyk.cloud.tyk.io/my-api/oauth/token/ \ + -H "Authorization: Basic bXktY2xpZW50LWlkOm15LWNsaWVudC1zZWNyZXQ=" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=client_credentials&client_id=my-client-id&client_secret=my-client-secret" +``` + +This command, issued by the *client* will obtain an access token for the `/my-api` proxy deployed on a Tyk Gateway at `https://tyk.cloud.tyk.io`. The basic auth key is the base64 encoded representation of `my-client-id:my-client-secret` The `client_id` and `client_secret` match those allocated by Tyk (the auth server) for the *client app*. + +The response payload contains: +- `access_token`: the token which can be used by the *client* to access the protected API +- `expires_in`: the expiration date/time of the access token +- `token_type`: set to `bearer` indicating that the access token should be provided in an [Auth Token](/api-management/authentication/bearer-token) request to the protected API + +For example: + +```json +{ + "access_token": "580defdbe1d21e0001c67e5c2a0a6c98ba8b4a059dc5825388501573", + "expires_in": 3600, + "token_type": "bearer" +} +``` + + + +Note that Client Credentials grant does not produce a *refresh token*. + + + + + +## Using the Resource Owner Password Grant +When using Tyk as the *authorization server* with the Resource Owner Password grant, the *client* provides the *user's* credentials when requesting an access token. There is no user login/authorization step (as seen with [Authorization Code grant](#using-the-authorization-code-grant)). **This flow is not recommended and is provided only for integration with legacy clients.** + +After [registering the Client App](#client-app-registration), the *client* simply requests an access token directly from the authorization server: + +Username and password grant sequence + +### Access Token Request + +The *client* obtains an access token for an API proxy on Tyk from a special endpoint automatically added to the proxy definition, accessible from `POST //oauth/token`. + +This endpoint is protected using [Basic Authentication](/api-management/authentication/basic-authentication) where the username is the client Id and the password is the client secret. + +The following parameters are required in the request: + +| Parameter | Value | +| :--------------- | :------------------------------------------------------ | +| `grant_type` | `password` | +| `client_id` | client Id | +| `username` | resource owner's username (`resource-owner-username`) | +| `password` | resource owner's password (`resource-owner-password`) | + +For example: + +```bash +curl -X POST \ + https://tyk.cloud.tyk.io/my-api/oauth/token/ \ + -H "Authorization: Basic bXktY2xpZW50LWlkOm15LWNsaWVudC1zZWNyZXQ=" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=password&client_id=my-client-id&username=resource-owner-username&password=resource-owner-password" +``` + +This command, issued by the *client* will obtain an access token for the `/my-api` proxy deployed on a Tyk Gateway at `https://tyk.cloud.tyk.io`. The basic auth key is the base64 encoded representation of `my-client-id:my-client-secret` The `client_id` and `client_secret` match those allocated by Tyk (the auth server) for the *client app*. + +The response payload contains: +- `access_token`: the token which can be used by the *client* to access the protected API +- `expires_in`: the expiration date/time of the access token +- `token_type`: set to `bearer` indicating that the access token should be provided in an [Auth Token](/api-management/authentication/bearer-token) request to the protected API +- `refresh_token`: [optional] a special token that can be used in the [Refresh Token](#using-refresh-tokens) flow + +For example: + +```json expandable +{ + "access_token": "580defdbe1d21e0001c67e5c2a0a6c98ba8b4a059dc5825388501573", + "expires_in": 3600, + "refresh_token": "YjdhOWFmZTAtNmExZi00ZTVlLWIwZTUtOGFhNmIwMWI3MzJj", + "token_type": "bearer" +} +``` + + +## Configuring your API Proxy + +As explained [previously](/api-management/client-authentication#how-does-tyk-implement-authentication-and-authorization), the AuthN/Z methods to be used to secure an API proxy are configured in the API definition. This permits granular application of the most appropriate method to each API deployed on Tyk Gateway. + +When using Tyk as the Authorization Server, the API configuration can be applied using the Tyk Dashboard's API Designer UI, or by direct modification of the API definition. We will provide examples here when using Tyk OAS APIs. If you are using Tyk Classic APIs, the process is very similar, though there are differences in the location and specific labelling of options. + +### Using the Tyk API Designer + +1. Client Authentication is configured on the **Settings** screen within the API Designer, within the **Server** section. Ensure that you are in **Edit** mode, click on the button to **Enable** *Authentication* and then select **Tyk OAuth 2.0** from the drop down options: + +Set Authentication Mode + +2. Select the OAuth Grant Type that you wish to use for the API, if appropriate you can also select the *Refresh Token* grant so that the Auth Server (Tyk) will generate both access and refresh tokens. + +3. Provide the requested configuration options depending on the selected Grant Type. Note that for *Authorization Code Grant*, **Redirect URL** should be the login page for your Identity Server and must be matched by the `redirect_uri` provided in the *client app* (and in the client's authentication request). The [Notifications](#oauth-token-notifications) configuration can be provided for *Authorization Code* and *Password* grants. + +4. Select **Save API** to apply the new settings. + +### Using the API Definition + +The OpenAPI Specification indicates the use of [OAuth 2.0 authentication](https://swagger.io/docs/specification/v3_0/authentication/oauth2/) in the `components.securitySchemes` object using the `type: oauth2`. Tyk supports the [authorizationCode](/api-management/authentication/oauth-2#using-the-authorization-code-grant), [clientCredentials](#using-the-client-credentials-grant) and [password](#using-the-resource-owner-password-grant) flows and implements Relative Endpoint URLs for the `authorizationUrl`, `tokenUrl` and `refreshUrl`. + +```yaml expandable +components: + securitySchemes: + myAuthScheme: + type: oauth2 + flows: + authorizationCode: + authorizationUrl: ... + tokenUrl: ... + scopes: ... + +security: + - myAuthScheme: [] +``` + +With this configuration provided by the OpenAPI description, in the Tyk Vendor Extension we need to enable authentication, to select this security scheme and to indicate where Tyk should look for the OAuth token. Usually the token will be provided in the `Authorization` header, but Tyk is configurable, via the Tyk Vendor Extension, to support custom header keys and credential passing via query parameter or cooke. + +```yaml expandable +x-tyk-api-gateway: + server: + authentication: + enabled: true + securitySchemes: + myAuthScheme: + enabled: true + header: + enabled: true + name: Authorization +``` + +Note that URL query parameter keys and cookie names are case sensitive, whereas header names are case insensitive. + +You can optionally [strip the user credentials](/api-management/client-authentication#managing-authorization-data) from the request prior to proxying to the upstream using the `authentication.stripAuthorizationData` field (Tyk Classic: `strip_auth_data`). + +With the OAuth method selected, you'll need to configure Tyk to handle the specific configuration of OAuth grants that you will support. All of the OAuth specific configuration is performed within the [authentication.securitySchemes.oauth](/api-management/gateway-config-tyk-oas#oauth) object in the Tyk Vendor Extension. + +For example: + +```json {hl_lines=["7-11", "14-24", "35-55"],linenos=true, linenostart=1} expandable +{ + "info": { + "title": "My OAuth API", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "security": [ + { + "oauth": [] + } + ], + "paths": {}, + "components": { + "securitySchemes": { + "oauth": { + "type": "oauth2", + "flows": { + "authorizationCode": { + "authorizationUrl": "/oauth/authorize", + "scopes": {}, + "tokenUrl": "/oauth/token" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "My OAuth API", + "state": { + "active": true, + } + }, + "server": { + "authentication": { + "enabled": true, + "securitySchemes": { + "oauth": { + "enabled": true, + "allowedAuthorizeTypes": [ + "code" + ], + "authLoginRedirect": "http:///client-redirect-uri", + "header": { + "enabled": true, + "name": "Authorization" + }, + "notifications": { + "onKeyChangeUrl": "http://notifyme.com", + "sharedSecret": "oauth-shared-secret" + }, + "refreshToken": true + } + } + }, + "listenPath": { + "strip": true, + "value": "/my-oauth-api/" + } + }, + "upstream": { + "url": "http://httpbin.org/" + } + } +} +``` + +In this example: + +- Client authentication has been enabled (line 44) +- The OpenAPI description declares the `oauth` security scheme that expects **Authorization Code** flow. Note that the `authorization URL` and `token URL` are declared relative to the API proxy listen path +- Authorization requests (made to `POST /my-oauth-api/oauth/authorize`) will be redirected to `http:///client-redirect-uri` where the *Resource Owner* should be prompted to authorize the request +- [Notifications](#oauth-token-notifications) of token issuance will be sent to `http://notifyme.com` with the `X-Tyk-Shared-Secret` header set to `oauth-shared-secret` + +The *auth server* (Tyk) will issue an *access token* and *refresh token* in exchange for a valid *authorization code*. Once the client has a valid access token, it will be expected in the `Authorization` header of the request. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk and, with correctly configured and integrated *identity server* can be used to try out OAuth Client Authentication using Tyk as the Authorization Server. + +### Using Tyk Classic APIs + +As noted in the Tyk Classic API [documentation](/api-management/gateway-config-tyk-classic#configuring-authentication-for-tyk-classic-apis), you can select the Tyk as OAuth Server method using the `use_oauth2` option. + +## Managing OAuth Tokens + +### Using Refresh Tokens + +The Refresh Token flow is used to obtain a new *access token* when the current token has expired or is about to expire. This allows clients to maintain a valid *access token* without requiring the user to go through the authentication and authorization process again. + +*Refresh tokens* are single use and, when used, automatically invalidate the access token with which they were issued. This prevents accidental duplication of access tokens granting authorized access to a resource (API). + +A *refresh token* can be issued by the *auth server* alongside the *access token* at the last stage of the OAuth flow for: +- Authentication Code grant +- Resource Owner Password grant + +You configure whether Tyk should issue a refresh token within the [API proxy definition](#configuring-your-api-proxy). + +#### Refreshing an Access Token + +If you have correctly configured your API, then Tyk will provide a *refresh token* with the *access token*. The *client* can subsequently exchange the *refresh token* for a new *access token* without having to re-authenticate, with another call to the `POST //oauth/token` endpoint as follows: + +Refresh Token flow + +This endpoint is protected using Basic Authentication where the username is the *client Id* and the password is the *client secret*. + +The following data is required in the request payload: + +| Parameter | Value | +| :--------------- | :--------------------------------------------------------- | +| `grant_type` | `refresh_token` | +| `client_id` | client Id | +| `client_secret` | client secret | +| `refresh_token` | The refresh token provided with the original access token | + +For example: + +```bash +curl -X POST \ + https://tyk.cloud.tyk.io/my-api/oauth/token/ \ + -H "Authorization: Basic bXktY2xpZW50LWlkOm15LWNsaWVudC1zZWNyZXQ=" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=refresh_token&client_id=my-client-id&client_secret=my-client-secret&refresh_token=YjdhOWFmZTAtNmExZi00ZTVlLWIwZTUtOGFhNmIwMWI3MzJj" +``` + +This command, issued by the *client* will obtain a new access token for the `/my-api` proxy deployed on a Tyk Gateway at `https://tyk.cloud.tyk.io`. The basic auth key is the base64 encoded representation of `my-client-id:my-client-secret` The `client_id` and `client_secret` match those allocated by Tyk (the auth server) for the *client app*. The `refresh_token` is a valid *refresh token* previously issued to the *client*. + +The response payload contains: +- `access_token`: a new *access token* which can be used by the *client* to access the protected API +- `expires_in`: the expiration date/time of the access token +- `token_type`: set to `bearer` indicating that the access token should be provided in an [Auth Token](/api-management/authentication/bearer-token) request to the protected API +- `refresh_token`: a new *refresh token* that can be used later to refresh the new *access token* + +For example: + +```json expandable +{ + "access_token": "580defdbe1d21e0001c67e5c2a0a6c98ba8b4a059dc5825388501573", + "expires_in": 3600, + "refresh_token": "NWQzNGVhMTItMDE4Ny00MDFkLTljOWItNGE4NzI1ZGI1NGU2", + "token_type": "bearer" +} +``` + +### Revoking Access Tokens + +OAuth access tokens have built in expiry, but if you need to [revoke](https://tools.ietf.org/html/rfc7009) a client's access to the API before this time, then you can use the option on the [OAuth Client management screen](#using-the-tyk-dashboard-ui) screen in Tyk Dashboard UI or the Tyk Dashboard API to do so. + +Using the **Tyk Dashboard API** you can revoke specific tokens (both access and refresh) or all tokens issued for a specific *client app* as follows: + +- [retrieve a list of all tokens for a client app](/api-management/dashboard-configuration#retrieve-all-current-tokens-for-specified-oauth20-client) +- [revoke a single token](/api-management/dashboard-configuration#revoke-a-single-oauth-client-token) +- [revoke all tokens for a client app](/api-management/dashboard-configuration#revoke-all-oauth-client-tokens) + +These endpoints are protected using the Dashboard API secret assigned to the user managing the tokens, which must be provided in the `Authorization` header. + +In this example, we issue a request to the `/revoke` endpoint of the *auth server* via the Tyk Dashboard API to invalidate a specific *access token*: + +```bash +curl -X POST \ + https://admin.cloud.tyk.io/api/apis/oauth/{CLIENT_ID}/revoke/ \ + -H "Authorization: " \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "token=580defdbe1d21e0001c67e5c2a0a6c98ba8b4a059dc5825388501573&token_type_hint=access_token&client_id=my-client-id&client_secret=my-client-secret" +``` + +Note that the `token_type_hint` must be set to `access_token` or `refresh_token` to match the type of `token` to be revoked. + + +### OAuth Token Notifications + +When operating as an OAuth authorization server, Tyk can generate an event whenever it issues an *access token*. You can configure a dedicated webhook that will be triggered to notify the Resource Owner service of the occurrence of the event. + +OAuth token notifications can only be configured when using **Authorization Code** or **Resource Owner Password Credentials** grants, not when using *Client Credentials* grant because this flow is primarily used for server-to-server communication, where the client acts on its own behalf without user-specific authorization changes. + +You can configure the URL that the webhook will issue a `POST` request and a "shared secret" value that will be provided in a header (`X-Tyk-Shared-Secret`) used to secure the communication to the target application. The OAuth token notification webhook does not support any other authentication method. + +The body of the webhook request will have this content: + +```json expandable +{ + "auth_code": "", + "new_oauth_token": "", + "refresh_token": "", + "old_refresh_token": "", + "notification_type": "" +} +``` + +where +- `auth_code` is the Authorization Code that has been issued +- `new_oauth_token` is the Access Token that has been issued +- `refresh_token` is the Refresh Token that has been issued +- `old_refresh_token` is the Refresh Token that has been consumed when refreshing an access token +- `notification_type` will indicate the cause of the event: + - `new`: a new access token has been issued + - `refresh`: a token has been refreshed and a new refresh token has been issued + +#### Configuring Notifications in the Tyk API Designer + +Client Authentication is configured on the **Settings** screen within the Tyk OAS API Designer, within the **Server** section. Ensuring that you are in **Edit** mode, go to the *Authentication* section where you should have selected **Tyk OAuth 2.0** from the drop down options. + +Here you will see the *Notifications* section where you can configure: + +- Notifications URL +- Notifications Shared Secret + +Remember to select **Save API** to apply these settings to your API. + +#### Configuring Notifications in the Tyk OAS API Definition + +The example given [above](#using-the-api-definition) includes the configuration necessary to issue notifications for token issuance (see lines 48-51 in the example). diff --git a/api-management/automations.mdx b/api-management/automations.mdx new file mode 100644 index 00000000..753ec794 --- /dev/null +++ b/api-management/automations.mdx @@ -0,0 +1,66 @@ +--- +title: "Tyk Automations Tools" +'og:description': "Tyk Tools that help with automating deployment and API Management operations" +tags: ['Tyk API Management', 'Tyk Sync', 'Tyk Operator', 'Github', 'Kubernetes', 'Automations'] +sidebarTitle: "Overview" +--- + +import { ResponsiveGrid } from '/snippets/ResponsiveGrid.mdx'; + +## Introduction + +Managing APIs across multiple environments can quickly become complex. Updating and overseeing multiple configurations, security policies, and deployments requires a significant amount of effort without the right tools. Tyk’s suite of automation tools simplifies this process by enabling automated control over API management tasks, helping teams ensure reliability, reduce manual errors, and maintain consistency across deployments. + +In this page, we’ll walk through the primary tools for automating API management with Tyk, including: + +* **Tyk Operator for Kubernetes**: Automate API deployments within Kubernetes environments. +* **Tyk Sync**: Sync configurations across environments for consistent API management. + +## Prerequisites + +Before diving into lifecycle automations with Tyk, ensure you have the following: + +- **A Tyk installation** (Self-Managed or Cloud) + - If you don't have Tyk installed, follow our [installation guide](/tyk-self-managed/install) + - For Tyk Cloud, sign up [here](https://tyk.io/sign-up/) + - Tyk Operator license key. Starting from Tyk Operator v1.0, a valid license key is required. + +- **Access to a Kubernetes cluster v1.19+** (for Tyk Operator sections) + - If you're new to Kubernetes, check out the official [Kubernetes documentation](https://kubernetes.io/docs/setup/) + +- **Helm 3+** (for installing Tyk Operator) + - If you don't have Helm installed, follow the [official Helm installation guide](https://helm.sh/docs/intro/install/) + - Verify your installation by running `helm version` in your terminal + +- **Tyk Dashboard v3+ access** (for Tyk Sync setup) + - Learn how to set up the Tyk Dashboard [here](/tyk-dashboard) + +- **Basic knowledge of Kubernetes, YAML** (important for Tyk Operator and Tyk Sync) + - For Kubernetes, visit the [official tutorials](https://kubernetes.io/docs/tutorials/) + - For YAML, check out this [YAML tutorial](https://yaml.org/spec/1.2/spec.html) + +If you're missing any of these prerequisites, please follow the provided links to set up the necessary components before proceeding with the lifecycle automation steps. + +## Automation Tools + + + + + +**Read time: 10 mins** + +Synchronize Tyk Environment With GitHub using Tyk Sync. + + + +**Read time: 10 mins** + +API Management in Kubernetes using Tyk Operator. + + + + + +## Conclusion + +With Tyk’s automation tools, you now have a set of options for streamlining API management, from handling deployments within Kubernetes to establishing consistency across multiple environments. By integrating these tools, you can simplify complex API workflows, maintain secure configurations, and save time through reduced manual intervention. \ No newline at end of file diff --git a/api-management/automations/operator.mdx b/api-management/automations/operator.mdx new file mode 100644 index 00000000..e1143e16 --- /dev/null +++ b/api-management/automations/operator.mdx @@ -0,0 +1,5053 @@ +--- +title: "Tyk Operator - API Management in Kubernetes" +'og:description': "Kubernetes native API management using Tyk Operator" +tags: ['Tyk API Management', 'Tyk Sync', 'Tyk Operator', 'Github', 'Kubernetes', 'Automations'] +sidebarTitle: "API Management in Kubernetes" +--- + +## Introduction + +Using Tyk Operator within Kubernetes allows you to manage API lifecycles declaratively. This section provides instructions for setting up and configuring the Tyk Operator to automate API creation, updates, and security in Kubernetes clusters, ensuring your APIs align with Kubernetes management practices. + + +## What is Tyk Operator? +If you’re using Kubernetes, or if you’re building an API that operates within a Kubernetes environment, the Tyk Operator is a powerful tool for automating the API lifecycle. + +Tyk Operator is a native Kubernetes operator, allowing you to define and manage APIs as code. This means you can deploy, update, and secure APIs using the same declarative configuration approach Kubernetes uses for other application components. + +Tyk Operator + + +## Key Concepts + +### GitOps With Tyk +With Tyk Operator, you can configure your APIs using Kubernetes native manifest files. You can use the manifest files in a GitOps workflow as the single source of truth for API deployment. + + + +If you use Tyk Operator to manage your APIs, you should set up RBAC such that human users cannot have the "write" permission on the API definition endpoints using Tyk Dashboard. + + + +#### What is GitOps? +β€œGitOps” refers to the operating model of using Git as the β€œsingle source of truth” to drive continuous delivery for infrastructure and software through automated CI/CD workflow. + +#### Tyk Operator in your GitOps workflow +You can install Argo CD, Flux CD or the GitOps tool of your choice in a cluster, and connect it to the Git repository where you version control your API manifests. The tool can synchronise changes from Git to your cluster. The API manifest updates in cluster would be detected by Tyk Operator, which has a Kubernetes controller to automatically reconcile the API configurations on your Tyk Gateway or Tyk Dashboard. + +**Kubernetes-Native Developer Experience** +API Developers enjoy a smoother Continuous Integration process as they can develop, test, and deploy the microservices. API configurations together use familiar development toolings and pipeline. + +**Reliability** +With declarative API configurations, you have a single source of truth to recover after any system failures, reducing the meantime to recovery from hours to minutes. + +#### Single Source of Truth for API Configurations +Tyk Operator will reconcile any divergence between the Kubernetes desired state and the actual state in [Tyk Gateway](/tyk-oss-gateway) or [Tyk Dashboard](/tyk-dashboard). Therefore, you should maintain the API definition manifests in Kubernetes as the single source of truth for your system. If you update your API configurations using Tyk Dashboard, those changes would be reverted by Tyk Operator eventually. + +To learn more about Gitops with Tyk, refer the following blog posts: +- [GitOps-enabled API management in Kubernetes](https://tyk.io/blog/gitops-enabled-api-management-in-kubernetes/) +- [A practical guide using Tyk Operator, ArgoCD, and Kustomize](https://tyk.io/blog/a-practical-guide-using-tyk-operator-argocd-and-kustomize/) + +### Custom Resources in Tyk + +In Kubernetes, a [Custom Resource (CR)](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) is an extension of the Kubernetes API that allows you to introduce custom objects in your cluster. Custom Resources enable you to define and manage custom configurations and settings specific to your applications, making Kubernetes highly extensible. These custom objects are defined using Custom Resource Definitions (CRDs), which specify the schema and structure of the resource. + +Tyk Operator manages multiple custom resources to help users create and maintain their API configurations: + +**TykOasApiDefinition**: Available from Tyk Operator v1.0. It represents a [Tyk OAS API configuration](/api-management/gateway-config-tyk-oas). Tyk OAS API is based on the OpenAPI specification (OAS) and is the recommended format for standard HTTP APIs. + +**ApiDefinition**: Available on all versions of Tyk Operator. It represents a [Tyk Classic API configuration](/api-management/gateway-config-tyk-classic). Tyk Classic API is the traditional format used for defining all APIs in Tyk, and now the recommended format for non-HTTP APIs such as TCP, GraphQL, and Universal Data Graph (UDG). Tyk Operator supports the major features of Tyk Classic API and the feature support details can be tracked [here](#apidefinition-crd). + +**TykStreamsApiDefinition**: Available from Tyk Operator v1.1. It represents an [Async API configuration](/api-management/event-driven-apis#configuration-options) which is based on [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas). Tyk Operator supports all [Tyk Streams](/api-management/event-driven-apis#) features as they become available on the Gateway. + +**SecurityPolicy**: Available on all versions of Tyk Operator. It represents a [Tyk Security Policy configuration](#security-policy-example). Security Policies in Tyk provide a way to define and enforce security controls, including authentication, authorization, and rate limiting for APIs managed in Tyk. Tyk Operator supports essential features of Security Policies, allowing users to centrally manage access control and security enforcement for all APIs across clusters. + +These custom resources enable users to leverage Kubernetes' declarative configuration management to define, modify, and version their APIs, seamlessly integrating with other Kubernetes-based workflows and tools. + +#### Custom Resources for API and Policy Configuration + +The following custom resources can be used to configure APIs and policies at [Tyk Gateway](/tyk-oss-gateway) or [Tyk Dashboard](/tyk-dashboard). + +| Kind | Group | Version | Description | +| :-------------------- | :------------- | :----------- | :--------------------------------------------------------------------------------------------------- | +| TykOasApiDefinition| tyk.tyk.io | v1alpha1 | Defines configuration of [Tyk OAS API Definition object](/api-management/gateway-config-tyk-oas) | +| ApiDefinition | tyk.tyk.io | v1alpha1 | Defines configuration of [Tyk Classic API Definition object](/api-management/gateway-config-tyk-classic) | +| TykStreamsApiDefinition| tyk.tyk.io | v1alpha1 | Defines configuration of [Tyk Streams](/api-management/event-driven-apis#configuration-options) | +| SecurityPolicy | tyk.tyk.io | v1alpha1 | Defines configuration of [security policies](/api-management/policies#what-is-a-security-policy). Operator supports linking ApiDefinition custom resources in SecurityPolicy's access list so that API IDs do not need to be hardcoded in the resource manifest. | +| SubGraph | tyk.tyk.io | v1alpha1 | Defines a [GraphQL federation subgraph](/api-management/graphql#subgraphs-and-supergraphs). | +| SuperGraph | tyk.tyk.io | v1alpha1 | Defines a [GraphQL federation supergraph](/api-management/graphql#subgraphs-and-supergraphs). | +| OperatorContext | tyk.tyk.io | v1alpha1 | Manages the context in which the Tyk Operator operates, affecting its overall behavior and environment. See [Operator Context](#multi-tenancy-in-tyk) for details. | + +#### Tyk Classic Developer Portal + +The following custom resources can be used to configure [Tyk Classic Developer Portal](/tyk-developer-portal/tyk-portal-classic). + +| Kind | Group | Version | Description | +| :-------------------- | :------------- | :----------- | :--------------------------------------------------------------------------------------------------- | +| APIDescription | tyk.tyk.io | v1alpha1 | Configures [Portal Documentation](/tyk-apis/tyk-portal-api/portal-documentation). | +| PortalAPICatalogue | tyk.tyk.io | v1alpha1 | Configures [Portal API Catalogue](/getting-started/key-concepts/api-catalogue). | +| PortalConfig | tyk.tyk.io | v1alpha1 | Configures [Portal Configuration](/tyk-apis/tyk-portal-api/portal-configuration). | + + +### Reconciliation With Tyk Operator +#### Controllers & Operators +In Kubernetes, [controllers](https://kubernetes.io/docs/concepts/architecture/controller/) watch one or more Kubernetes resources, which can be built-in types like *Deployments* or custom resources like *ApiDefinition* - in this case, we refer to Controller as Operator. The purpose of a controller is to match the desired state by using Kubernetes APIs and external APIs. + +> A [Kubernetes operator](https://www.redhat.com/en/topics/containers/what-is-a-kubernetes-operator) is an application-specific controller that extends the functionality of the Kubernetes API to create, configure, and manage instances of complex applications on behalf of a Kubernetes user. + +#### Desired State vs Observed State +Let’s start with the *Desired State*. It is defined through Kubernetes Manifests, most likely YAML or JSON, to describe what you want your system to be in. Controllers will watch the resources and try to match the actual state (the observed state) with the desired state for Kubernetes Objects. For example, you may want to create a Deployment that is intended to run three replicas. So, you can define this desired state in the manifests, and Controllers will perform necessary operations to make it happen. + +How about *Observed State*? Although the details of the observed state may change controller by controller, usually controllers update the status field of Kubernetes objects to store the observed state. For example, in Tyk Operator, we update the status to include *api_id*, so that Tyk Operator can understand that the object was successfully created on Tyk. + +#### Reconciliation +Reconciliation is a special design paradigm used in Kubernetes controllers. Tyk Operator also uses the same paradigm, which is responsible for keeping our Kubernetes objects in sync with the underlying external APIs - which is Tyk in our case. + +**When would reconciliation happen?** +
+Before diving into Tyk Operator reconciliation, let's briefly mention some technical details about how and when reconciliation happens. Reconciliation only happens when certain events happen on your cluster or objects. Therefore, Reconciliation will **NOT** be triggered when there is an update or modification on Tyk’s side. It only watches certain Kubernetes events and is triggered based on them. Usually, the reconciliation happens when you modify a Kubernetes object or when the cache used by the controller expires - side note, controllers, in general, use cached objects to reduce the load in the Kube API server. Typically, caches expire in ~10 hours or so but the expiration time might change based on Operator configurations. + +So, in order to trigger Reconciliation, you can either +- modify an object, which will trigger reconciliation over this modified object or, +- restart Tyk Operator pod, which will trigger reconciliation over each of the objects watched by Tyk Operator. + +**What happens during Reconciliation?** +
+Tyk Operator will compare desired state of the Kubernetes object with the observed state in Tyk. If there is a drift, Tyk Operator will update the actual state on Tyk with the desired state. In the reconciliation, Tyk Operator mainly controls three operations; DELETE, CREATE, and UPDATE. + +- **CREATE** - an object is created in Kubernetes but not exists in Tyk +- **UPDATE** - an object is in different in Kubernetes and Tyk (we compare that by hash) +- **DELETE** - an object is deleted in Kubernetes but exists in Tyk + +**Drift Detection** +
+If human operators or any other system delete or modify API Definition from Tyk Gateway or Dashboard, Tyk Operator will restore the desired state back to Tyk during reconciliation. This is called Drift Detection. It can protect your systems from unauthorized or accidental modifications. It is a best practice to limit user access rights on production environment to read-only in order to prevent accidental updates through API Manager directly. + + +### CRD Versioning + +Tyk follows standard practices for naming and versioning custom resources as outlined by the Kubernetes Custom Resource Definition [versioning guidelines](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definition-versioning/). Although we are currently on the `v1alpha1` version, no breaking changes will be introduced to existing Custom Resources without a version bump. This means that any significant changes or updates that could impact existing resources will result in a new version (e.g., `v1beta1` or `v1`) and Operator will continue supporting all CRD versions for a reasonable time before deprecating an older version. This ensures a smooth transition and compatibility, allowing you to upgrade without disrupting your current configurations and workflows. + +For more details on Kubernetes CRD versioning practices, refer to the Kubernetes Custom Resource Definition [Versioning documentation](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definition-versioning/). + + +### Operator User +Tyk Operator is a Kubernetes Controller that manages Tyk Custom Resources (CRs) such as API Definitions and Security Policies. Developers define these resources as [Custom Resource (CRs)](#custom-resources-in-tyk), and Tyk Operator ensures that the desired state is reconciled with the Tyk Gateway or Dashboard. This involves creating, updating, or deleting API configurations until the target state matches the desired state. + +For the Tyk Dashboard, Tyk Operator functions as a system user, bound by Organization and RBAC rules. + +During start up, Tyk Operator looks for these keys from `tyk-operator-conf` secret or from the environment variables (listed in the table below). + +| Key or Environment Variable | Description | +|:-----|:-------------| +| `TYK_MODE` | "ce" for OSS or "pro" for licensed users | +| `TYK_URL` | URL of Tyk Gateway or Dashboard API | +| `TYK_ORG` | Organization ID of Operator user | +| `TYK_AUTH` | API key of Operator user | + +These would be the default credentials Tyk Operator uses to connect to Tyk. + + +### Multi-tenancy in Tyk + +Tyk Dashboard is multi-tenant capable, which means you can use a single Tyk Dashboard instance to host separate [organizations](/api-management/dashboard-configuration#organizations) for each team or department. Each organization is a completely isolated unit with its own: + +- API Definitions +- API Keys +- Users +- Developers +- Domain +- Tyk Classic Portal + +This structure is ideal for businesses with a complex hierarchy, where distinct departments operate independently but within the same overall infrastructure. + +Multi-tenancy in Tyk Dashboard + +#### Define OperatorContext for Multi-Tenant API Management + +The `OperatorContext` in Tyk Operator allows you to create isolated management environments by defining specific access parameters for different teams or departments within a shared Tyk Operator instance. It helps you specify: + +- The Tyk Dashboard with which the Operator interacts +- The organization under which API management occurs +- The user identity utilized for requests +- The environment in which the Operator operates + +By setting different `OperatorContext` configurations, you can define unique access and management contexts for different teams. These contexts can then be referenced directly in your `ApiDefinition`, `TykOasApiDefinition`, `TykStreamsApiDefinition` or `SecurityPolicy` custom resource definitions (CRDs) using the `contextRef` field, enabling precise control over API configurations. + +#### Example Scenarios Using OperatorContext + +1. **No OperatorContext Defined** + - If no `OperatorContext` is defined, Tyk Operator defaults to using credentials from the `tyk-operator-conf` secret or from environment variables. This means all API management actions are performed under the system’s default user credentials, with no specific contextual isolation. + +2. **OperatorContext Defined but Not Referenced** + - When an `OperatorContext` is defined but not referenced in an API configuration, Tyk Operator continues to use the default credentials from `tyk-operator-conf`. The specified `OperatorContext` is ignored, resulting in API operations being managed under default credentials. + +3. **OperatorContext Defined and Referenced** + - If a specific `OperatorContext` is both defined and referenced in an API or policy, Tyk Operator utilizes the credentials and parameters from the referenced `OperatorContext` to perform API operations. This allows each API or policy to be managed with isolated configurations, enabling team-based or department-specific API management within the same Kubernetes cluster. + +Using `OperatorContext` offers flexibility for multi-tenancy, helping organizations manage and isolate API configurations based on their specific team or departmental needs. + +Multi-tenancy in Kubernetes Tyk Operator + +### TLS Certificates + +Tyk Operator is designed to offer a seamless Kubernetes-native experience by managing TLS certificates stored within Kubernetes for your API needs. Traditionally, to use a certificate (e.g., as a client certificate, domain certificate, or certificate for accessing an upstream service), you would need to manually upload the certificate to Tyk and then reference it using a 'Certificate ID' in your API definitions. This process can become cumbersome, especially in a Kubernetes environment where certificates are often managed as secrets and may rotate frequently. + +To address this challenge, Tyk Operator allows you to directly reference certificates stored as Kubernetes secrets within your custom resource definitions (CRDs). This reduces operational overhead, minimizes the risk of API downtime due to certificate mismatches, and provides a more intuitive experience for API developers. + +#### Benefits of Managing Certificates with Tyk Operator +- **Reduced operational overhead**: Automates the process of updating certificates when they rotate. +- **Minimized risk of API downtime**: Ensures that APIs continue to function smoothly, even when certificates are updated. +- **Improved developer experience**: Removes the need for API developers to manage certificate IDs manually. + +#### Examples + +| Certificate Type | Supported in ApiDefinition | Supported in TykOasApiDefinition | Supported in TykStreamsApiDefinition | +| :------------------ | :------------- | :--------- | :--------- | +| Client certifates | βœ… [Client mTLS](/basic-config-and-security/security/mutual-tls/client-mtls#setup-static-mtls-in-tyk-operator-using-the-tyk-classic-api-definition) | βœ… [Client mTLS](/basic-config-and-security/security/mutual-tls/client-mtls#setup-static-mtls-in-tyk-operator-using-tyk-oas-api-definition) | Certificate ID can be set in the API Definition but configuring certificates from Secrets in CRD is not supported. | +| Custom domain certificates | βœ… [TLS and SSL](/api-management/certificates#dynamically-setting-ssl-certificates-for-custom-domains) | βœ… [TLS and SSL](/api-management/certificates#dynamically-setting-ssl-certificates-for-custom-domains) | Certificate ID can be set in the API Definition but configuring certificates from Secrets in CRD is not supported. | +| Public keys pinning | βœ… [Certificate pinning](/api-management/upstream-authentication/mtls#using-tyk-operator-to-configure-mtls-for-tyk-classic-apis) | βœ… [Certificate pinning](/api-management/upstream-authentication/mtls#certificate-pinning) | Certificate ID can be set in the API Definition but configuring certificates from Secrets in CRD is not supported. | +| Upstream mTLS | βœ… [Upstream mTLS via Operator](/api-management/upstream-authentication/mtls#using-tyk-operator-to-configure-mtls-for-tyk-classic-apis) | βœ… [Upstream mTLS via Operator](/api-management/upstream-authentication/mtls#using-tyk-operator-to-configure-mtls) | Certificate ID can be set in the API Definition but configuring certificates from Secrets in CRD is not supported. | + + +## Install and Configure Tyk Operator + +We assume you have already installed Tyk. If you don’t have it, check out [Tyk +Cloud](/tyk-cloud#quick-start-tyk-cloud) or [Tyk Self +Managed](/tyk-self-managed) page. [Tyk Helm +Chart](/product-stack/tyk-charts/overview) is the preferred (and easiest) way to install Tyk on Kubernetes. + +In order for policy ID matching to work correctly, Dashboard must have `allow_explicit_policy_id` and +`enable_duplicate_slugs` set to `true` and Gateway must have `policies.allow_explicit_policy_id` set to `true`. + +Tyk Operator needs a [user credential](#operator-user) to connect with +Tyk Dashboard. The Operator user should have write access to the resources it is going to manage, e.g. APIs, Certificates, +Policies, and Portal. It is the recommended practice to turn off write access for other users for the above resources. See +[Using Tyk Operator to enable GitOps with Tyk](/api-management/automations) about +maintaining a single source of truth for your API configurations. + +### Install cert-manager + +Tyk Operator uses cert-manager to provision certificates for the webhook server. If you don't have cert-manager +installed, you can follow this command to install it: + +Alternatively, you have the option to manually handle TLS certificates by disabling the `cert-manager` requirement. For more details, please refer to this [configuration](#webhook-configuration). + +```console +$ kubectl apply --validate=false -f https://github.com/jetstack/cert-manager/releases/download/v1.8.0/cert-manager.yaml +``` + +Since Tyk Operator supports Kubernetes v1.19+, the minimum cert-manager version you can use is v1.8. If you run into the +cert-manager related errors, please ensure that the desired version of Kubernetes version works with the chosen version +of cert-manager by checking [supported releases page](https://cert-manager.io/docs/installation/supported-releases/) and +[cert-manager documentation](https://cert-manager.io/docs/installation/supported-releases/). + +Please wait for the cert-manager to become available before continuing with the next step. + +### Option 1: Install Tyk Operator via Tyk's Umbrella Helm Charts + +If you are using [Tyk Stack](/product-stack/tyk-charts/tyk-stack-chart), [Tyk Control +Plane](/product-stack/tyk-charts/tyk-control-plane-chart), or [Tyk Open +Source Chart](/product-stack/tyk-charts/tyk-oss-chart), you can install Tyk Operator alongside other Tyk +components by setting value `global.components.operator` to `true`. + +Starting from Tyk Operator v1.0, a license key is required to use the Tyk Operator. You can provide it while installing +Tyk Stack, Tyk Control Plane or Tyk OSS helm chart by setting `global.license.operator` field. You can also set license +key via a Kubernetes secret using `global.secrets.useSecretName` field. The secret should contain a key called +`OperatorLicense` + +Note: If you are using `global.secrets.useSecretName`, you must configure the operator license in the referenced Kubernetes secret. `global.license.operator` will not be used in this case. + +### Option 2: Install Tyk Operator via stand-alone Helm Chart + +If you prefer to install Tyk Operator separately, follow this section to install Tyk Operator using Helm. + +#### Configure Tyk Operator via environment variable or tyk-operator-conf secret + +Tyk Operator configurations can be set using `envVars` field of helm chart. See the table below for a list of expected +environment variable names and example values. + +```yaml expandable +envVars: + - name: TYK_OPERATOR_LICENSEKEY + value: "{YOUR_LICENSE_KEY}" + - name: TYK_MODE + value: "pro" + - name: TYK_URL + value: "http://dashboard-svc-tyk-tyk-dashboard.tyk.svc:3000" + - name: TYK_AUTH + value: "2d095c2155774fe36d77e5cbe3ac963b" + - name: TYK_ORG + value: "5e9d9544a1dcd60001d0ed20" +``` + +It can also be set via a Kubernetes secret. The default K8s secret name is `tyk-operator-conf`. If you want to use +another name, configure it through Helm Chart [envFrom](#install-tyk-operator-and-custom-resource-definitions-crds) value. + +The Kubernetes secret or envVars field should set the following keys: + + + + + +| Key | Mandatory | Example Value | Description | +| :--------------------------- | :-------- | :-------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------- | +| TYK_OPERATOR_LICENSEKEY | Yes | `` | Tyk Operator license key | +| TYK_MODE | Yes | pro | β€œce” for Tyk Open Source mode, β€œpro” for Tyk licensed mode. | +| TYK_URL | Yes | http://dashboard-svc-tyk-tyk-dashboard.tyk.svc:3000 | Management URL of Tyk Gateway (Open Source) or Tyk Dashboard | +| TYK_AUTH | Yes | 2d095c2155774fe36d77e5cbe3ac963b | Operator user API key. | +| TYK_ORG | Yes | 5e9d9544a1dcd60001d0ed20 | Operator user ORG ID. | +| TYK_TLS_INSECURE_SKIP_VERIFY | No | true | Set to `β€œtrue”` if the Tyk URL is HTTPS and has a self-signed certificate. If it isn't set, the default value is `false`. | +| WATCH_NAMESPACE | No | foo,bar | Comma separated list of namespaces for Operator to operate on. The default is to operate on all namespaces if not specified. | +| WATCH_INGRESS_CLASS | No | customclass | Define the ingress class Tyk Operator should watch. Default is `tyk` | +| TYK_HTTPS_INGRESS_PORT | No | 8443 | Define the ListenPort for HTTPS ingress. Default is `8443`. | +| TYK_HTTP_INGRESS_PORT | No | 8080 | Define the ListenPort for HTTP ingress. Default is `8080`. | + + + + + +**Note**: From Tyk Operator v1.0, although Tyk Operator is compatible with the Open Source Tyk Gateway, a valid license +key is required for running Tyk Operator. + +| Key | Mandatory | Example Value | Description | +| :--------------------------- | :-------- | :------------------------------------------------- | :--------------------------------------------------------------------------------------------------------------------------- | +| TYK_OPERATOR_LICENSEKEY | Yes | `` | Tyk Operator license key | +| TYK_MODE | Yes | ce | β€œce” for Tyk Open Source mode, β€œpro” for Tyk licensed mode. | +| TYK_URL | Yes | http://gateway-svc-tyk-ce-tyk-gateway.tyk.svc:8080 | Management URL of Tyk Gateway (Open Source) or Tyk Dashboard | +| TYK_AUTH | Yes | myapisecret | Operator user API key. | +| TYK_ORG | Yes | myorgid | Operator user ORG ID. | +| TYK_TLS_INSECURE_SKIP_VERIFY | No | true | Set to `β€œtrue”` if the Tyk URL is HTTPS and has a self-signed certificate. If it isn't set, the default value is `false`. | +| WATCH_NAMESPACE | No | foo,bar | Comma separated list of namespaces for Operator to operate on. The default is to operate on all namespaces if not specified. | +| WATCH_INGRESS_CLASS | No | customclass | Define the ingress class Tyk Operator should watch. Default is `tyk` | +| TYK_HTTPS_INGRESS_PORT | No | 8443 | Define the ListenPort for HTTPS ingress. Default is `8443`. | +| TYK_HTTP_INGRESS_PORT | No | 8080 | Define the ListenPort for HTTP ingress. Default is `8080`. | + + + + + +**Connect to Tyk Gateway or Dashboard** + +If you install Tyk using Helm Chart, `tyk-operator-conf` will have been created with the following keys: +`TYK_OPERATOR_LICENSEKEY, TYK_AUTH, TYK_MODE, TYK_ORG`, and `TYK_URL` by default. If you didn't use Helm Chart for +installation, please prepare `tyk-operator-conf` secret yourself using the commands below: + +```console expandable +$ kubectl create namespace tyk-operator-system + +$ kubectl create secret -n tyk-operator-system generic tyk-operator-conf \ + --from-literal "TYK_OPERATOR_LICENSEKEY=${TYK_OPERATOR_LICENSEKEY}" \ + --from-literal "TYK_AUTH=${TYK_AUTH}" \ + --from-literal "TYK_ORG=${TYK_ORG}" \ + --from-literal "TYK_MODE=${TYK_MODE}" \ + --from-literal "TYK_URL=${TYK_URL}" +``` + + + +User API key and Organization ID can be found under "Add / Edit User" page within Tyk Dashboard. `TYK_AUTH` corresponds +to Tyk Dashboard API Access Credentials. `TYK_ORG` corresponds to Organization ID. + + + + + +If the credentials embedded in the `tyk-operator-conf` are ever changed or updated, the tyk-operator-controller-manager +pod must be restarted to pick up these changes. + + + +**Watch Namespaces** + +Tyk Operator is installed with cluster permissions. However, you can optionally control which namespaces it watches by +setting the `WATCH_NAMESPACE` through `tyk-operator-conf` secret or the environment variable to a comma separated list +of k8s namespaces. For example: + +- `WATCH_NAMESPACE=""` will watch for resources across the entire cluster. +- `WATCH_NAMESPACE="foo"` will watch for resources in the `foo` namespace. +- `WATCH_NAMESPACE="foo,bar"` will watch for resources in the `foo` and `bar` namespace. + +**Watch custom ingress class** + +You can configure [Tyk Operator as Ingress Controller](#control-kubernetes-ingress-resources) so +that [Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) resources can be managed by Tyk as +APIs. By default, Tyk Operator looks for the value `tyk` in Ingress resources `kubernetes.io/ingress.class` annotation +and will ignore all other ingress classes. If you want to override this default behavior, you may do so by setting +[WATCH_INGRESS_CLASS](#configure-tyk-operator-via-environment-variable-or-tyk-operator-conf-secret) through `tyk-operator-conf` or the environment variable. + +#### Install Tyk Operator and Custom Resource Definitions (CRDs) + +You can install CRDs and Tyk Operator using the stand-alone Helm Chart by running the following command: + +```console +$ helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ +$ helm repo update + +$ helm install tyk-operator tyk-helm/tyk-operator -n tyk-operator-system +``` + +This process will deploy Tyk Operator and its required Custom Resource Definitions (CRDs) into your Kubernetes cluster +in `tyk-operator-system` namespace. + +**Helm configurations** + + + +Starting from Tyk Operator v1.2.0, `webhookPort` is deprecated in favor of `webhooks.port`. + + + +| Key | Type | Default | +| :------------------------------------------- | :------ | :-------------------------------------- | +| envFrom[0].secretRef.name | string | `"tyk-operator-conf"` | +| envVars[0].name | string | `"TYK_OPERATOR_LICENSEKEY"` | +| envVars[0].value | string | `"{OPERATOR_LICENSEKEY}"` | +| envVars[1].name | string | `"TYK_HTTPS_INGRESS_PORT"` | +| envVars[1].value | string | `"8443"` | +| envVars[2].name | string | `"TYK_HTTP_INGRESS_PORT"` | +| envVars[2].value | string | `"8080"` | +| extraVolumeMounts | list | `[]` | +| extraVolumes | list | `[]` | +| fullnameOverride | string | `""` | +| healthProbePort | int | `8081` | +| hostNetwork | bool | `false` | +| image.pullPolicy | string | `"IfNotPresent"` | +| image.repository | string | `"tykio/tyk-operator"` | +| image.tag | string | `"v1.0.0"` | +| imagePullSecrets | list | `[]` | +| metricsPort | int | `8080` | +| nameOverride | string | `""` | +| nodeSelector | object | `{}` | +| podAnnotations | object | `{}` | +| podSecurityContext.allowPrivilegeEscalation | bool | `false` | +| rbac.port | int | `8443` | +| rbac.resources | object | `{}` | +| replicaCount | int | `1` | +| resources | object | `{}` | +| serviceMonitor | bool | `false` | +| webhookPort | int | `9443` | +| webhooks.enabled | bool | `true` | +| webhooks.port | int | `9443` | +| webhooks.annotations | object | `{}` | +| webhooks.tls.useCertManager | bool | `true` | +| webhooks.tls.secretName | string | `webhook-server-cert` | +| webhooks.tls.certificatesMountPath | string | `/tmp/k8s-webhook-server/serving-certs`| + +### Upgrading Tyk Operator + +#### Upgrading from v0.x to v1.0+ + +Starting from Tyk Operator v1.0, a valid license key is required for the Tyk Operator to function. If Tyk Operator is +upgraded from v0.x versions to one of v1.0+ versions, Tyk Operator needs a valid license key that needs to be provided +during upgrade process. This section describes how to set Tyk Operator license key to make sure Tyk Operator continues +functioning. + +To provide the license key for Tyk Operator, Kubernetes secret used to configure Tyk Operator (typically named +tyk-operator-conf as described above) requires an additional field called `TYK_OPERATOR_LICENSEKEY`. Populate this field +with your Tyk Operator license key. + +To configure the license key: + +1. Locate the Kubernetes Secret used to configure Tyk Operator (typically named `tyk-operator-conf`). +2. Add a new field called `TYK_OPERATOR_LICENSEKEY` to this Secret. +3. Set the value of `TYK_OPERATOR_LICENSEKEY` to your Tyk Operator license key. + +After updating the Kubernetes secret with this field, proceed with the standard upgrade process outlined below. + +#### Upgrading Tyk Operator and CRDs + +You can upgrade Tyk Operator through Helm Chart by running the following command: + +```console +$ helm upgrade -n tyk-operator-system tyk-operator tyk-helm/tyk-operator --wait +``` + +[Helm does not upgrade or delete CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#some-caveats-and-explanations) +when performing an upgrade. Because of this restriction, an additional step is required when upgrading Tyk Operator with +Helm. + +```console +$ kubectl apply -f https://raw.githubusercontent.com/TykTechnologies/tyk-charts/refs/heads/main/tyk-operator-crds/crd-$TYK_OPERATOR_VERSION.yaml +``` + + + +Replace $TYK_OPERATOR_VERSION with the image tag corresponding to the Tyk Operator version to which +the Custom Resource Definitions (CRDs) belong. For example, to install CRDs compatible with Tyk Operator v1.0.0, set $TYK_OPERATOR_VERSION to v1.0.0. + + + + +### Uninstalling Tyk Operator + +To uninstall Tyk Operator, you need to run the following command: + +```console +$ helm delete tyk-operator -n tyk-operator-system +``` + +### Webhook Configuration + +Starting from Operator v1.2.0 release, [Kubernetes Webhooks](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers) can now be configured using the Helm chart by specifying the necessary settings in the values.yaml file of the operator. +Since webhooks are enabled by default, there will be no impact to existing users. + +``` expandable +webhooks: + enabled: true + port: 9443 + annotations: {} + tls: + useCertManager: true + secretName: webhook-server-cert + certificatesMountPath: "/tmp/k8s-webhook-server/serving-certs" +``` +- `enabled`: Enables or disables webhooks. +- `port`: Specifies the port for webhook communication. +- `annotations`: Allows adding custom annotations. +- `tls.useCertManager`: If true, Cert-Manager will handle TLS certificates. +- `tls.secretName`: The name of the Kubernetes Secret storing the TLS certificate. +- `tls.certificatesMountPath`: Path where the webhook server mounts its certificates. + +## Set Up Tyk OAS API +Setting up OpenAPI Specification (OAS) APIs with Tyk involves preparing an OAS-compliant API definition and configuring it within your Kubernetes cluster using Tyk Operator. This process allows you to streamline API management by storing the OAS definition in a Kubernetes ConfigMap and linking it to Tyk Gateway through a TykOasApiDefinition resource. + +### Create your Tyk OAS API +#### Prepare the Tyk OAS API Definition +First, you need to have a complete Tyk OAS API definition file ready. This file will contain all the necessary configuration details for your API in OpenAPI Specification (OAS) format. + +Here is an example of what the Tyk OAS API definition might look like. Note that Tyk extension `x-tyk-api-gateway` section should be present. + +```json {hl_lines=["9-25"],linenos=true} expandable +{ + "info": { + "title": "Petstore", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "components": {}, + "paths": {}, + "x-tyk-api-gateway": { + "info": { + "name": "Petstore", + "state": { + "active": true + } + }, + "upstream": { + "url": "https://petstore.swagger.io/v2" + }, + "server": { + "listenPath": { + "value": "/petstore/", + "strip": true + } + } + } +} +``` + +Save this API definition file (e.g., `oas-api-definition.json`) locally. + + + +**Tips** + +You can create and configure your API easily using Tyk Dashboard in a developer environment, and then obtain the Tyk OAS API definition following these instructions: + +1. Open the Tyk Dashboard +2. Navigate to the API you want to manage with the Tyk Operator +3. Click on the "Actions" menu button and select "View API Definition." +4. This will display the raw Tyk OAS API definition of your API, which you can then copy and save locally. + + + +#### Create a ConfigMap for the Tyk OAS API Definition + +You need to create a [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/#configmap-object) in Kubernetes to store your Tyk OAS API definition. The Tyk Operator will reference this ConfigMap to retrieve the API configuration. + +To create the ConfigMap, run the following command: + +```sh +kubectl create configmap tyk-oas-api-config --from-file=oas-api-definition.json -n tyk +``` + +This command creates a ConfigMap named `tyk-oas-api-config` in the `tyk` namespace (replace `tyk` with your actual namespace if different). + + + +**Notes** + +There is inherent size limit to a ConfigMap. The data stored in a ConfigMap cannot exceed 1 MiB. In case your OpenAPI document exceeds this size limit, it is recommended to split your API into smaller sub-APIs for easy management. For details, please consult [Best Practices for Describing Large APIs](https://learn.openapis.org/best-practices.html#describing-large-apis) from the OpenAPI initiative. + + + + + +**Notes** + +If you prefer to create ConfigMap with a manifest using `kubectl apply` command, you may get an error that the annotation metadata cannot exceed 256KB. It is because by using `kubectl apply`, `kubectl` automatically saves the whole configuration in the annotation [kubectl.kubernetes.io/last-applied-configuration](https://kubernetes.io/docs/reference/labels-annotations-taints/#kubectl-kubernetes-io-last-applied-configuration) for tracking changes. Your Tyk OAS API Definition may easily exceed the size limit of annotations (256KB). Therefore, `kubectl create` is used here to get around the problem. + + + +#### Create a TykOasApiDefinition Custom Resource + +Now, create a `TykOasApiDefinition` resource to tell the Tyk Operator to use the Tyk OAS API definition stored in the ConfigMap. + +Create a manifest file named `tyk-oas-api-definition.yaml` with the following content: + +```yaml expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: TykOasApiDefinition +metadata: + name: petstore +spec: + tykOAS: + configmapRef: + name: tyk-oas-api-config # Metadata name of the ConfigMap resource that stores the Tyk OAS API Definition + namespace: tyk # Metadata namespace of the ConfigMap resource + keyName: oas-api-definition.json # Key for retrieving Tyk OAS API Definition from the ConfigMap +``` + +#### Apply the TykOasApiDefinition Manifest + +Use `kubectl` to apply the `TykOasApiDefinition` manifest to your cluster: + +```sh +kubectl apply -f tyk-oas-api-definition.yaml +``` + +This command creates a new `TykOasApiDefinition` resource in your cluster. The Tyk Operator will watch for this resource and configures Tyk Gateway or Tyk Dashboard with a new API using the provided Tyk OAS API definition. + +#### Verify the Tyk OAS API Creation + +To verify that the API has been successfully created, check the status of the TykOasApiDefinition resource: + +```sh +kubectl get tykoasapidefinition petstore +``` + +You should see the status of the resource, which will indicate if the API creation was successful. + +```bash +NAME DOMAIN LISTENPATH PROXY.TARGETURL ENABLED SYNCSTATUS INGRESSTEMPLATE +petstore /petstore/ https://petstore.swagger.io/v2 true Successful +``` + +#### Test the Tyk OAS API +After the Tyk OAS API has been successfully created, you can test it by sending a request to the API endpoint defined in your OAS file. + +For example, if your API endpoint is `/store/inventory"`, you can use `curl` or any API client to test it: + +```sh +curl "TYK_GATEWAY_URL/petstore/store/inventory" +``` + +Replace TYK_GATEWAY_URL with a URL of Tyk Gateway. + +#### Manage and Update the Tyk OAS API +To make any changes to your API configuration, update the OAS file in your ConfigMap and then re-apply the ConfigMap using `kubectl replace`: + +```sh +kubectl create configmap tyk-oas-api-config --from-file=oas-api-definition.json -n tyk --dry-run=client -o yaml | kubectl replace -f - +``` + +The Tyk Operator will automatically detect the change and update the API in the Tyk Gateway. + + + +**Notes** + +`kubectl replace` without `--save-config` option is used here instead of `kubectl apply` because we do not want to save the Tyk OAS API definition in its annotation. If you want to enable `--save-config` option or use `kubectl apply`, the Tyk OAS API definition size would be further limited to at most 262144 bytes. + + + +#### Tyk OAS API Example +This example shows the minimum resources and fields required to define a Tyk OAS API using Tyk Operator. + +```yaml{hl_lines=["7-7", "41-44"],linenos=true} expandable +apiVersion: v1 +kind: ConfigMap +metadata: + name: cm + namespace: default +data: + test_oas.json: |- + { + "info": { + "title": "Petstore", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "components": {}, + "paths": {}, + "x-tyk-api-gateway": { + "info": { + "name": "Petstore", + "state": { + "active": true + } + }, + "upstream": { + "url": "https://petstore.swagger.io/v2" + }, + "server": { + "listenPath": { + "value": "/petstore/", + "strip": true + } + } + } + } +--- +apiVersion: tyk.tyk.io/v1alpha1 +kind: TykOasApiDefinition +metadata: + name: petstore +spec: + tykOAS: + configmapRef: + name: cm + namespace: default + keyName: test_oas.json +``` + +Here, a `ConfigMap` is created that contains the Tyk OAS API Definition with the `data` field with key `test_oas.json`. This is linked to from a `TykOasApiDefinition` resource via `spec.tykOAS.configmapRef`. + +To apply it, simply save the manifest into a file (e.g., `tyk-oas-api.yaml`) and use `kubectl apply -f tyk-oas-api.yaml` to create the required resources in your Kubernetes cluster. This command will create the necessary ConfigMap and TykOasApiDefinition resources in the `default` namespace. + + + +### Secure your Tyk OAS API +#### Update your Tyk OAS API Definition + +First, you'll modify your existing Tyk OAS API Definition to include the API key authentication configuration. + +When creating the Tyk OAS API, you stored your OAS definition in a file named `oas-api-definition.json` and created a ConfigMap named `tyk-oas-api-config` in the `tyk` namespace. + +Modify your Tyk OAS API Definition `oas-api-definition.json` as follow. + +```json {hl_lines=["8-14","16-20","33-40"],linenos=true} expandable +{ + "info": { + "title": "Petstore protected", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "components": { + "securitySchemes": { + "petstore_auth": { + "type": "apiKey", + "name": "Authorization", + "in": "header" + } + } + }, + "security": [ + { + "petstore_auth": [] + } + ], + "paths": {}, + "x-tyk-api-gateway": { + "info": { + "name": "Petstore", + "state": { + "active": true + } + }, + "upstream": { + "url": "https://petstore.swagger.io/v2" + }, + "server": { + "authentication": { + "enabled": true, + "securitySchemes": { + "petstore_auth": { + "enabled": true + } + } + }, + "listenPath": { + "value": "/petstore/", + "strip": true + } + } + } +} +``` + +In this example, we added the following sections to configure key authentication for this API. + +- `components.securitySchemes` defines the authentication method (in this case, `apiKey` in the header). +- `security`: Applies the authentication globally to all endpoints. +- `x-tyk-api-gateway.server.authentication`: Tyk-specific extension to enable the authentication scheme. + +You can configure your API for any Tyk supported authentication method by following the [Client Authentication](/api-management/client-authentication) documentation. + +Save your updated API definition in the same file, `oas-api-definition.json`. + +#### Update the ConfigMap with the new Tyk OAS API Definition + +Update the existing ConfigMap that contains your Tyk OAS API Definition with the following command: + +```sh +kubectl create configmap tyk-oas-api-config --from-file=oas-api-definition.json -n tyk --dry-run=client -o yaml | kubectl replace -f - +``` + +This command updates the existing ConfigMap named `tyk-oas-api-config` in the `tyk` namespace (replace `tyk` with your actual namespace if different) with the new Tyk OAS API Definition stored in `oas-api-definition.json`. + +Since a `TykOasApiDefinition` resource has been created with reference to this ConfigMap in the previous tutorial: + +```yaml expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: TykOasApiDefinition +metadata: + name: petstore +spec: + tykOAS: + configmapRef: + name: tyk-oas-api-config # Metadata name of the ConfigMap resource that stores the Tyk OAS API Definition + namespace: tyk # Metadata namespace of the ConfigMap resource + keyName: oas-api-definition.json # Key for retrieving Tyk OAS API Definition from the ConfigMap +``` + +Any changes in the ConfigMap would be detected by Tyk Operator. Tyk Operator will then automatically reconcile the changes and update the API configuration at Tyk. + +#### Verify the changes + +Verify that the `TykOasApiDefinition` has been updated successfully: + +```sh +kubectl get tykoasapidefinition petstore -o yaml +``` + +Look for the `latestTransaction` field in `status`: + +```yaml +status: + latestTransaction: + status: Successful + time: "2024-09-16T11:48:20Z" +``` + +The **Successful** status shows that Tyk Operator has reconciled the API with Tyk successfully. The last update time is shown in the `time` field. + +#### Test the API Endpoint +Now, test your API endpoint to confirm that it requires an API key. + +For example, if your API endpoint is `/store/inventory"`, you can use `curl` or any API client to test it: + +```sh +curl -v "TYK_GATEWAY_URL/petstore/store/inventory" +``` + +Replace TYK_GATEWAY_URL with a URL of Tyk Gateway. + +Request should fail with a `401 Unauthorized` response now as an API key is required for access. Your API has been secured by Tyk Gateway. + +## Set Up Tyk Classic API + +### Create a Tyk Classic API +First, specify the details of your API using the [ApiDefinition CRD](#apidefinition-crd), then deploy it to create the corresponding Kubernetes resource. Tyk Operator will take control of the CRD and create the actual API in the Tyk data plane. + +#### Create an ApiDefinition resource in YAML format +Create a file called `httpbin.yaml`, then add the following: + +```yaml expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin +spec: + name: httpbin + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true +``` + +You can also use other sample files from the following pages: + +- [HTTP Proxy example](#set-up-manifest-for-http) +- [TCP Proxy example](#set-up-manifest-for-tcp) +- [GraphQL Proxy example](#set-up-manifest-for-graphql) +- [UDG example](#set-up-manifest-for-udg) + +#### Deploy the ApiDefinition resource +We are going to create an ApiDefinition from the httpbin.yaml file, by running the following command: + +```console +$ kubectl apply -f httpbin.yaml +``` + +Or, if you don’t have the manifest with you, you can run the following command: + +```yaml expandable +cat <.`.svc.cluster.local DNS entry once they are created. +For example, if you have a service called `httpbin` in `default` namespace, you can contact `httpbin` service with `httpbin.default.svc` DNS record in the cluster, instead of IP addresses. +Please visit the official [Kubernetes documentation](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/) for more details. +Suppose you want to create a Deployment of [httpbin](https://hub.docker.com/r/kennethreitz/httpbin/) service using [ci/upstreams/httpbin.yaml](https://github.com/TykTechnologies/tyk-operator/blob/master/ci/upstreams/httpbin.yaml) file. You are going to expose the application through port `8000` as described under the Service [specification](https://github.com/TykTechnologies/tyk-operator/blob/master/ci/upstreams/httpbin.yaml#L10). +You can create Service and Deployment by either applying the manifest defined in our repository: + +```console +$ kubectl apply -f ci/upstreams/httpbin.yaml +``` + +Or, if you don’t have the manifest with you, you can run the following command: + +```yaml expandable +cat <` namespace as follows: + +```console +$ kubectl get service -n +``` + +You can update your `httpbin` as follows: + +```yaml expandable +cat <..svc:`). +Now, if you send your request to the `/httpbin` endpoint of the Tyk Gateway, the request will be proxied to the `httpbin Service`: + +```curl expandable +curl -sS http://localhost:8080/httpbin/headers +{ + "headers": { + "Accept": "*/*", + "Accept-Encoding": "gzip", + "Host": "httpbin.default.svc:8000", + "User-Agent": "curl/7.68.0" + } +} +``` + +As you can see from the response, the host that your request should be proxied to is `httpbin.default.svc:8000`. + +### Secure your Classic API +#### Update your API to Require a Key + +You might already have realized that our `httpbin` API is keyless. If you check the APIDefinition's specification, the `use_keyless` field is set to `true`. +Tyk keyless access represents completely open access for your API and causes Tyk to bypass any session-based middleware (middleware that requires access to token-related metadata). Keyless access will enable all requests through. +You can disable keyless access by setting `use_keyless` to false. + +1. Update your `httpbin.yaml` file + +```yaml expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin +spec: + name: httpbin + use_keyless: false + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true +``` + +2. Apply the changes + +```bash +kubectl apply -f httpbin.yaml +``` + +Or, if you don’t have the manifest with you, you can run the following command: + +```yaml expandable +cat < +Tyk Operator supported authentication types are listed in the [API Definition features](#apidefinition-crd) section. + + + +#### Create an API key + +You need to generate a key to access the `httpbin` API now. Follow [this guide](/getting-started/configure-first-api#create-an-api-key) to see how to create an API key for your installation. + +You can obtain the API name and API ID of our example `httpbin` API by following command: + +```yaml expandable +kubectl describe tykapis httpbin +Name: httpbin +Namespace: default +Labels: +Annotations: +API Version: tyk.tyk.io/v1alpha1 +Kind: ApiDefinition +Metadata: + ... +Spec: + ... + Name: httpbin + ... +Status: + api_id: ZGVmYXVsdC9odHRwYmlu +Events: +``` + +You can obtain the API name and API ID from `name` and `status.api_id` field. + +In our example, it is as follows: + +- {API-NAME}: httpbin +- {API-ID}: ZGVmYXVsdC9odHRwYmlu + +When you have successfully created a key, you can use it to access the `httpbin` API. + +```curl expandable +curl -H "Authorization: Bearer {Key ID}" localhost:8080/httpbin/get +{ + "args": {}, + "headers": { + "Accept": "*/*", + "Accept-Encoding": "gzip", + "Authorization": "Bearer {Key ID}", + "Host": "httpbin.org", + "User-Agent": "curl/7.77.0", + "X-Amzn-Trace-Id": "Root=1-6221de2a-01aa10dd56f6f13f420ba313" + }, + "origin": "127.0.0.1, 176.42.143.200", + "url": "http://httpbin.org/get" +} +``` +Since you have provided a valid key along with your request, you do not get a `HTTP 401 Unauthorized` response. + + +### Set Up Tyk Classic API Authentication +Client to Gateway Authentication in Tyk ensures secure communication between clients and the Tyk Gateway. Tyk supports various authentication methods to authenticate and authorize clients before they can access your APIs. These methods include API keys, Static Bearer Tokens, JWT, mTLS, Basic Authentication, and more. This document provides example manifests for each authentication method supported by Tyk. + +#### Keyless (Open) + +This configuration allows [keyless (open)](/basic-config-and-security/security/authentication-authorization/open-keyless) access to the API without any authentication. + +```yaml {hl_lines=["7-7"],linenos=false} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-keyless +spec: + name: httpbin-keyless + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true +``` + +#### Auth Token (Bearer Token) + +This setup requires a [bearer token](/api-management/authentication/bearer-token) for access. + +In the below example, the authentication token is set by default to the `Authorization` header of the request. You can customize this behavior by configuring the following fields: + +- `use_cookie`: Set to true to use a cookie value for the token. +- `cookie_name`: Specify the name of the cookie if use_cookie is enabled. +- `use_param`: Set to true to allow the token to be passed as a query parameter. +- `param_name`: Specify the parameter name if use_param is enabled. +- `use_certificate`: Enable client certificate. This allows you to create dynamic keys based on certificates. +- `validate_signature`: Enable [signature validation](/api-management/authentication/bearer-token#auth-token-with-signature). + +```yaml {hl_lines=["13-35"],linenos=false} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-auth-token +spec: + name: httpbin-auth-token + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true + use_standard_auth: true + auth_configs: + authToken: + # Auth Key Header Name + auth_header_name: Authorization + # Use cookie value + use_cookie: false + # Cookie name + cookie_name: "" + # Allow query parameter as well as header + use_param: false + # Parameter name + param_name: "" + # Enable client certificate + use_certificate: false + # Enable Signature validation + validate_signature: false + signature: + algorithm: "" + header: "" + secret: "" + allowed_clock_skew: 0 + error_code: 0 +``` + +#### JWT + +This configuration uses [JWT tokens](/basic-config-and-security/security/authentication-authorization/json-web-tokens) for authentication. + +Users can configure JWT authentication by defining the following fields: + +- `jwt_signing_method`: Specify the method used to sign the JWT. Refer to [JWT Signing Method](/basic-config-and-security/security/authentication-authorization/json-web-tokens#set-up-jwt-signing-method) for supported methods. +- `jwt_source`: Specify the public key used for verifying the JWT. +- `jwt_identity_base_field`: Define the identity source, typically set to `sub` (subject), which uniquely identifies the user or entity. +- `jwt_policy_field_name`: Specify the claim within the JWT payload that indicates the policy ID to apply. +- `jwt_default_policies` (Optional): Define default policies to apply if no policy claim is found in the JWT payload. + +The following example configures an API to use JWT authentication. It specifies the ECDSA signing method and public key, sets the `sub` claim as the identity source, uses the `pol` claim for policy ID, and assigns a default policy (`jwt-policy` SecurityPolicy in `default` namespace) if no policy is specified in the token. + +```yaml {hl_lines=["13-22"],linenos=false} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-jwt1 +spec: + name: httpbin-jwt1 + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin-jwt1 + strip_listen_path: true + enable_jwt: true + strip_auth_data: true + jwt_signing_method: ecdsa + # ecdsa pvt: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JR0hBZ0VBTUJNR0J5cUdTTTQ5QWdFR0NDcUdTTTQ5QXdFSEJHMHdhd0lCQVFRZ2V2WnpMMWdkQUZyODhoYjIKT0YvMk54QXBKQ3pHQ0VEZGZTcDZWUU8zMGh5aFJBTkNBQVFSV3oram42NUJ0T012ZHlIS2N2akJlQlNEWkgycgoxUlR3am1ZU2k5Ui96cEJudVE0RWlNbkNxZk1QV2lacUI0UWRiQWQwRTdvSDUwVnB1WjFQMDg3RwotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0t + # ecdsa pub: LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUZrd0V3WUhLb1pJemowQ0FRWUlLb1pJemowREFRY0RRZ0FFRVZzL281K3VRYlRqTDNjaHluTDR3WGdVZzJSOQpxOVVVOEk1bUVvdlVmODZRWjdrT0JJakp3cW56RDFvbWFnZUVIV3dIZEJPNkIrZEZhYm1kVDlQT3hnPT0KLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0t + jwt_source: LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUZrd0V3WUhLb1pJemowQ0FRWUlLb1pJemowREFRY0RRZ0FFRVZzL281K3VRYlRqTDNjaHluTDR3WGdVZzJSOQpxOVVVOEk1bUVvdlVmODZRWjdrT0JJakp3cW56RDFvbWFnZUVIV3dIZEJPNkIrZEZhYm1kVDlQT3hnPT0KLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0t + jwt_identity_base_field: sub + jwt_policy_field_name: pol + jwt_default_policies: + - default/jwt-policy +--- +apiVersion: tyk.tyk.io/v1alpha1 +kind: SecurityPolicy +metadata: + name: jwt-policy +spec: + access_rights_array: + - name: httpbin-jwt1 + namespace: default + versions: + - Default + active: true + name: jwt-policy + state: active +``` + +You can verify the API is properly authenticated with following command: + +1. JWT with default policy +```bash expandable +curl http://localhost:8080/httpbin-jwt1/get -H 'Authorization: Bearer eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ0ZXN0IiwiaWF0IjoxNTE2MjM5MDIyfQ.rgPyrCJYs2im7zG6im5XUqsf_oAf_Kqk-F6IlLb3yzZCSZvrQObhBnkLKgfmVTbhQ5El7Q6KskXPal5-eZFuTQ' +{ + "args": {}, + "headers": { + "Accept": "*/*", + "Accept-Encoding": "gzip", + "Host": "httpbin.org", + "Traceparent": "00-d2b93d763ca27f29181c8e508b5ac0c9-a446afa3bd053617-01", + "User-Agent": "curl/8.6.0", + "X-Amzn-Trace-Id": "Root=1-6696f0bf-1d9e532c6a2eb3a709e7086b" + }, + "origin": "127.0.0.1, 178.128.43.98", + "url": "http://httpbin.org/get" +} +``` + +2. JWT with explicit policy +```bash expandable +curl http://localhost:8080/httpbin-jwt1/get -H 'Authorization: Bearer eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ0ZXN0IiwiaWF0IjoxNTE2MjM5MDIyLCJwb2wiOiJaR1ZtWVhWc2RDOXFkM1F0Y0c5c2FXTjUifQ.7nY9TvYgsAZqIHLhJdUPqZtzqU_5T-dcNtCt4zt8YPyUj893Z_NopL6Q8PlF8TlMdxUq1Ff8rt4-p8gVboIqlA' +{ + "args": {}, + "headers": { + "Accept": "*/*", + "Accept-Encoding": "gzip", + "Host": "httpbin.org", + "Traceparent": "00-002adf6632ec20377cb7ccf6c3037e78-3c4cb97c70d790cb-01", + "User-Agent": "curl/8.6.0", + "X-Amzn-Trace-Id": "Root=1-6696f1dd-7f9de5f947c8c73279f7cca6" + }, + "origin": "127.0.0.1, 178.128.43.98", + "url": "http://httpbin.org/get" +} +``` + +#### Basic Authentication + +This configuration uses [Basic Authentication](/api-management/authentication/basic-authentication), requiring a username and password for access. + +```yaml {hl_lines=["13-13"],linenos=false} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-basic-auth +spec: + name: Httpbin Basic Authentication + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true + use_basic_auth: true +``` + +#### Custom Plugin Auth (go) + +This configuration uses a [Golang plugin](/api-management/plugins/golang#) for custom authentication. The following example shows how to create an API definition with a Golang custom plugin for `httpbin-go-auth`. + +For an example of Golang authentication middleware, see [Performing custom authentication with a Golang plugin](/api-management/plugins/golang#performing-custom-authentication-with-a-golang-plugin). + +```yaml {hl_lines=["7-7", "14-21"],linenos=false} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-go-auth +spec: + name: httpbin-go-auth + use_go_plugin_auth: true # Turn on GO auth + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true + custom_middleware: + driver: goplugin + pre: + - name: "AddFooBarHeader" + path: "/mnt/tyk-gateway/example-go-plugin.so" + auth_check: + name: "MyPluginCustomAuthCheck" + path: "/mnt/tyk-gateway/example-go-plugin.so" +``` + +#### Custom Plugin Auth (gRPC) + +This configuration uses a [gRPC plugin](/api-management/plugins/golang#) for custom authentication. The following example shows how to create an API definition with a gRPC custom plugin for `httpbin-grpc-auth`. + +For a detailed walkthrough on setting up Tyk with gRPC authentication plugins, refer to [Extending Tyk with gRPC Authentication Plugins](https://tyk.io/blog/how-to-setup-custom-authentication-middleware-using-grpc-and-java/). + +```yaml {hl_lines=["9-9", "14-26"],linenos=false} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-grpc-auth +spec: + name: httpbin-grpc-auth + protocol: http + active: true + enable_coprocess_auth: true + proxy: + target_url: http://httpbin.default.svc:8000 + listen_path: /httpbin-grpc-auth + strip_listen_path: true + custom_middleware: + driver: grpc + post_key_auth: + - name: "HelloFromPostKeyAuth" + path: "" + auth_check: + name: foo + path: "" + id_extractor: + extract_from: header + extract_with: value + extractor_config: + header_name: Authorization +``` + +#### Multiple (Chained) Auth + +This setup allows for [multiple authentication](/basic-config-and-security/security/authentication-authorization/multiple-auth) methods to be chained together, requiring clients to pass through each specified authentication provider. + +To enable multiple (chained) auth, you should set `base_identity_provided_by` field to one of the supported chained enums. Consult the [Multi (Chained) Authentication](/basic-config-and-security/security/authentication-authorization/multiple-auth) section for the supported auths. + +In this example, we are creating an API definition with basic authentication and mTLS with basic authentication as base identity for `httpbin-multiple-authentications`. + +```yaml {hl_lines=["19-21"],linenos=false} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-multiple-authentications +spec: + name: Httpbin Multiple Authentications + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + base_identity_provided_by: basic_auth_user + use_basic_auth: true + use_mutual_tls_auth: true +``` + +#### IP Allowlist + +To enable [IP Allowlist](/api-management/gateway-config-tyk-classic#ip-access-control), set the following fields: + +* `enable_ip_whitelisting`: Enables IPs allowlist. When set to `true`, only requests coming from the explicit list of IP addresses defined in (`allowed_ips`) are allowed through. +* `allowed_ips`: A list of strings that defines the IP addresses (in [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation) notation) that are allowed access via Tyk. + +In this example, only requests coming from 127.0.0.2 is allowed. + +```yaml {hl_lines=["10-12"],linenos=false} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin +spec: + name: httpbin + use_keyless: true + protocol: http + active: true + enable_ip_whitelisting: true + allowed_ips: + - 127.0.0.2 + proxy: + target_url: http://httpbin.default.svc:8000 + listen_path: /httpbin + strip_listen_path: true +``` + +#### IP Blocklist + +To enable [IP Blocklist](/api-management/gateway-config-tyk-classic#ip-access-control), set the following fields: + +* `enable_ip_blacklisting`: Enables IPs blocklist. If set to `true`, requests coming from the explicit list of IP addresses (blacklisted_ips) are not allowed through. +* `blacklisted_ips`: A list of strings that defines the IP addresses (in [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation) notation) that are blocked access via Tyk. This list is explicit and wildcards are currently not supported. + +In this example, requests coming from 127.0.0.2 will be forbidden (`403`). + +```yaml {hl_lines=["10-12"],linenos=false} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin +spec: + name: httpbin + use_keyless: true + protocol: http + active: true + enable_ip_blacklisting: true + blacklisted_ips: + - 127.0.0.2 + proxy: + target_url: http://httpbin.default.svc:8000 + listen_path: /httpbin + strip_listen_path: true +``` + + +### Set Up Manifest for GraphQL +In the example below we can see that the configuration is contained within the `graphql` configuration object. A GraphQL schema is specified within the `schema` field and the execution mode is set to `proxyOnly`. The [GraphQL public playground](/api-management/graphql#enabling-public-graphql-playground) is enabled with the path set to `/playground`. + +```yaml {hl_lines=["15-17", "18-92"],linenos=false} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: trevorblades +spec: + name: trevorblades + use_keyless: true + protocol: http + active: true + proxy: + target_url: https://countries.trevorblades.com + listen_path: /trevorblades + strip_listen_path: true + graphql: + enabled: true + version: "2" + execution_mode: proxyOnly + schema: | + directive @cacheControl(maxAge: Int, scope: CacheControlScope) on FIELD_DEFINITION | OBJECT | INTERFACE + + enum CacheControlScope { + PUBLIC + PRIVATE + } + + type Continent { + code: ID! + name: String! + countries: [Country!]! + } + + input ContinentFilterInput { + code: StringQueryOperatorInput + } + + type Country { + code: ID! + name: String! + native: String! + phone: String! + continent: Continent! + capital: String + currency: String + languages: [Language!]! + emoji: String! + emojiU: String! + states: [State!]! + } + + input CountryFilterInput { + code: StringQueryOperatorInput + currency: StringQueryOperatorInput + continent: StringQueryOperatorInput + } + + type Language { + code: ID! + name: String + native: String + rtl: Boolean! + } + + input LanguageFilterInput { + code: StringQueryOperatorInput + } + + type Query { + continents(filter: ContinentFilterInput): [Continent!]! + continent(code: ID!): Continent + countries(filter: CountryFilterInput): [Country!]! + country(code: ID!): Country + languages(filter: LanguageFilterInput): [Language!]! + language(code: ID!): Language + } + + type State { + code: String + name: String! + country: Country! + } + + input StringQueryOperatorInput { + eq: String + ne: String + in: [String] + nin: [String] + regex: String + glob: String + } + + """The `Upload` scalar type represents a file upload.""" + scalar Upload + playground: + enabled: true + path: /playground +``` + +### Set Up Manifest for HTTP +#### HTTP Proxy + +This example creates a basic API definition that routes requests to listen path `/httpbin` to target URL `http://httpbin.org`. + +Traffic routing can be configured under `spec.proxy`: +- `target_url` defines the upstream address (or target URL) to which requests should be proxied. +- `listen_path` is the base path on Tyk to which requests for this API should be sent. Tyk listens out for any requests coming into the host at this path, on the port that Tyk is configured to run on and processes these accordingly. For example, `/api/` or `/` or `/httpbin/`. +- `strip_listen_path` removes the inbound listen path (as accessed by the client) when generating the outbound request for the upstream service. For example, consider the scenario where the Tyk base address is `http://acme.com/`, the listen path is `example/` and the upstream URL is `http://httpbin.org/`: If the client application sends a request to `http://acme.com/example/get` then the request will be proxied to `http://httpbin.org/example/get` + +```yaml {hl_lines=["10-13"],linenos=false} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin +spec: + name: httpbin + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true +``` + +#### HTTP Host-based Proxy + +`spec.domain` is the domain to bind this API to. This enforces domain matching for client requests. + +In this example, requests to `httpbin.tyk.io` will be proxied to upstream URL `http://httpbin.org` + +```yaml {hl_lines=["10-10"],linenos=false} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin +spec: + name: httpbin + use_keyless: true + protocol: http + active: true + domain: httpbin.tyk.io + proxy: + target_url: http://httpbin.org + listen_path: / + strip_listen_path: true +``` + +#### HTTPS Proxy + +This example creates a API definition that routes requests to a http://httpbin.org via port 8443. + +```yaml {hl_lines=["35-38"],linenos=false} expandable +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer +spec: + selfSigned: { } +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: my-test-cert +spec: + secretName: my-test-tls + dnsNames: + - foo.com + - bar.com + privateKey: + rotationPolicy: Always + issuerRef: + name: selfsigned-issuer + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: Issuer + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io +--- +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin +spec: + name: httpbin + use_keyless: true + protocol: https + listen_port: 8443 + certificate_secret_names: + - my-test-tls + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true +``` + +### Set Up Manifest for TCP + +This example creates a API definition that proxies request from TCP port `6380` to `tcp://localhost:6379`. + +```yaml {hl_lines=["8-11"],linenos=false} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: redis-tcp +spec: + name: redis-tcp + active: true + protocol: tcp + listen_port: 6380 + proxy: + target_url: tcp://localhost:6379 +``` + +### Set Up Manifest for UDG +#### UDG v2 (Tyk 3.2 and above) + +If you are on Tyk 3.2 and above, you can use the following manifest to create an UDG API. This example configures a Universal Data Graph from a [GraphQL datasource](/api-management/data-graph#graphql) and a [REST Datasource](/api-management/data-graph#rest). + +```yaml {hl_lines=["20-39", "46-80"],linenos=false} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: udg +spec: + name: Universal Data Graph v2a + use_keyless: true + protocol: http + active: true + proxy: + target_url: "" + listen_path: /udg + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + graphql: + enabled: true + execution_mode: executionEngine + schema: | + type Country { + name: String + code: String + restCountry: RestCountry + } + + type Query { + countries: [Country] + } + + type RestCountry { + altSpellings: [String] + subregion: String + population: Int + } + version: "2" + last_schema_update: "2022-10-12T14:27:55.511+03:00" + type_field_configurations: [] + playground: + enabled: true + path: /playground + engine: + field_configs: + - disable_default_mapping: false + field_name: countries + path: + - "countries" + type_name: Query + - disable_default_mapping: true #very important for rest APIs + field_name: restCountry + path: [] + type_name: Country + data_sources: + - kind: "GraphQL" + name: "countries" + internal: false + root_fields: + - type: Query + fields: + - "countries" + config: + url: "https://countries.trevorblades.com/" + method: "POST" + headers: {} + body: "" + - kind: "REST" + internal: false + name: "restCountries" + root_fields: + - type: "Country" + fields: + - "restCountry" + config: + url: "https://restcountries.com/v2/alpha/{{ .object.code }}" + method: "GET" + body: "" + headers: {} +``` + +#### UDG v1 (Tyk 3.1 or before) + +If you are on Tyk 3.1, you can use the following manifest to create an UDG API. This example creates a Universal Data Graph with GraphQL datasource and HTTP JSON datasource. + +```yaml expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: udg +spec: + name: Universal Data Graph Example + use_keyless: true + protocol: http + active: true + proxy: + target_url: "" + listen_path: /udg + strip_listen_path: true + graphql: + enabled: true + execution_mode: executionEngine + schema: | + type Country { + name: String + code: String + restCountry: RestCountry + } + + type Query { + countries: [Country] + } + + type RestCountry { + altSpellings: [String] + subregion: String + population: String + } + type_field_configurations: + - type_name: Query + field_name: countries + mapping: + disabled: false + path: countries + data_source: + kind: GraphQLDataSource + data_source_config: + url: "https://countries.trevorblades.com" + method: POST + status_code_type_name_mappings: [] + - type_name: Country + field_name: restCountry + mapping: + disabled: true + path: "" + data_source: + kind: HTTPJSONDataSource + data_source_config: + url: "https://restcountries.com/v2/alpha/{{ .object.code }}" + method: GET + default_type_name: RestCountry + status_code_type_name_mappings: + - status_code: 200 + playground: + enabled: true + path: /playground +``` + +## Set Up Tyk Streams API +Tyk Streams integrates natively with Tyk OpenAPI Specification (OAS), allowing you to manage APIs as code and automate processes in Kubernetes using Tyk Operator. Setting up Tyk Streams API is similar to configuring a standard Tyk OAS API. You can store the Tyk Streams OAS definition in a Kubernetes ConfigMap and connect it to Tyk Gateway through a `TykStreamsApiDefinition` resource. + +### Create your Tyk Streams API +#### Prepare the Tyk Streams API Definition +To create a Tyk Streams API, start by preparing a complete Tyk Streams API definition in the OpenAPI Specification (OAS) format. This file must include: + +- The `x-tyk-api-gateway` extension for Tyk-specific settings. +- The `x-tyk-streaming` extension for Tyk Streams configuration. + +Here’s an example of a Tyk Streams API definition: + +```json {hl_lines=["17-54"],linenos=true} expandable +{ + "info": { + "title": "Simple streaming demo", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "servers": [ + { + "url": "http://tyk-gw.local/streams/" + } + ], + "security": [], + "paths": {}, + "components": { + "securitySchemes": {} + }, + "x-tyk-streaming": { + "streams": { + "example-publisher": { + "input": { + "http_server": { + "allowed_verbs": [ + "POST" + ], + "path": "/pub", + "timeout": "1s" + } + }, + "output": { + "http_server": { + "ws_path": "/ws" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "Simple streaming demo", + "state": { + "active": true, + "internal": false + } + }, + "server": { + "listenPath": { + "strip": true, + "value": "/streams/" + } + }, + "upstream": { + "url": "https://not-needed" + } + } +} +``` + +#### Create a TykStreamsApiDefinition Custom Resource +Once your Tyk Streams API definition is ready, use a Kubernetes ConfigMap to store the definition and link it to a `TykStreamsApiDefinition` custom resource. + +Example manifest: + +```yaml expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: TykStreamsApiDefinition +metadata: + name: simple-stream +spec: + tykStreams: + configmapRef: + name: simple-stream-cm #k8s resource name of configmap + namespace: default #The k8s namespace of the resource being targeted. If Namespace is not provided, + #we assume that the ConfigMap is in the same namespace as TykStreamsApiDefinition resource. + keyName: test_stream.json +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: simple-stream-cm +data: + test_stream.json: |- + { + "components": {}, + "info": { + "title": "Simple streaming demo", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": {}, + "x-tyk-api-gateway": { + "info": { + "name": "Simple streaming demo", + "state": { + "active": true + } + }, + "server": { + "detailedTracing": { + "enabled": true + }, + "listenPath": { + "strip": true, + "value": "/streams/" + } + }, + "upstream": { + "url": "https://not-needed" + } + }, + "x-tyk-streaming": { + "streams": { + "example-publisher": { + "input": { + "http_server": { + "path": "/pub", + "allowed_verbs": ["POST"], + "timeout": "1s" + } + }, + "output": { + "http_server": { + "ws_path": "/ws" + } + } + } + } + } + } +``` + +#### Apply the TykStreamsApiDefinition Manifest + +Use the `kubectl` command to apply the `TykStreamsApiDefinition` manifest to your Kubernetes cluster: + +```sh +kubectl apply -f tyk-streams-api-definition.yaml +``` + +This will create a new `TykStreamsApiDefinition` resource. The Tyk Operator watches this resource and configures the Tyk Gateway or Tyk Dashboard with the new API. + +#### Verify the Tyk Streams API Creation + +Check the status of the `TykStreamsApiDefinition` resource to ensure that the API has been successfully created: + +```sh +kubectl get tykstreamsapidefinitions simple-stream +``` + +You should see output similar to this: + +```bash +NAME DOMAIN LISTENPATH ENABLED SYNCSTATUS +simple-stream /streams/ true Successful +``` + +#### Manage and Update the Tyk Streams API +To update your API configuration, modify the linked `ConfigMap`. The Tyk Operator will automatically detect changes and update the API in the Tyk Gateway. + +### Secure your Tyk Streams API +To secure your Tyk Streams API, configure security fields in the OAS definition just as you would for a standard Tyk OAS API. For more details, refer to the [Secure your Tyk OAS API](#secure-your-tyk-oas-api) guide. + +## Add a Security Policy to your API +To further protect access to your APIs, you will want to add a security policy. +Below, we take you through how to define the security policy but you can also find [Security Policy Example](#security-policy-example) below. + +#### Define the Security Policy manifest + +To create a security policy, you must define a Kubernetes manifest using the `SecurityPolicy` CRD. The following example illustrates how to configure a default policy for trial users for a Tyk Classic API named `httpbin`, a Tyk OAS API named `petstore`, and a Tyk Streams API named `http-to-kafka`. + +```yaml expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: SecurityPolicy +metadata: + name: trial-policy # Unique Kubernetes name +spec: + name: Default policy for trial users # Descriptive name for the policy + state: active + active: true + access_rights_array: + - name: httpbin # Kubernetes name of referenced API + namespace: default # Kubernetes namespace of referenced API + kind: ApiDefinition # Omit this field or use `ApiDefinition` if you are referencing Tyk Classic API + versions: + - Default # The default version of Tyk Classic API is "Default" + - name: petstore + namespace: default + kind: TykOasApiDefinition # Use `TykOasApiDefinition` if you are referencing Tyk OAS API + versions: + - "" # The default version of Tyk OAS API is "" + - name: http-to-kafka + namespace: default + kind: TykStreamsApiDefinition # Use `TykStreamsApiDefinition` if you are referencing Tyk Streams API + versions: + - "" # The default version of Tyk Streams API is "" + quota_max: 1000 + quota_renewal_rate: 3600 + rate: 120 + per: 60 + throttle_interval: -1 + throttle_retry_limit: -1 +``` + +Save the manifest locally in a file, e.g. `trial-policy.yaml` + +In this example, we have defined a security policy as described below: + +**Define Security Policy status and metadata** + + - **`name`**: A descriptive name for the security policy. + - **`active`**: Marks the policy as active (true or false). + - **`state`**: The current state of the policy. It can have one of three values: + - **`active`**: Keys connected to this policy are enabled and new keys can be created. + - **`draft`**: Keys connected to this policy are disabled; no new keys can be created. + - **`deny`**: Policy is not published to Gateway; no keys can be created. + - **`tags`**: A list of tags to categorize or label the security policy, e.g. + + ```yaml expandable + tags: + - Hello + - World + ``` + + - **`meta_data`**: Key-value pairs for additional metadata related to the policy, e.g. + + ```yaml + meta_data: + key: value + hello: world + ``` + +**Define Access Lists for APIs** + + - **`access_rights_array`**: Defines the list of APIs that the security policy applies to and the versions of those APIs. + - **`name`**: The Kubernetes metadata name of the API resource to which the policy grants access. + - **`namespace`**: The Kubernetes namespace where the API resource is deployed. + - **`kind`**: Tyk OAS APIs (`TykOasApiDefinition`), Tyk Streams (`TykStreamsApiDefinition`) and Tyk Classic APIs (`ApiDefinition`) can be referenced here. The API format can be specified by `kind` field. If omitted, `ApiDefinition` is assumed. + - **`versions`**: Specifies the API versions the policy will cover. If the API is not versioned, include the default version here. The default version of a Classic API is "Default". The default version of a Tyk OAS API is "". + +In this example, the security policy will apply to an `ApiDefinition` resource named `httpbin` in the `default` namespace, a `TykOasApiDefinition` resource named `petstore` in the `default` namespace, and a `TykStreamsApiDefinition` resource named `http-to-kafka` in the `default` namespace. Note that with Tyk Operator, you do not need to specify API ID as in the raw [Policy definition](/api-management/policies#policies-guide). Tyk Operator will automatically retrieve the API ID of referenced API Definition resources for you. + +**Define Rate Limits, Usage Quota, and Throttling** + +- **`rate`**: The maximum number of requests allowed per time period (Set to `-1` to disable). +- **`per`**: The time period (in seconds) for the rate limit (Set to `-1` to disable). +- **`throttle_interval`**: The interval (in seconds) between each request retry (Set to `-1` to disable). +- **`throttle_retry_limit`**: The maximum number of retry attempts allowed (Set to `-1` to disable). +- **`quota_max`**: The maximum number of requests allowed over a quota period (Set to `-1` to disable). +- **`quota_renewal_rate`**: The time, in seconds, after which the quota is renewed. + +In this example, trial users under this security policy can gain access to the `httpbin` API at a rate limit of maximum 120 times per 60 seconds (`"rate": 120, "per": 60`), with a usage quota of 1000 every hour (`"quota_max": 1000, "quota_renewal_rate": 3600`), without any request throttling (`throttle_interval: -1, throttle_retry_limit: -1`). + +#### Apply the Security Policy manifest +Once you have defined your security policy manifest, apply it to your Kubernetes cluster using the `kubectl apply` command: + +```bash +kubectl apply -f trial-policy.yaml +``` + +#### Verify the Security Policy + +After applying the manifest, you can verify that the security policy has been created successfully by running: + +```bash +kubectl describe securitypolicy trial-policy + +... +Status: + Latest CRD Spec Hash: 901732141095659136 + Latest Tyk Spec Hash: 5475428707334545086 + linked_apis: + Kind: ApiDefinition + Name: httpbin + Namespace: default + Kind: TykOasApiDefinition + Name: petstore + Namespace: default + Kind: TykStreamsApiDefinition + Name: http-to-kafka + Namespace: default + pol_id: 66e9a27bfdd3040001af6246 +Events: +``` + +From the `status` field, you can see that this security policy has been linked to `httpbin`, `petstore`, and `http-to-kafka` APIs. + + +

Security policy example

+
Key-level per-API rate limits and quota
+By configuring per-API limits, you can set specific rate limits, quotas, and throttling rules for each API in the access rights array. When these per-API settings are enabled, the API inherits the global limit settings unless specific limits and quotas are set in the `limit` field for that API. + +The following manifest defines a security policy with per-API rate limits and quotas for two APIs: `httpbin` and `petstore`. + +```yaml {hl_lines=["15-21", "27-33", "40-41"],linenos=true} +apiVersion: tyk.tyk.io/v1alpha1 +kind: SecurityPolicy +metadata: + name: policy-per-api-limits +spec: + name: Policy with Per API Limits + state: active + active: true + access_rights_array: + - name: httpbin # Kubernetes name of referenced API + namespace: default # Kubernetes namespace of referenced API + kind: ApiDefinition # `ApiDefinition` (Default), `TykOasApiDefinition` or `TykStreamsApiDefinition` + versions: + - Default # The default version of Tyk Classic API is "Default" + limit: # APILimit stores quota and rate limit on ACL level + rate: 10 # Max 10 requests per 60 seconds + per: 60 # Time period for rate limit + quota_max: 100 # Max 100 requests allowed over the quota period + quota_renewal_rate: 3600 # Quota renewal period in seconds (1 hour) + throttle_interval: -1 # No throttling between retries + throttle_retry_limit: -1 # No limit on request retries + - name: petstore + namespace: default + kind: TykOasApiDefinition # Use `TykOasApiDefinition` for Tyk OAS API + versions: + - "" # The default version of Tyk OAS API is "" + limit: + rate: 5 # Max 5 requests per 60 seconds + per: 60 # Time period for rate limit + quota_max: 100 # Max 100 requests allowed over the quota period + quota_renewal_rate: 3600 # Quota renewal period in seconds (1 hour) + throttle_interval: -1 # No throttling between retries + throttle_retry_limit: -1 # No limit on request retries + rate: -1 # Disable global rate limit + per: -1 # Disable global rate limit period + throttle_interval: -1 # Disable global throttling + throttle_retry_limit: -1 # Disable global retry limit + quota_max: -1 # Disable global quota + quota_renewal_rate: 60 # Quota renewal rate in seconds (1 minute) +``` expandable + +With this security policy applied: + +For the `httpbin` API: +- The rate limit allows a maximum of 10 requests per 60 seconds. +- The quota allows a maximum of 100 requests per hour (3600 seconds). +- There is no throttling or retry limit (throttle_interval and throttle_retry_limit are set to -1). + +For the `petstore` API: +- The rate limit allows a maximum of 5 requests per 60 seconds. +- The quota allows a maximum of 100 requests per hour (3600 seconds). +- There is no throttling or retry limit (throttle_interval and throttle_retry_limit are set to -1). + +Global Rate Limits and Quota: +- All global limits (rate, quota, and throttling) are disabled (-1), so they do not apply. + +By setting per-API rate limits and quotas, you gain granular control over how each API is accessed and used, allowing you to apply different limits for different APIs as needed. This configuration is particularly useful when you want to ensure that critical APIs have stricter controls while allowing more flexibility for others. Use this example as a guideline to tailor your security policies to your specific requirements. + +
Key-level per-endpoint rate limits
+By configuring key-level per-endpoint limits, you can restrict the request rate for specific API clients to a specific endpoint of an API. + +The following manifest defines a security policy with per-endpoint rate limits for two APIs: `httpbin` and `petstore`. + +```yaml {hl_lines=["15-29", "35-49"],linenos=true} +apiVersion: tyk.tyk.io/v1alpha1 +kind: SecurityPolicy +metadata: + name: policy-per-api-limits +spec: + name: Policy with Per API Limits + state: active + active: true + access_rights_array: + - name: httpbin # Kubernetes name of referenced API + namespace: default # Kubernetes namespace of referenced API + kind: ApiDefinition # `ApiDefinition` (Default), `TykOasApiDefinition` or `TykStreamsApiDefinition` + versions: + - Default # The default version of Tyk Classic API is "Default" + endpoints: # Per-endpoint rate limits + - path: /anything + methods: + - name: POST + limit: + rate: 5 + per: 60 + - name: PUT + limit: + rate: 5 + per: 60 + - name: GET + limit: + rate: 10 + per: 60 + - name: petstore + namespace: default + kind: TykOasApiDefinition # Use `TykOasApiDefinition` for Tyk OAS API + versions: + - "" # The default version of Tyk OAS API is "" + endpoints: # Per-endpoint rate limits + - path: /pet + methods: + - name: POST + limit: + rate: 5 + per: 60 + - name: PUT + limit: + rate: 5 + per: 60 + - name: GET + limit: + rate: 10 + per: 60 + rate: -1 # Disable global rate limit + per: -1 # Disable global rate limit period + throttle_interval: -1 # Disable global throttling + throttle_retry_limit: -1 # Disable global retry limit + quota_max: -1 # Disable global quota + quota_renewal_rate: 60 # Quota renewal rate in seconds (1 minute) +``` + +
Path based permissions
+You can secure your APIs by specifying [allowed URLs](/api-management/policies#secure-your-apis-by-method-and-path) (methods and paths) for each API within a security policy. This is done using the `allowed_urls` field under `access_rights_array`. + +The following manifest defines a security policy that allows access only to specific URLs and HTTP methods for two APIs: `httpbin`(a Tyk Classic API) and `petstore` (a Tyk OAS API). + +```yaml {hl_lines=["15-18", "24-28"],linenos=true} +apiVersion: tyk.tyk.io/v1alpha1 +kind: SecurityPolicy +metadata: + name: policy-with-allowed-urls +spec: + name: Policy with allowed URLs + state: active + active: true + access_rights_array: + - name: httpbin # Kubernetes name of referenced API + namespace: default # Kubernetes namespace of referenced API + kind: ApiDefinition # `ApiDefinition` (Default), `TykOasApiDefinition` or `TykStreamsApiDefinition` + versions: + - Default # The default version of Tyk Classic API is "Default" + allowed_urls: # Define allowed paths and methods + - url: /get # Only allow access to the "/get" path + methods: + - GET # Only allow the GET method + - name: petstore + namespace: default + kind: TykOasApiDefinition # Use `TykOasApiDefinition` for Tyk OAS API + versions: + - "" # The default version of Tyk OAS API is "" + allowed_urls: # Define allowed paths and methods + - url: "/pet/(.*)" # Allow access to any path starting with "/pet/" + methods: + - GET # Allow GET method + - POST # Allow POST method +``` expandable + +With this security policy applied: + +- Allowed access: + - `curl -H "Authorization: Bearer $KEY_AUTH" http://tyk-gw.org/petstore/pet/10` returns a `200 OK` response. + - `curl -H "Authorization: Bearer $KEY_AUTH" http://tyk-gw.org/httpbin/get` returns a `200 OK` response. + +- Restricted access: + - `curl -H "Authorization: Bearer $KEY_AUTH" http://tyk-gw.org/petstore/pet` returns a `403 Forbidden` response with the message: + + ```json + { "error": "Access to this resource has been disallowed" } + ``` + + - `curl -H "Authorization: Bearer $KEY_AUTH" http://tyk-gw.org/httpbin/anything` returns a `403 Forbidden` response with the message: + + ```json + { "error": "Access to this resource has been disallowed" } + ``` + +
Partitioned policies
+[Partitioned policies](/api-management/policies#partitioned-policies) allow you to selectively enforce different segments of a security policy, such as quota, rate limiting, access control lists (ACL), and GraphQL complexity rules. This provides flexibility in applying different security controls as needed. + +To configure a partitioned policy, set the segments you want to enable in the `partitions` field: + +```yaml +apiVersion: tyk.tyk.io/v1alpha1 +kind: SecurityPolicy +metadata: + name: partitioned-policy-example +spec: + name: Partitioned Policy Example + state: active + active: true + access_rights_array: + - name: httpbin # Kubernetes name of referenced API + namespace: default # Kubernetes namespace of referenced API + kind: ApiDefinition # `ApiDefinition` (Default), `TykOasApiDefinition` or `TykStreamsApiDefinition` + versions: + - Default # The default version of Tyk Classic API is "Default" + - name: petstore + namespace: default + kind: TykOasApiDefinition # Use `TykOasApiDefinition` if you are referencing Tyk OAS API + versions: + - "" # The default version of Tyk OAS API is "" + partitions: + quota: false # Do not enforce quota rules + rate_limit: false # Do not enforce rate limiting rules + acl: true # Enforce access control rules + complexity: false # Do not enforce GraphQL complexity rules +``` expandable + +- **`quota`**: Set to true to enforce quota rules (limits the number of requests allowed over a period). +- **`rate_limit`**: Set to true to enforce rate limiting rules (limits the number of requests per second or minute). +- **`acl`**: Set to true to enforce access control rules (controls which APIs or paths can be accessed). +- **`complexity`**: Set to true to enforce GraphQL complexity rules (limits the complexity of GraphQL queries to prevent resource exhaustion). + + +## Migrate Existing APIs to Tyk Operator +If you have existing APIs and Policies running on your Tyk platform, and you want to start using Tyk Operator to manage them, you probably would not want to re-create the APIs and Policies on the platform using Operator CRDs. It is because you will lose keys, policies, and analytics linked to the APIs. You can instead link existing APIs and Policies to a CRD by specifying the API ID or Policy ID in the CRD spec. This way, Operator will update the existing API or Policy according to the CRD spec. Any keys, policies and analytics linked to the API will continue to operate the same. This is great for idempotency. + +### Export existing configurations to CRDs + +Instead of creating the API and Policy CRDs from scratch, you can try exporting them from Dashboard using a snapshot tool. You can find the detail usage guide [here](https://github.com/TykTechnologies/tyk-operator/blob/master/pkg/snapshot/README.md). This is great if you want to have a quick start. However, this is still a PoC feature so we recommend you to double check the output files before applying them to your cluster. + +### Migration of existing API + +If there are existing APIs that you want to link to a CRD, it's very easy to do so. You need to simply add the `api_id` from your API Definition to the YAML of your `ApiDefinition` type. Then, the Operator will take care of the rest. + +Example: + +1. From the existing API Definition, grab the following field: + +```json +"api_id": "5e0fac4845bb46c77543be28300fd9d7" +``` + +2. Simply add this value to your YAML, in the `spec.api_id`field: + +```yaml +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: my-existing-api +spec: + api_id: 5e0fac4845bb46c77543be28300fd9d7 + name: existing API + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true +``` + +3. Then apply your changes: + +```console +$ kubectl apply -f config/samples/httpbin_protected.yaml +apidefinition.tyk.tyk.io/my-existing-api created +``` expandable + + + +The source of truth for the API definition is now the CRD, meaning it will override any differences in your existing API definition. + + + +### Migration of existing Policy +If you have existing pre-Operator policies, you can easily link them to a CRD, which will allow you to modify them through the YAML moving forward. +Simply set the id field in the SecurityPolicy YAML to the _id field in the existing Policy's JSON. This will allow the Operator to make the link. +Note that the YAML becomes the source of truth and will overwrite any changes between it and the existing Policy. + +**Example**: +1. Find out your existing Policy ID, e.g. `5f8f3933f56e1a5ffe2cd58c` + +2. Stick the policy ID `5f8f3933f56e1a5ffe2cd58c` into the YAML's `spec.id` field like below + +```yaml +my-security-policy.yaml: +apiVersion: tyk.tyk.io/v1alpha1 +kind: SecurityPolicy +metadata: + name: new-httpbin-policy +spec: + id: 5f8f3933f56e1a5ffe2cd58c + name: My New HttpBin Policy + state: active + active: true + access_rights_array: + - name: new-httpbin-api # name of your ApiDefinition object. + namespace: default # namespace of your ApiDefinition object. + versions: + - Default +``` + +The `spec.access_rights_array` field of the YAML must refer to the ApiDefinition object that the policy identified by the id will affect. + +To find available ApiDefinition objects: + +```console +$ kubectl get tykapis -A +NAMESPACE NAME DOMAIN LISTENPATH PROXY.TARGETURL ENABLED +default new-httpbin-api /httpbin http://httpbin.org true +``` + +3. And then apply this file: + +```console +$ kubectl apply -f my-security-policy.yaml +securitypolicy.tyk.tyk.io/new-httpbin-policy created +``` expandable + +Now the changes in the YAML were applied to the existing Policy. You can now manage this policy through the CRD moving forward. +Note, if this resource is unintentionally deleted, the Operator will recreate it with the same `id` field as above, allowing keys to continue to work as before the delete event. + +### Idempotency + +Because of the ability to declaratively define the `api_id`, this gives us the ability to preserve Keys that are tied to APIs or policies which are tied to APIs. +Imagine any use case where you have keys tied to policies, and policies tied to APIs. +Now imagine that these resources are unintentionally destroyed. Our database goes down, or our cluster, or something else. +Well, using the Tyk Operator, we can easily re-generate all our resources in a non-destructive fashion. That's because the operator intelligently constructs the unique ID using the unique namespaced name of our CRD resources. For that reason. +Alternatively, if you don't explicitly state it, it will be hard-coded for you by Base64 encoding the namespaced name of the CRD. + +For example: + +1. we have keys tied to policies tied to APIs in production. +2. Our production DB gets destroyed, all our Policies and APIs are wiped +3. The Tyk Operator can resync all the changes from our CRDs into a new environment, by explicitly defining the Policy IDs and API IDs as before. +4. This allows keys to continue to work normally as Tyk resources are generated idempotently through the Operator. + + +## Publish Your API to Dev Portal +For Tyk Self Managed or Tyk Cloud, you can set up a Developer Portal to expose a facade of your APIs and then allow third-party developers to register and use your APIs. +You can make use of Tyk Operator CRDs to publish the APIs as part of your CI/CD workflow. If you have followed this Getting Started guide to create the httpbin example API, you can publish it to your Tyk Classic Developer Portal in a few steps. + + + +Currently Operator only supports publishing Tyk Classic API to the Tyk Classic Portal. + + + +### Publish an API with Tyk Operator +1. **Creating a security policy** + +When you publish an API to the Portal, Tyk actually publishes a way for developers to enroll in a policy, not into the API directly. Therefore, you should first set up a security policy for the developers, before proceeding with the publishing. + +To do that, you can use the following command: + +```yml +cat < +```yaml {hl_lines=["6-7", "10-48"],linenos=true} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: httpbin-ingress + annotations: + kubernetes.io/ingress.class: tyk # <----------------- REFERENCES TYK INGRESS CONTROLLER +spec: + rules: + - host: httpbin.ahmet + http: + paths: + - path: / # host routing: http://httpbin.ahmet/ + pathType: Prefix + backend: + service: + name: httpbin1 + port: + number: 8000 + - path: /httpbin # host + path routing: http://httpbin.ahmet/httpbin + pathType: Prefix + backend: + service: + name: httpbin2 + port: + number: 8000 + - http: + paths: + - path: /pathonly # path only routing: http://IPADDRESS/pathonly + pathType: Prefix + backend: + service: + name: httpbin3 + port: + number: 8000 + - host: "*.foo.com" # wildcard + # curl http://bar.foo.com/httpbin/get === OK Matches based on shared suffix + # curl http://baz.bar.foo.com/httpbin/get === NOK No match, wildcard only covers a single DNS label + # curl http://foo.com/httpbin/get === NOK No match, wildcard only covers a single DNS label + http: + paths: + - path: /httpbin + pathType: Prefix + backend: + service: + name: httpbin4 + port: + number: 8000 +``` expandable + +In this example, 4 APIs will be created by Tyk Operator. It illustrates how different Ingress rules: host based routing, path based routing, host + path based routing, and wildcard hosts are handled by Tyk Operator. + +| API Name | Custom Domain | Listen Path | Target URL | Example request that would be handled by this API | +| :---------- | :--------------- | :------------- | :------------ | :--------------------------------------------------- | +| default-httpbin-ingress-a1863f096 | httpbin.ahmet | / | http://httpbin1.default.svc.cluster.local:8000 | http://httpbin.ahmet/ | +| default-httpbin-ingress-d33713b8b | httpbin.ahmet | /httpbin | http://httpbin2.default.svc.cluster.local:8000 | http://httpbin.ahmet/httpbin | +| default-httpbin-ingress-00254eeb0 | | /pathonly | http://httpbin3.default.svc.cluster.local:8000 | http://IPADDRESS/pathonly | +| default-httpbin-ingress-3af1fef04 | `{?:[^.]+}.foo.com` | /httpbin | http://httpbin4.default.svc:8000 | http://bar.foo.com/httpbin | + + +#### HTTPS with cert-manager integration + + +```yaml {hl_lines=["7-7", "13-13", "15-24", "58-58"],linenos=true} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: httpbin-ingress-tls + annotations: + kubernetes.io/ingress.class: tyk # <----------------- REFERENCES TYK INGRESS CONTROLLER + cert-manager.io/cluster-issuer: "letsencrypt-staging" # this annotation indicates the issuer to use. + acme.cert-manager.io/http01-edit-in-place: "true" +spec: + tls: + - hosts: # < placing a host in the TLS config will determine what ends up in the cert's subjectAltNames + - myingress.do.poc.tyk.technology + secretName: httpbin-ingress-tls-secret # < cert-manager will store the created certificate in this secret. + rules: + - host: myingress.do.poc.tyk.technology + http: + paths: + - path: /httpbin + pathType: Prefix + backend: + service: + name: httpbin + port: + number: 8000 +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-staging +spec: + acme: + server: https://acme-staging-v02.api.letsencrypt.org/directory + email: ahmet@tyk.io + privateKeySecretRef: + name: letsencrypt-staging + solvers: + - http01: + ingress: + class: tyk +--- +apiVersion: v1 +kind: Service +metadata: + name: ingress-gateway + namespace: tyk +spec: + ports: + - name: http + targetPort: 8000 + port: 80 + protocol: TCP + - name: https + targetPort: 443 + port: 443 + protocol: TCP + selector: + app: gateway-tyk-stack-tyk-gateway + type: LoadBalancer + externalTrafficPolicy: Local +``` expandable + +A common use-case for [cert-manager](https://cert-manager.io/docs/usage/ingress/) is requesting TLS signed certificates to secure your ingress resources. This can be done by simply adding [annotations](https://cert-manager.io/docs/usage/ingress/#supported-annotations), such as `cert-manager.io/cluster-issuer`, to your Ingress resources and cert-manager will facilitate creating the `Certificate` resource for you. + +In this example, cert-manager watches the ingress resource `httpbin-ingress-tls` and ensures a TLS secret named `httpbin-ingress-tls-secret` (provided by the `tls.secretName` field) in the same namespace will be created and configured as described on the Ingress. This example also exposes Tyk Gateway as a `LoadBalancer` service with the `ingress-gateway` resource. This is essential for completing the ACME challenge from [Let\'s Encrypt](https://letsencrypt.org). + +With this configuration, Tyk Gateway can serve HTTPS requests via port 443, with a TLS certificate provisioned by cert-manager. An API is created by Tyk Operator to serve the ingress traffic at https://myingress.do.poc.tyk.technology/httpbin, and forwards the request to http://httpbin.default.svc:8000 within the cluster. + + +#### ApiDefinition Template + + +```yaml{hl_lines=["5-6"],linenos=true} +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: myapideftemplate + labels: + template: "true" # Instructs Tyk Operator to skip reconciliation for this resource +spec: + name: foo + protocol: http + use_keyless: true + proxy: + target_url: http://example.com +``` + +This example defines an `ApiDefinition` resource that can be used as configuration template for APIs created for Ingresses. It has a label `template: "true"` which let Tyk Operator knows that it is not a real resource, and hence does not require reconciliation. This will allow the ApiDefinition to be stored inside Kubernetes as a resource, but will not reconcile the ApiDefinition inside Tyk. All mandatory fields inside the ApiDefinition spec are still mandatory, but can be replaced with placeholders as they will be overwritten by the Ingress reconciler. + +```yaml{hl_lines=["7-8"],linenos=true} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: my-ingress + annotations: + kubernetes.io/ingress.class: tyk # <----------------- REFERENCES TYK INGRESS CONTROLLER + tyk.io/template: myapideftemplate # The metadata name of the ApiDefinition or TykOasApiDefinition resource in the same namespace + tyk.io/template-kind: ApiDefinition # Can be "ApiDefinition" (Default) or "TykOasApiDefinition" +... +``` + +To make use of the ApiDefinition template, make sure to add annotations `tyk.io/template` and `tyk.io/template-kind` to your Ingress resource. Here, we specify that the template to be used is named "myapideftemplate", and the resource represents a Tyk Classic API "ApiDefinition". + + +#### TykOasApiDefinition Template + + +```yaml{hl_lines=["39-40"],linenos=true} +apiVersion: v1 +data: + test_oas.json: |- + { + "info": { + "title": "OAS Template", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "components": {}, + "paths": {}, + "x-tyk-api-gateway": { + "info": { + "name": "OAS Template", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://example" + }, + "server": { + "listenPath": { + "value": "/example/", + "strip": true + } + } + } + } +kind: ConfigMap +metadata: + name: cm + namespace: default +--- +apiVersion: tyk.tyk.io/v1alpha1 +kind: TykOasApiDefinition +metadata: + name: oasapitemplate + labels: + tyk.io/ingress-template: "true" +spec: + tykOAS: + configmapRef: + name: cm + namespace: default + keyName: test_oas.json +``` + +Here provides a minimum template as `TykOasApiDefinition`. The `TykOasApiDefinition` must have a label `tyk.io/ingress-template: "true"` so that Tyk Operator will not reconcile it with Tyk. + +```yaml{hl_lines=["7-8"],linenos=true} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: my-ingress + annotations: + kubernetes.io/ingress.class: tyk # <----------------- REFERENCES TYK INGRESS CONTROLLER + tyk.io/template: oasapitemplate # The metadata name of the ApiDefinition or TykOasApiDefinition resource in the same namespace + tyk.io/template-kind: TykOasApiDefinition # Can be "ApiDefinition" (Default) or "TykOasApiDefinition" +... +``` expandable + +To make use of the TykOasApiDefinition template, make sure to add annotations `tyk.io/template` and `tyk.io/template-kind` to your Ingress resource. Here, we specify that the template to be used is named "oasapitemplate", and the resource represents a Tyk OAS API "TykOasApiDefinition". + + +### Ingress Class + +The value of the `kubernetes.io/ingress.class` annotation identifies the IngressClass that will process Ingress objects. + +Tyk Operator by default looks for the value `tyk` and will ignore all other ingress classes. If you wish to override this default behavior, + you may do so by setting the environment variable `WATCH_INGRESS_CLASS` in the operator manager deployment. [See Installing Tyk Operator](#install-and-configure-tyk-operator) for further information. + +### API name + +Tyk Ingress Controller will create APIs in Tyk for each path defined for a specific rule in Ingress resource. Each API created inside Tyk will follow a special naming convention as follows: + +``` +-- +``` + +For example, the following ingress resource will create an ApiDefinition called `default-httpbin-ingress-78acd160d` inside Tyk's Gateway. +ApiDefinition's name comes from: + +- `default`: The namespace of this Ingress resource, +- `httpbin-ingress`: The name of this Ingress resource, +- `78acd160d`: Short hash (first 9 characters) of Host (`""`) and Path (`/httpbin`). The hash algorithm is SHA256. + +```yaml{hl_lines=["4-4"],linenos=true} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: httpbin-ingress + annotations: + kubernetes.io/ingress.class: tyk # <----------------- REFERENCES TYK INGRESS CONTROLLER + tyk.io/template: myapideftemplate # <---------------- REFERENCE TO APIDEFINITION IN SAME NAMESPACE +spec: + rules: + - http: + paths: + - path: /httpbin + pathType: Prefix + backend: + service: + name: httpbin + port: + number: 8000 +``` expandable + +### Ingress Path Types + +Each path in an Ingress must have its own particular path type. Kubernetes offers three types of path types: `ImplementationSpecific`, `Exact`, and `Prefix`. Currently, not all path types are supported. The below table shows the unsupported path types for [Sample HTTP Ingress Resource](#set-up-manifest-for-http) based on the examples in the [Kubernetes Ingress documentation](https://kubernetes.io/docs/concepts/services-networking/ingress/#examples). + +| Kind | Path(s) | Request path(s) | Expected to match? | Works as Expected | +| :-------- | :----------- | :----------------- | :---------------------------------- | :----------------------------------------- | +| Exact | /foo | /foo/ | No | No. | +| Prefix | /foo/ | /foo, /foo/ | Yes | No, /foo/ matches, /foo does not match. | +| Prefix | /aaa/bb | /aaa/bbb | No | No, the request forwarded to service. | +| Prefix | /aaa/bbb/ | /aaa/bbb | Yes, ignores trailing slash | No, /aaa/bbb does not match. | +| Prefix | /aaa/bbb | /aaa/bbbxyz | No, does not match string prefix | No, the request forwarded to service. | + +Please bear in mind that if `proxy.strip_listen_path` is set to true on API Definition, Tyk strips the listen-path (for example, the listen-path for the Ingress under [Sample HTTP Ingress Resource](#set-up-manifest-for-http) is /httpbin) with an empty string. + +The following table shows an example of path matching if the listen-path is set to `/httpbin` or `/httpbin/`. + +| Kind | Path(s) | Request path(s) | Matches? | +| :------------------------ | :----------- | :--------------------------- | :------------------------------------------------------- | +| Exact | /httpbin | /httpbin, /httpbin/ | Yes. The request forwarded as `/` to your service. | +| Prefix | /httpbin | /httpbin, /httpbin/ | Yes. The request forwarded as `/` to your service. | +| ImplementationSpecific | /httpbin | /httpbin, /httpbin/ | Yes. The request forwarded as `/` to your service. | +| Exact | /httpbin | /httpbinget, /httpbin/get | Yes. The request forwarded as `/get` to your service. | +| Prefix | /httpbin | /httpbinget, /httpbin/get | Yes. The request forwarded as `/get` to your service. | +| ImplementationSpecific | /httpbin | /httpbinget, /httpbin/get | Yes. The request forwarded as `/get` to your service. | +| Exact | /httpbin/ | /httpbin/, /httpbin/get | Yes. The request forwarded as `/get` to your service. | +| Prefix | /httpbin/ | /httpbin/, /httpbin/get | Yes. The request forwarded as `/get` to your service. | +| ImplementationSpecific | /httpbin/ | /httpbin/, /httpbin/get | Yes. The request forwarded as `/get` to your service. | +| Exact | /httpbin/ | /httpbin | No. Ingress cannot find referenced service. | +| Prefix | /httpbin/ | /httpbin | No. Ingress cannot find referenced service. | +| ImplementationSpecific | /httpbin/ | /httpbin | No. Ingress cannot find referenced service. | + +## GraphQL Federation with Tyk Operator + +Tyk, with release *v4.0* offers [GraphQL federation](/api-management/graphql#federation-version-support) that allows you to divide GraphQL implementation across multiple back-end +services, while still exposing them all as a single graph for the consumers. + +Tyk Operator supports GraphQL Federation subgraph and supergraph with following Custom Resources. + +### Custom Resources + +GraphQL Federation uses concepts of Subgraph and Supergraph. + +**Subgraph** is a representation of a back-end service and defines a distinct GraphQL schema. It can be queried directly as a separate service or it can be federated into a larger schema of a supergraph. + +**Supergraph** is a composition of several subgraphs that allows the execution of a query across multiple services in the backend. + +Tyk Operator uses Custom Resources called [SubGraph](#subgraph) and [SuperGraph](#supergraph), that allows users to model the relationship between Subgraphs and Supergraphs. + +#### SubGraph + +```yaml +apiVersion: tyk.tyk.io/v1alpha1 +kind: SubGraph +metadata: + name: users-subgraph +spec: + schema: | + + sdl: | + +``` + +SubGraph Custom Resource Definitions (CRD) takes `schema` and `sdl` values for your subgraph. + +To create a Subgraph API in Tyk, you can reference the subgraph's metadata name through `graphql.graph_ref` field, as follows: + +```yaml +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: subgraph-api +spec: + name: Federation - Subgraph + ... + graphql: + enabled: true + execution_mode: subgraph + graph_ref: users-subgraph ## corresponds to Subgraph resource's metadata name + version: "2" + playground: + enabled: false + path: "" + proxy: + target_url: http://users.default.svc:4001/query + listen_path: /users-subgraph/ + disable_strip_slash: true +``` + +An ApiDefinition must adhere to the following rules in order to represent an ApiDefinition for your SubGraph CRDs. + +1. ApiDefinition and SubGraph must be in the same namespace, +2. `graphql.execution_mode` must be set to `subgraph`, +3. `graphql.graph_ref` must be set to the metdata name of the SubGraph resource that you would like to refer. + +#### SuperGraph + +```yaml +apiVersion: tyk.tyk.io/v1alpha1 +kind: SuperGraph +metadata: + name: social-media-supergraph +spec: + subgraph_refs: + - name: users-subgraph + namespace: default + schema: |- + +``` + +SuperGraph CRD takes `subgraph_refs` and `schema` values for your supergraph. `subgraph_refs` is an array of SubGraph Custom Resource(CR) references which expects the name and namespace of the referenced subgraph. If `namespace` is not specified, Operator will check SubGraphs in the current namespace. + +Tyk Operator will update your SuperGraph ApiDefinition when one of the subgraphs that you reference in `subgraph_refs` changes. + +To create a SuperGraph API in Tyk, you can reference the supergraph's metadata name through `graphql.graph_ref field`, as follows: + +```yaml +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: federation-supergraph +spec: + name: Federated - Social Media APIS + ... + graphql: + execution_mode: supergraph + graph_ref: social-media-supergraph ## corresponds to SuperGraph resource's metadata name + enabled: true + version: "2" + playground: + enabled: true + path: /playground + proxy: + target_url: "" + strip_listen_path: true + listen_path: /social-media-apis-federated/ +``` expandable + +An ApiDefinition must adhere to the following rules in order to represent an ApiDefinition for your SuperGraph CRDs. + +1. ApiDefinition and SuperGraph must be in the same namespace, +2. `graphql.execution_mode` must be set to `supergraph`, +3. `graphql.graph_ref` must be set to the metdata name of the SuperGraph resource that you would like to refer. + +### Propagating Subgraph Changes to Supergraph + +Tyk Operator will automatically propagate changes in SubGraph CRD to the corresponding Subgraph ApiDefinition. Also, if the SubGraph is referenced by a SuperGraph, the corresponding SuperGraph CR and corresponding supergraph ApiDefinition will be updated too. + +Therefore, once you make an update on SubGraph CR, you do not need to update your supergraph. It will be updated by Tyk Operator. With this approach, multiple teams can work on SubGraph CRDs and Tyk Operator will update the relevant SuperGraph ApiDefinition. + +#### Example + +Let's assume that a developer responsible for the [Users SubGraph](#users-subgraph) would like to delete `username` field from the Users SubGraph. +Also, the [Supergraph](#supergraph-1) called Social Media already uses the Users Subgraph. + +To achieve this, the developer should update the Users SubGraph CRD. Once the SubGraph CRD is updated, Tyk Operator will: +1. Update Users SubGraph CRD, +2. Update Social Media Supergraph ApiDefinition since it is referencing the Users SubGraph CRD. + +#### Deleting SubGraph + +- **SubGraph without any reference** + + If the subgraph is not referenced in any ApiDefinition CRD or SuperGraph CRD, it is easy to delete SubGraph CRDs as follows: + ```bash + kubectl delete subgraphs.tyk.tyk.io + ``` + +- **SubGraph referenced in ApiDefinition** + + If you have a subgraph which is referenced in any ApiDefinition, Tyk Operator will not delete the SubGraph. + + In order to delete this subgraph, the corresponding ApiDefinition CR must be updated, such that it has no reference to the + subgraph in `graph_ref` field. + +- **SubGraph referenced in SuperGraph** + + Although the subgraph is not referenced in any ApiDefinition, if it is referenced in the SuperGraph, Tyk Operator will + not delete the subgraph again. + + In order to delete this subgraph, SuperGraph CR should not have reference to corresponding subgraph in the `subgraph_ref`. + +#### Deleting SuperGraph + +- **SuperGraph without any reference** + + If the supergraph is not referenced in any ApiDefinition CRD, it can be deleted as follows: + + ```bash + kubectl delete supergraphs.tyk.tyk.io + ``` + +- **SuperGraph referenced in ApiDefinition** + + If a supergraph is referenced in any ApiDefinition, the Tyk Operator will not delete the SuperGraph CRD. + + In order to delete this supergraph, the ApiDefinition that has a reference to the supergraph must de-reference the supergraph + or be deleted. + +#### Users Subgraph + +```yaml +# Create Namespace & Service & Deployment for Users API +--- +apiVersion: v1 +kind: Namespace +metadata: + name: users-ns +--- +apiVersion: v1 +kind: Service +metadata: + name: users + namespace: users-ns + labels: + app: users +spec: + ports: + - name: http + port: 4201 + targetPort: 4201 + selector: + app: users +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: users + namespace: users-ns +spec: + replicas: 1 + selector: + matchLabels: + app: users + version: v1 + template: + metadata: + labels: + app: users + version: v1 + spec: + containers: + - image: zalbiraw/go-api-test-service:v2.0.0 + imagePullPolicy: Always + name: users + command: ["./services/graphql-subgraphs/users/server"] + env: + - name: PORT + value: "4201" +--- +apiVersion: tyk.tyk.io/v1alpha1 +kind: SubGraph +metadata: + name: users-subgraph + namespace: users-ns +spec: + schema: | + directive @extends on OBJECT | INTERFACE + + directive @external on FIELD_DEFINITION + + directive @key(fields: _FieldSet!) on OBJECT | INTERFACE + + directive @provides(fields: _FieldSet!) on FIELD_DEFINITION + + directive @requires(fields: _FieldSet!) on FIELD_DEFINITION + + scalar _Any + + union _Entity = User + + scalar _FieldSet + + type _Service { + sdl: String + } + + type Address { + street: String! + suite: String! + city: String! + zipcode: String! + geo: GeoLocation! + } + + type Company { + name: String! + catchPhrase: String! + bs: String! + } + + type Entity { + findUserByID(id: ID!): User! + } + + type GeoLocation { + lat: String! + lng: String! + } + + type Query { + user(id: ID!): User! + users: [User!]! + _entities(representations: [_Any!]!): [_Entity]! + _service: _Service! + } + + type User { + id: ID! + name: String! + username: String! + email: String! + address: Address! + phone: String! + website: String! + company: Company! + } + sdl: | + extend type Query { + user(id: ID!): User! + users: [User!]! + } + + type User @key(fields: "id") { + id: ID! + name: String! + username: String! + email: String! + address: Address! + phone: String! + website: String! + company: Company! + } + + type Address { + street: String! + suite: String! + city: String! + zipcode: String! + geo: GeoLocation! + } + + type GeoLocation { + lat: String! + lng: String! + } + + type Company { + name: String! + catchPhrase: String! + bs: String! + } +--- +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: federation-users-subgraph + namespace: users-ns +spec: + name: Federation - Users Subgraph + protocol: "http" + do_not_track: false + use_keyless: true + active: true + internal: true + graphql: + enabled: true + execution_mode: subgraph + graph_ref: users-subgraph + version: "2" + playground: + enabled: false + path: "" + proxy: + target_url: http://users.users-ns.svc:4201/query + listen_path: /users-subgraph/ + disable_strip_slash: true +``` + +#### Posts Subgraph + +```yaml +# Create Service & Deployment of Posts API +--- +apiVersion: v1 +kind: Service +metadata: + name: posts + labels: + app: posts +spec: + ports: + - name: http + port: 4202 + targetPort: 4202 + selector: + app: posts +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: posts +spec: + replicas: 1 + selector: + matchLabels: + app: posts + version: v1 + template: + metadata: + labels: + app: posts + version: v1 + spec: + containers: + - image: zalbiraw/go-api-test-service:v2.0.0 + imagePullPolicy: Always + name: posts + command: ["./services/graphql-subgraphs/posts/server"] + env: + - name: PORT + value: "4202" +--- +apiVersion: tyk.tyk.io/v1alpha1 +kind: SubGraph +metadata: + name: posts-subgraph +spec: + schema: | + directive @extends on OBJECT | INTERFACE + + directive @external on FIELD_DEFINITION + + directive @key(fields: _FieldSet!) on OBJECT | INTERFACE + + directive @provides(fields: _FieldSet!) on FIELD_DEFINITION + + directive @requires(fields: _FieldSet!) on FIELD_DEFINITION + + scalar _Any + + union _Entity = Post | User + + scalar _FieldSet + + type _Service { + sdl: String + } + + type Entity { + findPostByID(id: ID!): Post! + findUserByID(id: ID!): User! + } + + type Post { + id: ID! + userId: ID! + title: String! + body: String! + } + + type Query { + post(id: ID!): Post! + posts: [Post!]! + _entities(representations: [_Any!]!): [_Entity]! + _service: _Service! + } + + type User { + id: ID! + posts: [Post!]! + } + sdl: | + extend type Query { + post(id: ID!): Post! + posts: [Post!]! + } + + type Post @key(fields: "id") { + id: ID! + userId: ID! + title: String! + body: String! + } + + extend type User @key(fields: "id") { + id: ID! @external + posts: [Post!]! + } + +--- +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: federation-posts-subgraph +spec: + name: Federation - Posts Subgraph + protocol: "http" + do_not_track: false + use_keyless: true + active: true + internal: true + graphql: + enabled: true + execution_mode: subgraph + graph_ref: posts-subgraph + version: "2" + playground: + enabled: false + path: "" + proxy: + target_url: http://posts.default.svc:4202/query + listen_path: /posts-subgraph/ + disable_strip_slash: true +``` + +#### Supergraph + +```yaml +apiVersion: tyk.tyk.io/v1alpha1 +kind: SuperGraph +metadata: + name: social-media-supergraph +spec: + subgraph_refs: + - name: users-subgraph + namespace: users-ns + - name: posts-subgraph # Since namespace is not specified for posts-subgraph, Operator uses the namespace of this SuperGraph CRD which is default for our example. + schema: |- + type Query { + user(id: ID!): User! + users: [User!]! + post(id: ID!): Post! + posts: [Post!]! + } + + type User { + id: ID! + name: String! + username: String! + email: String! + address: Address! + phone: String! + website: String! + company: Company! + posts: [Post!]! + } + + type Address { + street: String! + suite: String! + city: String! + zipcode: String! + geo: GeoLocation! + } + + type GeoLocation { + lat: String! + lng: String! + } + + type Company { + name: String! + catchPhrase: String! + bs: String! + } + + type Post { + id: ID! + userId: ID! + title: String! + body: String! + } +--- +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: federation-supergraph +spec: + name: Federated - Social Media APIS + protocol: "http" + do_not_track: false + use_keyless: true + active: true + graphql: + enabled: true + execution_mode: supergraph + graph_ref: social-media-supergraph + version: "2" + playground: + enabled: true + path: /playground + proxy: + target_url: "" + strip_listen_path: true + listen_path: /social-media-apis-federated/ +``` expandable + +## Custom Plugins with Tyk Operator + +Using Tyk Classic APIs, developers can implement API-level custom plugins that can be optionally setup to execute for +each of the following [hooks](/api-management/plugins/plugin-types#plugin-and-hook-types) in the API request +lifecycle: [Pre (Request)](/api-management/plugins/plugin-types#request-plugins), [Authentication](/api-management/plugins/plugin-types#authentication-plugins), +[Post (Request)](/api-management/plugins/plugin-types#request-plugins), [Post +Authentication](/api-management/plugins/plugin-types#request-plugins), [Response](/api-management/plugins/plugin-types#response-plugins) +and [Analytics](/api-management/plugins/plugin-types#analytics-plugins). Subsequently, users can execute, or β€œhook”, their +plugin into these phases of the API request lifecycle based on their specific use case. + +This document explains how to configure the following plugin types with different drivers (plugin languages): + +- Pre (Request) +- Authentication +- Post-Auth (Request) +- Post (Request) +- Response + +Please refer to [Analytics Plugins](/api-management/plugins/plugin-types#analytics-plugins) to learn how to configure Analytics +plugins using Tyk Operator. + +### How It Works + +In Tyk Classic APIs, the _custom_middleware_ section of the Tyk Classic API Definition is where you configure plugins +that will run at different points during the lifecycle of an API request. + +The table below illustrates the Tyk Classic API configuration parameters that correspond to each phase of the API +request lifecycle: + +| Phase | Description | Config Value | +| :--------- | :------------------------------------------------------------------------- | :------------- | +| Pre | Occurs before main request processing. | pre | +| Auth | Custom authentication can be handled during this phase. | auth_check | +| Post Auth | Occurs after key authentication | post_key_auth | +| Post | Occurs after the main request processing but before the response is sent. | post | +| Response | Occurs after the main request processing but before the response is sent. | response | + +The example configuration below illustrates how to set up multiple plugins for different phases of the request +lifecycle: + +```json {linenos=true, linenostart=1} +{ + "custom_middleware": { + "pre": [ + { + "name": "PreHook1", + "path": "/path/to/plugin1.so", + "require_session": false, + "raw_body_only": false + } + ], + "auth_check": { + "name": "AuthCheck", + "path": "/path/to/plugin.so", + "require_session": false, + "raw_body_only": false + }, + "post_key_auth": [ + { + "name": "PostKeyAuth", + "path": "/path/to/plugin.so", + "require_session": false, + "raw_body_only": false + } + ], + "post": [ + { + "name": "PostHook1", + "path": "/path/to/plugin1.so", + "require_session": false, + "raw_body_only": false + }, + { + "name": "PostHook2", + "path": "/path/to/plugin2.so", + "require_session": false, + "raw_body_only": false + } + ], + "response": [ + { + "name": "ResponseHook", + "path": "/path/to/plugin.so", + "require_session": false, + "raw_body_only": false + } + ], + "driver": "goplugin" + } +} +``` expandable + +In the `custom_middleware` section of the API definition above we can see that there are Golang custom authentication +(`auth_check`), post authentication (`post_key_auth`), post, pre and response plugins configured. + +It can be seen that each plugin is configured with the specific function name and associated source file path of the +file that contains the function. Furthermore, each lifecycle phase can have a list of plugins configured, allowing for +complex processing workflows. For example, you might develop one plugin for logging and another for modifying the +request in the pre request phase. + +The `driver` configuration parameter describes the plugin implementation language. Please refer to the [supported +languages](/api-management/plugins/overview#plugin-driver-names) section for list of supported plugin driver names. + +Each plugin can have additional settings, such as: + +- `raw_body_only`: When true, indicates that only the raw body should be processed. +- `require_session`: When true, indicates that session metadata will be available to the plugin. This is applicable only + for post, post authentication and response plugins. + +### Unsupported + +##### Per-Endpoint Plugins + +At the endpoint-level, Tyk provides the facility to attach a custom Golang plugin at the end of the request processing +chain (immediately before the API-level post-plugin is executed). Please note that +[per-endpoint](/api-management/plugins/plugin-types#per-endpoint-custom-plugins) level plugins are not currently +supported by Tyk Operator. + +--- + +### Examples + +#### Configure Custom Plugins (JavaScript) With Tyk Operator + +In this example we will create a JavaScript plugin that will inject a request header _Hello_ with a value of _World_. +This will be configured as a pre request hook. + +1. **Implement Plugin** + + The first step is to create the plugin source code. + + ```javascript + var exampleJavaScriptMiddlewarePreHook = new TykJS.TykMiddleware.NewMiddleware({}); + + exampleJavaScriptMiddlewarePreHook.NewProcessRequest(function (request, session) { + // You can log to Tyk console output by calling the built-in log() function: + log("Hello from the Tyk JavaScript middleware pre hook function"); + + // Add a request header + request.SetHeaders["Hello"] = "World"; + + // You must return both the request and session metadata + return exampleJavaScriptMiddlewarePreHook.ReturnData(request, {}); + }); + ``` + + Copy the source code above and save it to the following file on the Gateway file system at + `/opt/tyk-gateway/middleware/example-javascript-middleware.js` + +2. **Create API Definition Resource** + + The example API Definition resource listed below listens on path _/httpbin_ and forwards requests to upstream + *http://httpbin.org*. + + ```yaml {linenos=table,hl_lines=["14-18"],linenostart=1} + apiVersion: tyk.tyk.io/v1alpha1 + kind: ApiDefinition + metadata: + name: httpbin + spec: + name: httpbin + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true + custom_middleware: + driver: otto # Javascript driver name + pre: + - name: "exampleJavaScriptMiddlewarePreHook" + path: "middleware/example-javascript-middleware.js" + ``` + + At lines 14-18 we can see the _custom_middleware_ section contains the configuration for our plugin: + + - The `driver` configuration parameter is set to `otto` at line 15, since our plugin is a Javascript plugin. For other + valid values please refer to the [plugins driver page](/api-management/plugins/overview#plugin-driver-names). + - A plugin hook configuration block is specified at line 16, containing the `name` and `path` for our plugin. The plugin + configuration block identifies the "hook" or phase in the API request lifecycle when Tyk Gateway will execute the + plugin. In the example above the configuration block is for a `pre` request plugin that will be executed before any + middleware. Valid values are the same as the [Tyk Classic API Definition](#how-it-works) equivalent, i.e. + `pre`, `auth_check`, `post`, `post-auth` and `response`. We can see that the following fields are set within the `pre` + plugin hook configuration object: + + - The `name` field represents the name of the function that implements the plugin in your source code. For Javascript + plugins this must match the name of the middleware object that was created. In the example above we created the + middleware object, `exampleJavaScriptMiddlewarePreHook`, by calling + `var exampleJavaScriptMiddlewarePreHook = new TykJS.TykMiddleware.NewMiddleware({});`. + - The `path` field contains the path to the source file `middleware/example-javascript-middleware.js`, relative to the + base installation folder, i.e `/opt/tyk-gateway`. + + Save the API Definition to file and create the APIDefinition resource: + + ```bash + $ kubectl apply -f path_to_your_apidefinition.yaml + apidefinition.tyk.tyk.io/httpbin created + ``` + +3. **Test Plugin** + + We can test that our plugin injects a _Hello_ header with a corresponding value of _World_ by using the curl command: + + ```bash + $ curl http://localhost:8080/httpbin/headers + { + "headers": { + "Accept": "*/*", + "Accept-Encoding": "gzip", + "Hello": "World", + "Host": "httpbin.org" + } + } + ``` + + The header `"Hello: World"` should be injected by the custom plugin. + + +#### Configure Custom Plugins (Python) using plugin bundles via Tyk Operator + +Tyk Operator also supports configuring custom plugins using plugin bundles, where the source code and associated +configuration is packaged into a zip file and uploaded to a remote webserver. Tyk Gateway will then download, extract, +cache and execute the plugin bundles for each of the configured phases of the [API +request lifecycle](/api-management/traffic-transformation#request-middleware-chain). + +For a detailed guide, check out our blog post +[How to Deploy Python Plugins in Tyk Running on Kubernetes](https://tyk.io/blog/how-to-deploy-python-plugins-in-tyk-running-on-kubernetes/), +which walks you through all the steps required to create Python [plugin +bundles](/api-management/plugins/overview#plugin-bundles), load them into the Tyk Gateway, and configure an API +Definition to use them with the Tyk Operator. + +## Multi-Organization Management With Tyk Operator + +If you want to set up multi-tenant API management with Tyk, follow these steps to define an OperatorContext for connecting and authenticating with a Tyk Dashboard and reference it in your API definitions for specific configurations. + +### Defining OperatorContext + +An [OperatorContext](#multi-tenancy-in-tyk) specifies the parameters for connecting and authenticating with a Tyk Dashboard. Below is an example of how to define an `OperatorContext`: + +```yaml +apiVersion: tyk.tyk.io/v1alpha1 +kind: OperatorContext +metadata: + name: team-alpha + namespace: default +spec: + env: + # The mode of the admin api + # ce - community edition (open source gateway) + # pro - dashboard (requires a license) + mode: pro + # Org ID to use + org: *YOUR_ORGANIZATION_ID* + # The authorization token this will be set in x-tyk-authorization header on the + # client while talking to the admin api + auth: *YOUR_API_ACCESS_KEY* + # The url to the Tyk Dashboard API + url: http://dashboard.tyk.svc.cluster.local:3000 + # Set this to true if you want to skip tls certificate and host name verification + # this should only be used in testing + insecureSkipVerify: true + # For ingress the operator creates and manages ApiDefinition resources, use this to configure + # which ports the ApiDefinition resources managed by the ingress controller binds to. + # Use this to override default ingress http and https port + ingress: + httpPort: 8000 + httpsPort: 8443 +``` + +For better security, you can also replace sensitive data with values contained within a referenced secret with `.spec.secretRef`. + +In this example, API access key `auth` and organization ID `org` are not specified in the manifest. They are provided through a Kubernetes secret named `tyk-operator-conf` in `alpha` namespace. The secret contains keys `TYK_AUTH` and `TYK_ORG` which correspond to the `auth` and `org` fields respectively. + +```yaml +apiVersion: tyk.tyk.io/v1alpha1 +kind: OperatorContext +metadata: + name: team-alpha + namespace: default +spec: + secretRef: + name: tyk-operator-conf## Secret containing keys TYK_AUTH and TYK_ORG + namespace: alpha + env: + mode: pro + url: http://tyk.tykce-control-plane.svc.cluster.local:8001 + insecureSkipVerify: true + ingress: + httpPort: 8000 + httpsPort: 8443 + user_owners: + - a1b2c3d4f5e6f7 + user_group_owners: + - 1a2b3c4d5f6e7f +``` expandable + +You can provide the following fields through secret as referenced by `secretRef`. The table shows mappings between `.spec.env` properties and secret `.spec.data` keys. If a value is configured in both the secret and OperatorContext `spec.env` field, the value from secret will take precedence. + +| Secret key | .spec.env | +| :------------ | :----------- | +| TYK_MODE | mode | +| TYK_URL | url | +| TYK_AUTH | auth | +| TYK_ORG | org | +| TYK_TLS_INSECURE_SKIP_VERIFY | insecureSkipVerify | +| TYK_USER_OWNERS (comma separated list) | user_owners | +| TYK_USER_GROUP_OWNERS (comma separated list) | user_group_owners | + +### Using contextRef in API Definitions + +Once an `OperatorContext` is defined, you can reference it in your API Definition objects using `contextRef`. Below is an example: + +```yaml +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin + namespace: alpha +spec: + contextRef: + name: team-alpha + namespace: default + name: httpbin + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true +``` expandable + +In this example, the `ApiDefinition` object references the `team-alpha` context, ensuring that the configuration is applied within the `alpha` organization. + +## Internal Looping With Tyk Operator + +The concept of [internal looping](/advanced-configuration/transform-traffic/looping) allows you to use URL Rewriting to redirect your URL to *another API endpoint* or to *another API* in the Gateway. In Tyk, looping is generally targeted using the `tyk:///` scheme, which requires prior knowledge of the `API_ID`. Tyk Operator simplifies the management and transformation of API traffic within Kubernetes environments by abstracting APIs as objects, managing them and dynamically assigning `API_ID`s by its Kubernetes metedata name and namespace. + +### Configuring looping to internal ApiDefinition resources + +Looping can be configured within Tyk Operator for [URL Rewrites](#url-rewrites), [URL Rewrite Triggers](#url-rewrite-triggers) and [Proxy to internal APIs](#proxy-to-internal-apis) by configuring the `rewrite_to_internal` in `url_rewrite`, `rewrite_to_internal` in `triggers`, and `proxy.target_internal` fields respectively with these properties: + +- **Path**: The `path` property specifies the endpoint on the target API where the request should be directed. This is the portion of the URL that follows the domain and is crucial for ensuring that the request reaches the correct resource. For example, setting a value of `"/myendpoint"` means that the request will be forwarded to the `/myendpoint` path on the target API. + +- **Query**: The `query` property allows you to append additional query parameters to the target URL. These parameters can be used to modify the behavior of the target API or to pass along specific request information. For instance, setting `query: "check_limits=true"` will include this query string in the redirected request, potentially triggering special handling by the target API. + +- **Target**: The `target` property identifies the API resource to which the request should be routed. It consists of two components: `name` and `namespace`. The `name` is the identifier of the target API, while the `namespace` specifies the Kubernetes namespace where the API resource resides. Together, these elements ensure that Tyk Operator accurately locates and routes the request to the intended API. For example, `name: "proxy-api"` and `namespace: "default"` direct the request to the `proxy-api` resource in the `default` namespace. + +Tyk Operator would dynamically update the API definition by generating internal looping URL in the form of `tyk:///`. This mechanism is essential for routing traffic within a microservices architecture or when managing APIs across different namespaces in Kubernetes. Using this object you can effectively manage and optimize API traffic within your Tyk Gateway. + +--- + +### URL Rewrites + +[URL rewriting](/transform-traffic/url-rewriting#url-rewrite-middleware) in Tyk enables the alteration of incoming API request paths to align with the expected endpoint format of your backend services. + +Assume that we wish to redirect incoming `GET /basic/` requests to an API defined by an ApiDefinition object named `proxy-api` in the `default` namespace. We want the `/basic/` prefix to be stripped from the request path and the redirected path should be of the format `/proxy/$1`, where the context variable `$1` is substituted with the remainder of the path request. For example `GET /basic/456` should become `GET /proxy/456`. + +In this case we can use a `rewrite_to_internal` object to instruct Tyk Operator to automatically generate the API rewrite URL on our behalf for the API identified by name `proxy-api` in the `default` namespace: + +```yaml +url_rewrites: + - path: "/{id}" + match_pattern: "/basic/(.*)" + method: GET + rewrite_to_internal: + target: + name: proxy-api + namespace: default + path: proxy/$1 +``` + +In the above example an incoming request of `/basic/456` would be matched by the `match_pattern` rule `/basic/(.*)` for `GET` requests specified in the `method` field. The `456` part of the URL will be captured and replaces `{id}` in the `path` field. Tyk Operator will use the `rewrite_to_internal` configuration to generate the URL rewrite for the API named `proxy-api` in the `default` namespace, and update the `rewrite_to` field accordingly: + +```yaml +url_rewrites: +- match_pattern: /basic/(.*) + method: GET + path: /{id} + rewrite_to: tyk://ZGVmYXVsdC9wcm94eS1hcGk/proxy/$1 +``` expandable + +Here we can see that the `rewrite_to` field has been generated with the value `tyk://ZGVmYXVsdC9wcm94eS1hcGk/proxy/$1` where `ZGVmYXVsdC9wcm94eS1hcGk` represents the API ID for the `proxy-api` API resource in the `default` namespace. Notice also that path `proxy/$1` is appended to the base URL `tyk://ZGVmYXVsdC9wcm94eS1hcGk` and contains the context variable `$1`. This will be substituted with the value of `{id}` in the `path` configuration parameter. + +### URL Rewrite Triggers + +[Triggers](/transform-traffic/url-rewriting#url-rewrite-triggers) are configurations that specify actions based on certain conditions present in HTTP headers, query parameters, path parameters etc. + +Triggers are essential for executing specific actions when particular criteria are met, such as rewriting URLs. They are useful for automating actions based on real-time data received in requests. For example, you might use triggers to: + +- Redirect users to different APIs in the Gateway based on their authentication status. +- Enforce business rules by redirecting requests to different APIs in the Gateway based on certain parameters. + +The process for configuring internal looping in triggers to is similar to that explained in section [URL Rewrites](#url-rewrites). + +Assume that we wish to instruct Tyk Operator to redirect all *Basic Authentication* requests to the API identified by `basic-auth-internal` within the `default` namespace. Subsequently, we can use a `rewrite_to_internal` object as follows: + +```yaml +triggers: + - "on": "all" + options: + header_matches: + "Authorization": + match_rx: "^Basic" + rewrite_to_internal: + target: + name: basic-auth-internal + namespace: default + path: "basic/$2" +``` + +Here we we can see that a trigger is configured for all requests that include an `Authorization` header containing `Basic` in the header value. + +A `rewrite_to_internal` configuration object is used to instruct Tyk Operator to generate a redirect to the API identified by the `basic-auth-internal` API resource in the `default` namespace. The redirect path will be prefixed with `basic`. For example, a basic authentication request to path `/` will be redirected to `/basic/`. + +Tyk Operator will automatically generate a URL Rewrite (`rewrite_to`) to redirect the request to the API identified by `basic-auth-internal` within the `default` namespace as follows: + +```yaml +triggers: +- "on": all + options: + header_matches: + Authorization: + match_rx: ^Basic + rewrite_to: tyk://ZGVmYXVsdC9iYXNpYy1hdXRoLWludGVybmFs/basic/$2 +``` + +Here we can see that the `rewrite_to` field has been generated with the value `tyk://ZGVmYXVsdC9iYXNpYy1hdXRoLWludGVybmFs/proxy/$1` where `ZGVmYXVsdC9iYXNpYy1hdXRoLWludGVybmFs` represents the API ID for the `proxy-api` API resource in the `default` namespace. Notice also that path `basic/$2` is appended to the base URL `tyk://ZGVmYXVsdC9iYXNpYy1hdXRoLWludGVybmFs` and contains the context variable `$2`. This will be substituted with the remainder of the request path. + +### Proxy to Internal APIs + +Internal looping can also be used for [proxying to internal APIs](/advanced-configuration/transform-traffic/looping). + +Assume that we wish to redirect all incoming requests on listen path `/users` to an API defined by an ApiDefinition object named `users-internal-api` in the `default` namespace. + +In this case we can use a `proxy.target_internal` field to instruct Tyk Operator to automatically generate the target URL on our behalf for the API identified by name `users-internal-api` in the `default` namespace: + +```yaml {linenos=true, linenostart=1, hl_lines=["12-15"]} +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: users +spec: + name: Users API + protocol: http + active: true + use_keyless: true + proxy: + target_url: "" + target_internal: + target: + name: users-internal-api + namespace: default + listen_path: /users + strip_listen_path: true +``` + +The proxy object’s `target_internal` field references other API resources. This field shares the same properties as those described for `rewrite_to_internal`, ensuring consistent configuration. + +Tyk Operator will automatically generate the target URL to redirect the request to the API identified by `users-internal-api` within the `default` namespace as follows: + +```yaml + target_url: "tyk://ZGVmYXVsdC91c2Vycy1pbnRlcm5hbC1hcGk" +``` expandable + +--- + +### Example + +Assume a business has legacy customers who authenticate with a service using *Basic Authentication*. The business also wants to support API Keys, enabling both client types to access the same ingress. + +To facilitate this, Tyk must be configured for dynamic authentication, accommodating both *Basic Authentication* and *Auth Token* methods. + +This setup requires configuring four API Definitions within Tyk: + +1. Entry Point API +2. BasicAuth Internal API +3. AuthToken Internal API +4. Proxy Internal API + +When a request arrives at the ingress route, a URL rewrite can direct it to either the *BasicAuth Internal* or *AuthToken Internal* API, depending on the authentication method used. + +These internal APIs will authenticate the requests. Assuming successful authentication (the happy path), they will forward the requests to the *Proxy Internal API*, which handles the proxying to the underlying service. + +
+ + + +There are no actual HTTP redirects in this scenario, meaning that there is no performance penalty in performing any of these *Internal Redirects*. + + + +#### Entry Point API + +The *Entry Point* API is the first point of entry for a client request. It inspects the header to determine if the incoming client request requires authentication using *Basic Authentication* or *Auth Token*. Consequently, it then redirects the request to the *BasicAuth Internal* or *AuthToken Internal* API depending upon the header included in the client request. + +The API definition resource for the *Entry Point* API is listed below. It is configured to listen for requests on the `/entry` path and forward requests upstream to `http://example.com` + +We can see that there is a URL Rewrite rule (`url_rewrites`) with two triggers configured to match Basic Authentication and Auth Token requests: + +- **Basic Authentication trigger**: Activated for incoming client requests that include an *Authorization* header containing a value starting with *Basic*. In this case a `rewrite_to_internal` configuration object is used to instruct Tyk Operator to redirect the request to the *BasicAuthInternal* API, identified by name `basic-auth-internal` in the `default` namespace. The request URL is rewritten, modifying the path to `/basic/`. +- **Auth Token trigger**: Activated for incoming client requests that include an *Authorization* header containing a value starting with *Bearer*. In this case a `rewrite_to_internal` configuration object is used to instruct Tyk Operator to redirect the request to the *AuthTokenInternal* API, identified by name `auth-token-internal` in the `default` namespace. The request URL is rewritten, modifying the path to `/token/`. + + ```yaml {linenos=true, linenostart=1, hl_lines=["21-45"]} +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: entrypoint-api +spec: + name: Entrypoint API + protocol: http + active: true + proxy: + listen_path: /entry + target_url: http://example.com + use_keyless: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + extended_paths: + url_rewrites: + - path: "/{id}" + match_pattern: "/(.*)/(.*)" + method: GET + triggers: + - "on": "all" + options: + header_matches: + "Authorization": + match_rx: "^Basic" + rewrite_to_internal: + target: + name: basic-auth-internal + namespace: default + path: "basic/$2" + - "on": "all" + options: + header_matches: + "Authorization": + match_rx: "^Bearer" + rewrite_to_internal: + target: + name: auth-token-internal + namespace: default + path: "token/$2" +``` + +#### BasicAuth Internal API + +The *BasicAuth Internal* API listens to requests on path `/basic` and forwards them upstream to `http://example.com`. + +The API is configured with a URL rewrite rule in `url_rewrites` to redirect incoming `GET /basic/` requests to the API in the Gateway represented by name `proxy-api` in the `default` namespace. The `/basic/` prefix will be stripped from the request URL and the URL will be rewritten with the format `/proxy/$1`. The context variable `$1` is substituted with the remainder of the path request. For example `GET /basic/456` will become `GET /proxy/456`. + +Furthermore, a header transform rule is configured within `transform_headers` to add the header `x-transform-api` with value `basic-auth`, to the request. + +```yaml {linenos=true, linenostart=1, hl_lines=["21-35"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: basic-auth-internal +spec: + name: BasicAuth Internal API + protocol: http + proxy: + listen_path: "/basic" + target_url: http://example.com + active: true + use_keyless: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + extended_paths: + url_rewrites: + - path: "/{id}" + match_pattern: "/basic/(.*)" + method: GET + rewrite_to_internal: + target: + name: proxy-api + namespace: default + path: proxy/$1 + transform_headers: + - add_headers: + x-transform-api: "basic-auth" + method: GET + path: "/{id}" + delete_headers: [] +``` + +#### AuthToken Internal API + +The *AuthToken Internal* API listens to requests on path `/token` and forwards them upstream to `http://example.com`. + +The API is configured with a URL rewrite rule in `url_rewrites` to redirect incoming `GET /token/` requests to the API in the Gateway represented by name `proxy-api` in the `default` namespace. The `/token/` prefix will be stripped from the request URL and the URL will be rewritten to the format `/proxy/$1`. The context variable `$1` is substituted with the remainder of the path request. For example `GET /token/456` will become `GET /proxy/456`. + +Furthermore, a header transform rule is configured within `transform_headers` to add the header `x-transform-api` with value `token-auth`, to the request. + +```yaml {linenos=true, linenostart=1, hl_lines=["21-35"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: auth-token-internal +spec: + name: AuthToken Internal API + protocol: http + proxy: + listen_path: "/token" + target_url: http://example.com + active: true + use_keyless: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + extended_paths: + url_rewrites: + - path: "/{id}" + match_pattern: "/token/(.*)" + method: GET + rewrite_to_internal: + target: + name: proxy-api + namespace: default + path: proxy/$1 + transform_headers: + - add_headers: + x-transform-api: "token-auth" + method: GET + path: "/{id}" + delete_headers: [] +``` + +#### Proxy Internal API + +The *Proxy Internal* API is keyless and responsible for listening to requests on path `/proxy` and forwarding upstream to `http://httpbin.org`. The listen path is stripped from the request before it is sent upstream. + +This API receives requests forwarded from the internal *AuthToken Internal* and *BasicAuth Internal APIs*. Requests will contain the header `x-transform-api` with value `token-auth` or `basic-auth`, depending upon which internal API the request originated from. + +```yaml {linenos=true, linenostart=1, hl_lines=["10-13"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: proxy-api +spec: + name: Proxy API + protocol: http + active: true + internal: true + proxy: + listen_path: "/proxy" + target_url: http://httpbin.org + strip_listen_path: true + use_keyless: true +``` + +## Manage API MetaData + + +### API Name + +#### Tyk OAS API and Tyk Streams API + +API name can be set through `x-tyk-api-gateway.info.name` field in [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas) object. + +#### Tyk Classic API + +To set the name of an API in the `ApiDefinition`, use the `spec.name` string field. This name is displayed on the Tyk Dashboard and should concisely describe what the API represents. + +Example: + +```yaml {linenos=true, linenostart=1, hl_lines=["6-6"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: example-api # This is the metadata name of the Kubernetes resource +spec: + name: Example API # This is the "API NAME" in Tyk + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://example.com + listen_path: /example + strip_listen_path: true +``` + +### API Status + +#### API Active Status + +An active API will be loaded to the Gateway, while an inactive API will not, resulting in a 404 response when called. + +#### Tyk OAS API and Tyk Streams API + +API active state can be set through `x-tyk-api-gateway.info.state.active` field in [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas) object. + +#### Tyk Classic API + +The active status of an API can be set by modifying the `spec.active` configuration parameter. When set to `true`, this enables the API so that Tyk will listen for and process requests made to the `listenPath`. + +```yaml {linenos=true, linenostart=1, hl_lines=["9-9"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: inactive-api +spec: + name: Inactive API + use_keyless: true + protocol: http + active: false + proxy: + target_url: http://inactive.example.com + listen_path: /inactive + strip_listen_path: true +``` + +### API Accessibility + +An API can be configured as internal so that external requests are not processed. + +#### Tyk OAS API and Tyk Streams API + +API accessibility can be set through `x-tyk-api-gateway.info.state.internal` field in [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas) object. + +#### Tyk Classic API + +API accessibility can be set through the `spec.internal` configuration parameter as shown in the example below. + +```yaml {linenos=true, linenostart=1, hl_lines=["10-10"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: inactive-api +spec: + name: Inactive API + use_keyless: true + protocol: http + active: true + internal: true + proxy: + target_url: http://inactive.example.com + listen_path: /inactive + strip_listen_path: true +``` + +### API ID + +#### Creating a new API + +If you're creating a new API using Tyk Operator, you don't need to specify the ID. The API ID will be generated in a deterministic way. + +#### Tyk OAS API and Tyk Streams API + +The generated ID is stored in `status.id` field. Run the following command to inspect generated API ID of a Tyk OAS API. + +```bash +% kubectl get tykoasapidefinition [API_NAME] --namespace [NAMESPACE] -o jsonpath='{.status.id}' +ZGVmYXVsdC9wZXRzdG9yZQ +``` + +In this example, the generated API ID is `ZGVmYXVsdC9wZXRzdG9yZQ`. + +#### Tyk Classic API + +The generated ID is stored in `status.api_id` field. Run the following command to inspect generated API ID of a Tyk Classic API. + +```bash +% kubectl get apidefinition [API_NAME] --namespace [NAMESPACE] -o jsonpath='{.status.api_id}' +ZGVmYXVsdC90ZXN0 +``` + +In this example, the generated API ID is `ZGVmYXVsdC90ZXN0`. + +### Updating an existing API + +#### Tyk OAS API and Tyk Streams API + +If you already have API configurations created in the Tyk Dashboard and want to start using Tyk Operator to manage these APIs, you can include the existing API ID in the manifest under the `x-tyk-api-gateway.info.id` field in [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas) object. + +#### Tyk Classic API + +If you already have API configurations created in the Tyk Dashboard and want to start using Tyk Operator to manage these APIs, you can include the existing API ID in the manifest under the `spec.api_id` field. This way, when you apply the manifest, Tyk Operator will not create a new API in the Dashboard. Instead, it will update the original API with the Kubernetes spec. + +Example + +```yaml {linenos=true, linenostart=1, hl_lines=["8-8"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: existing-api + namespace: default +spec: + name: Existing API + api_id: 12345 + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://existing.example.com + listen_path: /existing + strip_listen_path: true +``` + +In this example, the API with ID `12345` will be updated according to the provided spec instead of creating a new API. + + +### API Categories +[API categories](/api-management/dashboard-configuration#governance-using-api-categories) are configured differently for Tyk OAS APIs and Tyk Classic APIs. Please see below for examples. + +#### Tyk OAS API + +API categories can be specified through `categories` field in `TykOasApiDefinition` CRD. + +Here's an example: + +```yaml {linenos=true, linenostart=1, hl_lines=["7-9"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: TykOasApiDefinition +metadata: + name: oas-api-with-categories + namespace: tyk +spec: + categories: + - category 1 + - category 2 + tykOAS: + configmapRef: + keyName: oas-api-definition.json + name: tyk-oas-api-config + namespace: tyk +``` + +#### Tyk Streams API + +As of Tyk Operator v1.1, API categories is not supported in `TykStreamsApiDefinition` CRD. + +#### Tyk Classic API + +For a Tyk Classic API, you can specify the category name using the `name` field with a `#` qualifier. This will categorize the API in the Tyk Dashboard. See [How API categories work](/api-management/dashboard-configuration#tyk-classic-apis) to learn about limitations on API names. + +Example + +```yaml {linenos=true, linenostart=1, hl_lines=["6-6"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: categorized-api +spec: + name: "my-classic-api #global #staging" + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://categorized.example.com + listen_path: /categorized + strip_listen_path: true +``` + +### API Versioning +[API versioning](/api-management/api-versioning) are configured differently for [Tyk OAS APIs](#tyk-oas-api) and [Tyk Classic APIs](#tyk-classic-api). Please see below for examples. + +#### Configuring API Version in Tyk OAS API Definition + +In the [Tyk OAS API Definition](/api-management/api-versioning), versioning can be configured via `x-tyk-api-gateway.versioning` object of the Base API, where the child API's IDs are specified. In the Kubernetes environment with Tyk Operator, where we reference API resources through its Kubernetes name and namespace, this is not desired. Therefore, we add support for versioning configurations through the field `versioning` in `TykOasApiDefinition` custom resource definition (CRD). + +Here's an example: + +```yaml{linenos=true, linenostart=1, hl_lines=["12-24"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: TykOasApiDefinition +metadata: + name: order-api + namespace: default +spec: + tykOAS: + configmapRef: + namespace: default + name: order-api + keyName: order-api-definition-v1.json + versioning: + enabled: true + location: header + key: x-api-version + name: v1 + default: v1 + fallbackToDefault: true + stripVersioningData: true + versions: + - name: v2 + tykOasApiDefinitionRef: + name: order-api-v2 + namespace: default +--- +apiVersion: tyk.tyk.io/v1alpha1 +kind: TykOasApiDefinition +metadata: + name: order-api-v2 + namespace: default +spec: + tykOAS: + configmapRef: + namespace: default + name: order-api-v2 + keyName: order-api-definition-v2.json +``` + +In this example, two different versions of an API are defined: `order-api` (v1) and `order-api-v2` (v2). + +`versioning` is configured at `order-api` (v1), the Base API, and it has similiar structure as [Tyk OAS API Definition](/api-management/api-versioning): + +- `versioning`: This object configures API versioning for the `order-api`. + - `enabled`: Set to true to enable versioning. + - `name`: an identifier for this version of the API (v1). + - `default`: Specifies the default version (v1), which will be used if no version is specified in the request. + - `location`: Specifies where the version key is expected (in this case, in the header). It can be set to `header` or `url-param`. + - `key`: Specifies the versioning identifier key (`x-api-version`) to identify the version. In this example, the version is determined by an HTTP header named `x-api-version`. + - `fallbackToDefault`: When set to true, if an unspecified or invalid version is requested, the default version (v1) will be used. + - `stripVersioningData`: When true, removes versioning identifier (like headers or query parameters) from the upstream request to avoid exposing internal versioning details. + - `urlVersioningPattern`: Specifies a regex that matches the format that you use for the versioning identifier (name) if you are using stripVersioningData and fallBackToDefault with location=url with Tyk 5.5.0 or later + - `versions`: Defines the list of API versions available: + - `name`: an identifier for this version of the API (v2). + - `tykOasApiDefinitionRef`: Refers to a separate TykOasApiDefinition resource that represent a new API version. + - `name`: Kubernetes metadata name of the resource (`order-api-v2`). + - `namespace`: Kubernetes metadata namespace of the resource (`default`). + +With Tyk Operator, you can easily associate different versions of your APIs using their Kubernetes names. This eliminates the need to include versioning information directly within the base API's definition (`x-tyk-api-gateway.versioning` object), which typically requires referencing specific API IDs. Instead, the Operator allows you to manage versioning declaratively in the `TykOasApiDefinition` CRD, using the `versioning` field to specify versions and their Kubernetes references (names and namespaces). + +When using the CRD for versioning configuration, you don't have to worry about knowing or managing the unique API IDs within Tyk. The Tyk Operator handles the actual API definition configuration behind the scenes, reducing the complexity of version management. + +In case if there is original versioning information in the base API Definition, the versioning information will be kept and be merged with what is specified in CRD. If there are conflicts between the Tyk OAS API Definition and CRD, we will make use of CRD values as the final configuration. + +Tyk Operator would also protect you from accidentally deleting a version of an API that is being referenced by another API, maintaining your API integrity. + +#### Configuring API Version in Tyk Streams API Definition + +As of Tyk Operator v1.1, API versioning is not supported in `TykStreamsApiDefinition` CRD. This can be configured natively in the Tyk Streams API Definition. + +#### Configuring API Version in Tyk Classic API Definition + +For Tyk Classic API, versioning can be configured via `ApiDefinition` custom resource definition (CRD). See [Tyk Classic versioning](/api-management/gateway-config-tyk-classic#tyk-classic-api-versioning) for a comprehensive example of configuring API versioning for Tyk Classic API with Tyk Operator. + +### API Ownership + +Please consult the [API Ownership](/api-management/user-management#api-ownership) documentation for the fundamental concepts of API Ownership in Tyk and [Operator Context](#multi-tenancy-in-tyk) documentation for an overview of the use of OperatorContext to manage resources for different teams effectively. + +The guide includes practical examples for managing API ownership via OperatorContext. Key topics include defining user owners and user group owners in OperatorContext for connecting and authenticating with a Tyk Dashboard, and using `contextRef` in `TykOasApiDefinition` or `ApiDefinition` objects to ensure configurations are applied within specific organizations. The provided YAML examples illustrate how to set up these configurations. + +#### How API Ownership works in Tyk Operator + +In Tyk Dashboard, API Ownership ensures that only designated 'users' who own an API can modify it. This security model is crucial for maintaining control over API configurations, especially in a multi-tenant environment where multiple teams or departments may have different responsibilities and permissions. + +Tyk Operator is designed to interact with Tyk Dashboard as a system user. For the Tyk Dashboard, Tyk Operator is just another user that must adhere to the same access controls and permissions as any other user. This means: + +- Tyk Operator needs the correct access rights to modify any APIs. +- It must be capable of managing APIs according to the ownership rules set in Tyk Dashboard. + +To facilitate API ownership and ensure secure operations, Tyk Operator must be able to 'impersonate' different users for API operations. This is where `OperatorContext` comes into play. Users can define different `OperatorContext` objects that act as different agents to connect to Tyk Dashboard. Each `OperatorContext` can specify different access parameters, including the user access key and organization it belongs to. Within `OperatorContext`, users can specify the IDs of owner users or owner user groups. All APIs managed through that `OperatorContext` will be owned by the specified users and user groups, ensuring compliance with Tyk Dashboard's API ownership model. + +Enabling API ownership with OperatorContext + +#### OperatorContext + +Here's how `OperatorContext` allows Tyk Operator to manage APIs under different ownerships: + +```yaml expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: OperatorContext +metadata: + name: team-alpha + namespace: default +spec: + env: + # The mode of the admin api + # ce - community edition (open source gateway) + # pro - dashboard (requires a license) + mode: pro + # Org ID to use + org: *YOUR_ORGANIZATION_ID* + # The authorization token this will be set in x-tyk-authorization header on the + # client while talking to the admin api + auth: *YOUR_API_ACCESS_KEY* + # The url to the Tyk Dashboard API + url: http://dashboard.tyk.svc.cluster.local:3000 + # Set this to true if you want to skip tls certificate and host name verification + # this should only be used in testing + insecureSkipVerify: true + # For ingress the operator creates and manages ApiDefinition resources, use this to configure + # which ports the ApiDefinition resources managed by the ingress controller binds to. + # Use this to override default ingress http and https port + ingress: + httpPort: 8000 + httpsPort: 8443 + # Optional - The list of users who are authorized to update/delete the API. + # The user pointed by auth needs to be in this list, if not empty. + user_owners: + - a1b2c3d4e5f6 + # Optional - The list of groups of users who are authorized to update/delete the API. + # The user pointed by auth needs to be a member of one of the groups in this list, if not empty. + user_group_owners: + - 1a2b3c4d5e6f +``` + +#### Tyk OAS API and Tyk Streams API + +Once an `OperatorContext` is defined, you can reference it in your Tyk OAS or Tyk Streams API Definition objects using `contextRef`. Below is an example with TykOasApiDefinition: + +```yaml {hl_lines=["40-43"],linenos=true} expandable +apiVersion: v1 +data: + test_oas.json: |- + { + "info": { + "title": "Petstore", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "components": {}, + "paths": {}, + "x-tyk-api-gateway": { + "info": { + "name": "Petstore", + "state": { + "active": true + } + }, + "upstream": { + "url": "https://petstore.swagger.io/v2" + }, + "server": { + "listenPath": { + "value": "/petstore/", + "strip": true + } + } + } + } +kind: ConfigMap +metadata: + name: cm + namespace: default +--- +apiVersion: tyk.tyk.io/v1alpha1 +kind: TykOasApiDefinition +metadata: + name: petstore +spec: + contextRef: + name: team-alpha + namespace: default + tykOAS: + configmapRef: + name: cm + namespace: default + keyName: test_oas.json +``` + +In this example, the `TykOasApiDefinition` object references the `team-alpha` context, ensuring that it is managed under the ownership of the specified users and user groups. + +#### Tyk Classic API + +Similarly, if you are using Tyk Classic API, you can reference it in your API Definition objects using `contextRef`. Below is an example: + +```yaml expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin + namespace: alpha +spec: + contextRef: + name: team-alpha + namespace: default + name: httpbin + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true +``` + +In this example, the `ApiDefinition` object references the `team-alpha` context, ensuring that it is managed under the ownership of the specified users and user groups. + +## Troubleshooting and FAQ + +### Can I use Tyk Operator with non-Kubernetes Tyk installations? + +While Tyk Operator is designed to work within a Kubernetes environment, you can still use it to manage non-Kubernetes Tyk installations. You'll need to: + +1. Run Tyk Operator in a Kubernetes cluster. +2. Configure Tyk Operator to point to your external Tyk installation, e.g. via `tyk-operator-conf`, environment variable, or OperatorContext: +```yaml + TYK_MODE: pro + TYK_URL: http://external-tyk-dashboard + TYK_AUTH: api-access-key + TYK_ORG: org-id +``` + +This allows you to manage your external Tyk installation using Kubernetes resources. + + +### Tyk Operator changes not applied + +From [Tyk Operator v0.15.0](https://github.com/TykTechnologies/tyk-operator/releases/tag/v0.15.0), we introduce a new status [subresource](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#subresources) in APIDefinition CRD, called _latestTransaction_ which holds information about reconciliation status. + +> The [Status subresource](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#status-subresource) in Kubernetes is a specialized endpoint that allows developers and operators to retrieve the real-time status of a specific Kubernetes resource. By querying this subresource, users can efficiently access essential information about a resource's current state, conditions, and other relevant details without fetching the entire resource, simplifying monitoring and aiding in prompt decision-making and issue resolution. + +The new status subresource _latestTransaction_ consists of a couple of fields that show the latest result of the reconciliation: +- `.status.latestTransaction.status`: shows the status of the latest reconciliation, either Successful or Failed; +- `.status.latestTransaction.time`: shows the time of the latest reconciliation; +- `.status.latestTransaction.error`: shows the message of an error if observed in the latest transaction. + +#### Example: Find out why an APIDefinition resource cannot be deleted +Consider the scenario when APIDefinition and SecurityPolicy are connected. Usually, APIDefinition cannot be deleted directly since it is protected by SecurityPolicy. The proper approach to remove an APIDefinition is to first remove the reference to the SecurityPolicy (either by deleting the SecurityPolicy CR or updating SecurityPolicy CR’s specification), and then remove the APIDefinition itself. However, if we directly delete this APIDefinition, Tyk Operator won’t delete the APIDefinition unless the link between SecurityPolicy and APIDefinition is removed. It is to protect the referential integrity between your resources. + +```console +$ kubectl delete tykapis httpbin +apidefinition.tyk.tyk.io "httpbin" deleted +^C% +``` + +After deleting APIDefinition, the operation hangs, and we suspect that something is wrong. +Users might still look through the logs to comprehend the issue, as they did in the past, but they can now examine their APIDefinition’s status subresource to make their initial, speedy issue diagnosis. + +```console +$ kubectl get tykapis httpbin +NAME DOMAIN LISTENPATH PROXY.TARGETURL ENABLED STATUS +httpbin /httpbin http://httpbin.org true Failed +``` +As seen in the STATUS column, something went wrong, and the STATUS is Failed. + +To get more information about the APIDefinition resource, we can use `kubectl describe` or `kubectl get`: +```console expandable +$ kubectl describe tykapis httpbin +Name: httpbin +Namespace: default +API Version: tyk.tyk.io/v1alpha1 +Kind: ApiDefinition +Metadata: + ... +Spec: + ... +Status: + api_id: ZGVmYXVsdC9odHRwYmlu + Latest CRD Spec Hash: 9169537376206027578 + Latest Transaction: + Error: unable to delete api due to security policy dependency=default/httpbin + Status: Failed + Time: 2023-07-18T07:26:45Z + Latest Tyk Spec Hash: 14558493065514264307 + linked_by_policies: + Name: httpbin + Namespace: default +``` +or +```console expandable +$ kubectl get tykapis httpbin -o json | jq .status.latestTransaction +{ + "error": "unable to delete api due to security policy dependency=default/httpbin", + "status": "Failed", + "time": "2023-07-18T07:26:45Z" +} +``` +Instead of digging into Tyk Operator's logs, we can now diagnose this issue simply by looking at the `.status.latestTransaction` field. As `.status.latestTransaction.error` implies, the error is related to *SecurityPolicy* dependency. + + + + +### Can I use Tyk Operator with multiple Tyk installations? + +Yes, you can use Tyk Operator to manage multiple Tyk installations. You'll need to create separate `OperatorContext` resources for each installation: + +```yaml expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: OperatorContext +metadata: + name: prod-context +spec: + env: + TYK_MODE: pro + TYK_URL: http://tyk-dashboard-staging + TYK_AUTH: prod-secret +--- +apiVersion: tyk.tyk.io/v1alpha1 +kind: OperatorContext +metadata: + name: staging-context +spec: + env: + TYK_MODE: pro + TYK_URL: http://tyk-dashboard-staging + TYK_AUTH: staging-secret +``` + +Then, you can specify which context to use in your API and Policy resources: + +```yaml expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: my-api +spec: + name: My API + context: prod-context + # ... other API configuration +``` + +### What Features Are Supported By Tyk Operator? + +#### APIDefinition CRD +Tyk stores API configurations as JSON objects called API Definitions. If you are using Tyk Dashboard to manage Tyk, then these are stored in either Postgres or MongoDB, as specified in the database settings. On the other hand, if you are using Tyk OSS, these configurations are stored as files in the /apps directory of the Gateway which is located at the default path /opt/tyk-gateway. + +An API definition includes various settings and middleware that control how incoming requests are processed. + +##### API Types +Tyk supports various API types, including HTTP, HTTPS, TCP, TLS, and GraphQL. It also includes Universal Data Graph versions for unified data access and federation, allowing seamless querying across multiple services. + +| Type | Support | Supported From | Comments | +| :-------------------------------- | :--------- | :---------------- | :------------------------------ | +| HTTP | βœ… | v0.1 | Standard HTTP proxy for API requests. | +| HTTPS | βœ… | v0.4 | Secure HTTP proxy using SSL/TLS encryption. | +| TCP | βœ… | v0.1 | Handles raw TCP traffic, useful for non-HTTP APIs. | +| TLS | βœ… | v0.1 | Handles encrypted TLS traffic for secure communication. | +| GraphQL - Proxy | βœ… | v0.1 | Proxy for GraphQL APIs, routing queries to the appropriate service. | +| Universal Data Graph v1 | βœ… | v0.1 | Supports Universal Data Graph v1 for unified data access. | +| Universal Data Graph v2 | βœ… | v0.12 | Supports the newer Universal Data Graph v2 for more advanced data handling. | +| GraphQL - Federation | βœ… | v0.12 | Supports GraphQL Federation for querying multiple services as one API. | + +##### Management of APIs +Tyk offers flexible API management features such as setting active/inactive status, categorizing and naming APIs, versioning, and defining ownership within teams or organizations for streamlined administration. + +| Type | Support | Supported From | Comments | +| :-------------------------------- | :--------- | :---------------- | :------------------------------ | +| API Name | βœ… | v0.1 | Assign and manage names for your APIs. | +| API Status (inactive/active) | βœ… | v0.2 | Toggle API status between active and inactive. | +| API Categories | βœ… | v0.1 | Categorize APIs for easier management. | +| API ID | βœ… | v0.1 | Assign unique IDs to APIs for tracking and management. | +| API Ownership | βœ… | v0.12 | Define ownership of APIs within teams or organizations. | +| API Versioning | βœ… | v0.1 | Enable version control for APIs. | + +##### Traffic Routing +Tyk enables traffic routing through path-based or host-based proxies and allows redirection to specific target URLs, providing control over how requests are directed to backend services. + +| Type | Supported | Supported From | Comments | +| :--------------------------- | :--------- | :-------------- | :---------------------------- | +| Path-Based Proxy | βœ… | v0.1 | Route traffic based on URL path. | +| Host-Based Proxy | βœ… | v0.1 | Route traffic based on the request host. | +| Target URL | βœ… | v0.1 | Redirect traffic to a specific target URL. | + +##### Client to Gateway Authentication and Authorization +Tyk provides multiple authentication options for client-to-gateway interactions, including keyless access, JWT, client mTLS, IP allow/block lists, and custom authentication plugins for enhanced security. + +| Type | Supported | Supported From | Comments | +| :----------------------------- | :--------- | :-------------- | :----------------------------------------------- | +| Keyless | βœ… | v0.1 | No authentication required, open access. | +| Auth Token | βœ… | v0.1 | Requires an authentication token (Bearer token).| +| JWT | βœ…οΈ | v0.5 | Uses JSON Web Tokens for secure authentication. | +| OpenID Connect | ❌ | - | Recommended to use JWT for OIDC authentication. | +| OAuth2 | ❌ | - | OAuth2 not supported, JWT is recommended. | +| Client mTLS | βœ… | v0.11 | Supports static client mutual TLS authentication. | +| HMAC | ❌ | - | HMAC authentication is not implemented. | +| Basic Authentication | βœ… | v0.12 | Only supports enabling with default metadata. | +| Custom Authentication Plugin (Go) | βœ… | v0.11 | Custom authentication plugin written in Go. | +| Custom Authentication Plugin (gRPC) | βœ… | v0.1 | Custom authentication plugin using gRPC. | +| Multiple Authentication | βœ… | v0.14 | Chain multiple authentication methods. | +| IP Allowlist | βœ… | v0.5 | Allows access only from specific IP addresses. | +| IP Blocklist | βœ… | v0.5 | Blocks access from specific IP addresses. | + +##### Gateway to Upstream Authentication +Tyk supports secure upstream connections through mutual TLS, certificate pinning, and public key verification to ensure data integrity between the gateway and backend services. For full details, please see the [Upstream Authentication](/api-management/upstream-authentication) section. + +| Type | Supported | Supported From | +| :------------------------------------------------- | :----------- | :---------------- | +| Mutual TLS for upstream connectioons | βœ… | v0.9 | Mutual TLS authentication for upstream connections. | +| Public Key Certificate Pinning | βœ… | v0.9 | Ensures that the upstream certificate matches a known key. | +| Upstream Request Signing using HMAC | βœ… | v1.2.0 | Attach an encrypted signature to requests to verify the gateway as the sender. | + +##### API-level (Global) Features +Tyk offers global features for APIs, such as detailed traffic logging, CORS management, rate limiting, header transformations, and analytics plugins, with support for tagging, load balancing, and dynamic variables. + +| Feature | Supported | Supported From | Comments | +| :-------------------------------------- | :----------- | :---------------- | :------------------------------------------------------------------------ | +| Detailed recording (in Log Browser) | βœ… | v0.4.0 | Records detailed API traffic logs for analysis. | +| Config Data | βœ… | v0.8.2 | Stores additional configuration data for APIs. | +| Context Variables | βœ… | v0.1 | Enables dynamic context-based variables in APIs. | +| Cross Origin Resource Sharing (CORS) | βœ… | v0.2 | Manages CORS settings for cross-domain requests. | +| Service Discovery | ⚠️ | - | Service discovery is untested in this version. | +| Segment Tags | βœ… | v0.1 | Tags APIs for segmentation across environments. | +| Internal API (not exposed by Gateway)| βœ… | v0.6.0 | Internal APIs are not exposed via the Gateway. | +| Global (API-level) Header Transform | βœ… | v0.1.0 | Transforms request and response headers at the API level. | +| Global (API-level) Rate Limit | βœ… | v0.10 | Sets rate limits globally for APIs. | +| Custom Plugins | βœ… | v0.1 | Supports the use of custom plugins for API processing. | +| Analytics Plugin | βœ… | v0.16.0 | Integrates analytics plugins for API monitoring. | +| Batch Requests | ❌ | - | Batch requests are not supported. | +| Custom Analytics Tags (Tag Headers) | βœ… | v0.10.0 | Custom tags for API analytics data. | +| Expire Analytics After | ❌ | - | Not supported in this version. | +| Do not track Analytics (per API) | βœ… | v0.1.0 | Disable analytics tracking on specific APIs. | +| Webhooks | ❌ | - | Webhook support is not available. | +| Looping | βœ… | v0.6 | Enables internal looping of API requests. | +| Round Robin Load Balancing | βœ… | - | Supports round-robin load balancing across upstream servers. | + +##### Endpoint-level Features +For specific API endpoints, Tyk includes features like caching, circuit breaking, request validation, URL rewriting, and response transformations, allowing for precise control over request processing and response handling at an endpoint level. + +| Endpoint Middleware | Supported | Supported From | Comments | +| :----------------------------------- | :----------- | :---------------- | :------------------------------------------------ | +| Allow list | βœ…οΈ | v0.8.2 | Allows requests only from approved sources. | +| Block list | βœ…οΈ | v0.8.2 | Blocks requests from disapproved sources. | +| Cache | βœ… | v0.1 | Caches responses to reduce latency. | +| Advance Cache | βœ… | v0.1 | Provides advanced caching capabilities. | +| Circuit Breaker | βœ… | v0.5 | Prevents service overload by breaking circuits. | +| Track Endpoint | βœ… | v0.1 | Tracks API endpoint usage for analysis. | +| Do Not Track Endpoint | βœ… | v0.1 | Disables tracking for specific endpoints. | +| Enforced Timeouts | βœ… | v0.1 | Ensures timeouts for long-running requests. | +| Ignore Authentication | βœ… | v0.8.2 | Bypasses authentication for selected endpoints.| +| Internal Endpoint | βœ… | v0.1 | Restricts access to internal services. | +| URL Rewrite | βœ…οΈ | v0.1 | Modifies request URLs before processing. | +| Validate Request | βœ… | v0.8.2 | Validates incoming requests before forwarding. | +| Rate Limit | ❌ | - | Rate limiting is not supported per endpoint. | +| Request Size Limit | βœ…οΈ | v0.1 | Limits the size of requests to prevent overload.| +| Request Method Transform | βœ… | v0.5 | Modifies HTTP methods for incoming requests. | +| Request Header Transform | βœ… | v0.1 | Transforms request headers. | +| Request Body Transform | βœ… | v0.1 | Transforms request bodies for processing. | +| Request Body JQ Transform | ⚠️ | v0.1 | Requires JQ support on the Gateway Docker image.| +| Response Header Transform | βœ… | v0.1 | Transforms response headers. | +| Response Body Transform | βœ… | v0.1 | Transforms response bodies. | +| Response Body JQ Transform | ⚠️ | v0.1 | Requires JQ support on the Gateway Docker image.| +| Mock Response | βœ… | v0.1 | Simulates API responses for testing. | +| Virtual Endpoint | βœ… | v0.1 | Allows creation of dynamic virtual endpoints. | +| Per-Endpoint Plugin | ❌ | - | Plugin support per endpoint is not available. | +| Persist Graphql | ❌ | - | Not supported in this version. | + + +#### TykOasAPIDefinition CRD +The TykOasApiDefinition Custom Resource Definition (CRD) manages [Tyk OAS API Definition objects](/api-management/gateway-config-tyk-oas) within a Kubernetes environment. This CRD enables the integration and management of Tyk API definitions using Kubernetes-native tools, simplifying the process of deploying and managing OAS APIs on the Tyk Dashboard. + +##### TykOasApiDefinition Features + +`TykOasApiDefinition` can support all features of the Tyk OAS API definition. You just need to provide the Tyk OAS API definition via a ConfigMap. In addition to managing the CRUD (Create, Read, Update, Delete) of Tyk OAS API resources, the Tyk Operator helps you better manage resources through object linking to Ingress, Security Policies, and certificates stored as Kubernetes secrets. See below for a list of Operator features and examples: + +| Features | Support | Supported From | Comments | Example | +| :---------- | :--------- | :----------------- | :---------- | :-------- | +| API Category | βœ… | v1.0 | - | [Manage API Categories](#api-categories) | +| API Version | βœ… | v1.0 | - | [Manage API versioning](#api-versioning) | +| API Ownership via OperatorContext | βœ… | v1.0 | - | [API Ownership](/api-management/user-management#when-to-use-api-ownership) | +| Client Certificates | βœ… | v1.0 | - | [Manage TLS certificate](#tls-certificates) | +| Custom Domain Certificates | βœ… | v1.0 | - | [Manage TLS certificate](#tls-certificates) | +| Public keys pinning | βœ… | v1.0 | - | [Manage TLS certificate](#tls-certificates) | +| Upstream mTLS | βœ… | v1.0 | - | [Manage TLS certificate](#tls-certificates) | +| Kubernetes Ingress | βœ… | v1.0 | - | [Kubernetes Ingress Controller](#control-kubernetes-ingress-resources) | +| Link with SecurityPolicy | βœ… | v1.0 | - | [Protect an API](#add-a-security-policy-to-your-api) | + +#### TykStreamsApiDefinition CRD +The TykStreamsApiDefinition Custom Resource Definition (CRD) manages [Async API configuration](/api-management/event-driven-apis#configuration-options) within a Kubernetes environment. + +##### TykStreamsApiDefinition Features + +`TykStreamsApiDefinition` can support all features of [Tyk Streams](/api-management/event-driven-apis#). You just need to provide the Tyk Streams API definition via a ConfigMap. In addition to managing the CRUD (Create, Read, Update, Delete) of Tyk Streams API resources, the Tyk Operator helps you better manage resources through object linking to Security Policies. See below for a list of Operator features and examples: + +| Features | Support | Supported From | Comments | Example | +| :---------- | :--------- | :----------------- | :---------- | :-------- | +| API Ownership via OperatorContext | βœ… | v1.0 | - | [API Ownership](/api-management/user-management#when-to-use-api-ownership) | +| Link with SecurityPolicy | βœ… | v1.0 | - | [Protect an API](#add-a-security-policy-to-your-api) | + +#### Version Compatability +Ensuring compatibility between different versions is crucial for maintaining stable and efficient operations. This document provides a comprehensive compatibility matrix for Tyk Operator with various versions of Tyk and Kubernetes. By understanding these compatibility details, you can make informed decisions about which versions to deploy in your environment, ensuring that you leverage the latest features and maintain backward compatibility where necessary. + +##### Compatibility with Tyk +Tyk Operator can work with all version of Tyk beyond Tyk 3.x+. Since Tyk is backward compatible, you can safely use the +latest version of Tyk Operator to work with any version of Tyk. +However, if you're using a feature that was not yet available on an earlier version of Tyk, e.g. Defining a Subgraph with Tyk 3.x, you'll see error in Tyk Operator controller manager logs. + +See [Release notes](/developer-support/release-notes/operator) to check for each Tyk Operator release, +which version of Tyk it is tested against. + +| Tyk Version | 3.2 | 4.0 | 4.1 | 4.2 | 4.3 | 5.0 | 5.2 | 5.3 | 5.4 | 5.5 | 5.6 | 5.7 | +| :-------------------- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | :--- | +| Tyk Operator v0.13 | Y | | | | Y | | | | | | | | +| Tyk Operator v0.14 | Y | Y | | | Y | Y | | | | | | | +| Tyk Operator v0.14.1 | Y | Y | | | Y | Y | | | | | | | +| Tyk Operator v0.15.0 | Y | Y | | | Y | Y | | | | | | | +| Tyk Operator v0.15.1 | Y | Y | | | Y | Y | | | | | | | +| Tyk Operator v0.16.0 | Y | Y | | | Y | Y | Y | | | | | | +| Tyk Operator v0.17.0 | Y | Y | | | Y | Y | Y | Y | | | | | +| Tyk Operator v0.17.1 | Y | Y | | | | Y | Y | Y | | | | | +| Tyk Operator v0.18.0 | Y | Y | | | | Y | Y | Y | Y | | | | +| Tyk Operator v1.0.0 | Y | Y | | | | Y | | Y | | Y | Y | | +| Tyk Operator v1.1.0 | Y | Y | | | | Y | | Y | | Y | Y | Y | + +##### Compatibility with Kubernetes Version + +See [Release notes](https://github.com/TykTechnologies/tyk-operator/releases) to check for each Tyk Operator release, +which version of Kubernetes it is tested against. + +| Kubernetes Version | 1.19 | 1.20 | 1.21 | 1.22 | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | 1.28 | 1.29 | 1.30 | +| :-------------------- | :---- | :---- | :---- | :---- | :---- | :---- | :---- | :---- | :---- | :---- | :---- | :---- | +| Tyk Operator v0.13 | Y | Y | Y | Y | Y | Y | Y | | | | | | +| Tyk Operator v0.14 | Y | Y | Y | Y | Y | Y | Y | | | | | | +| Tyk Operator v0.14.1 | | Y | Y | Y | Y | Y | Y | Y | | | | | +| Tyk Operator v0.15.0 | | Y | Y | Y | Y | Y | Y | Y | | | | | +| Tyk Operator v0.15.1 | | Y | Y | Y | Y | Y | Y | Y | | | | | +| Tyk Operator v0.16.0 | | Y | Y | Y | Y | Y | Y | Y | | | | | +| Tyk Operator v0.17.0 | | | | | | | Y | Y | Y | Y | Y | | +| Tyk Operator v0.17.1 | | | | | | | Y | Y | Y | Y | Y | | +| Tyk Operator v0.18.0 | | | | | | | Y | Y | Y | Y | Y | | +| Tyk Operator v1.0.0 | | | | | | | Y | Y | Y | Y | Y | Y | +| Tyk Operator v1.1.0 | | | | | | | Y | Y | Y | Y | Y | Y | + + +#### Security Policy CRD +The SecurityPolicy custom resource defines configuration of [Tyk Security Policy object](/api-management/policies). + +Here are the supported features: + +| Features | Support | Supported From | Example | +| :-------------------------------- | :----------- | :---------------- | :--------- | +| API Access | βœ… | v0.1 | [API Access](#define-the-security-policy-manifest) | +| Rate Limit, Throttling, Quotas | βœ… | v0.1 | [Rate Limit, Throttling, Quotas](#define-the-security-policy-manifest) | +| Meta Data & Tags | βœ… | v0.1 | [Tags and Meta-data](#define-the-security-policy-manifest) | +| Path and Method based permissions | βœ… | v0.1 | [Path based permission](#security-policy-example) | +| Partitions | βœ… | v0.1 | [Partitioned policies](#security-policy-example) | +| Per API limit | βœ… | v1.0 | [Per API Limit](#security-policy-example) | +| Per-Endpoint limit | βœ… | v1.0 | [Per Endpoint Limit](#security-policy-example) | diff --git a/api-management/automations/sync.mdx b/api-management/automations/sync.mdx new file mode 100644 index 00000000..d5fab631 --- /dev/null +++ b/api-management/automations/sync.mdx @@ -0,0 +1,506 @@ +--- +title: "Tyk Sync - Synchronize Tyk Environment With GitHub" +'og:description': "How to synchronize Tyk configuration with Github using Tyk Sync" +tags: ['Tyk API Management', 'Tyk Sync', 'Tyk Operator', 'Github', 'Kubernetes', 'Automations'] +sidebarTitle: "Synchronizing Tyk Configuration" +--- + +## Introduction + +Tyk Sync enables you to export and import Tyk configurations directly from Git, keeping environments aligned without manual configuration updates. This section covers the setup and use of Tyk Sync, providing steps to ensure consistent configurations across different environments. + + +## Tyk Sync Features +Tyk Sync works with *Tyk Dashboard* installation. With Tyk Dashboard, Tyk Sync supports managing API definitions, security policies, and API templates. + +| Tyk Sync Feature | Tyk Dashboard (Licensed) | +| :--------------------------------------------------------------------------- | :-------------------------- | +|

Backup objects from Tyk to a directory

If you want to backup your API definitions, policies and templates in Tyk, you can use the `dump` command. It allows you to save the objects in transportable files. You can use this command to backup important API configurations before upgrading Tyk, or to save API configurations from one Dashboard instance and then use `update`, `publish`, or `sync` commands to update the API configurations to another Dashboard instance. | βœ… | +|

Synchronise objects from Git (or any VCS) to Tyk

To implement GitOps for API management, store your API definitions, policies and templates in Git or any version control system. Use the `sync` command to synchronise those objects to Tyk. During this operation, Tyk Sync will delete any objects in the Dashboard that cannot be found in the VCS, and update those that can be found and create those that are missing. | βœ… | +|

Update objects

The `update` command will read from VCS or file system and will attempt to identify matching API definitions, policies and templates in the target Dashboard, and update them. Unmatched objects will not be created. | βœ… | +|

Publish objects

The `publish` command will read from VCS or file system and create API definitions, policies, and templates in target Dashboard. This will not update any existing objects. If it detects a collision, the command will stop. | βœ… | +|

Show and import Tyk examples

The `examples` command allow you to show and import [Tyk examples](https://github.com/TykTechnologies/tyk-examples). An easy way to load up your Tyk installation with some interesting examples!| βœ… | + +**Working with OAS APIs** + +Starting with Sync v1.5+ and Dashboard v5.3.2+, Tyk Sync supports both [Tyk OAS APIs](/api-management/gateway-config-tyk-oas) and [Tyk Classic APIs](/api-management/gateway-config-tyk-classic) when working with the Tyk Dashboard, without requiring special flags or configurations. + +For Sync versions v1.4.1 to v1.4.3, enabling Tyk Sync for Tyk OAS APIs requires the [allow-unsafe-oas](/tyk-dashboard/configuration#allow_unsafe_oas) configuration in the Dashboard, along with the `--allow-unsafe-oas` flag when invoking Tyk Sync. Note that Tyk Sync versions v1.4.1 to 1.4.3 do not support API Category for Tyk OAS APIs. + +**Working with Tyk Streams APIs** + +Tyk Streams API support was introduced in Tyk Dashboard v5.7.0. Tyk Sync v2.0 and later is compatible with Tyk Streams APIs and manages them similarly to Tyk OAS APIs. With Tyk Sync, you can seamlessly sync, publish, update, and dump Tyk Streams APIs just like OAS APIs. + +Note: The Streams API validator is not applied during these operations. + +**Working with Open Source Gateway** + +From Sync v2.0, compatibility with the Open Source Tyk Gateway has been removed, making Tyk Sync v2.0 compatible exclusively with licensed Tyk Dashboard. As a result, Tyk Sync is no longer usable with the Open Source (OSS) version of the Tyk Gateway. + + +## Set up Tyk Sync +### Installation +Currently the application is available via [Docker](https://hub.docker.com/r/tykio/tyk-sync) and [Packagecloud](https://packagecloud.io/tyk/tyk-sync). + +### Docker + +To install Tyk Sync using Docker, follow these steps: + +#### Pull the Docker image from the Tyk repository + +Make sure to specify the version tag you need. For example, to pull version v1.5.0, use the following command: + +```bash +SYNC_VERSION=v1.5.0 +docker pull tykio/tyk-sync:$SYNC_VERSION +``` + +All docker images are available on the [Tyk Sync Docker Hub](https://hub.docker.com/r/tykio/tyk-sync/tags) page. + +#### Run Tyk Sync + +```bash +SYNC_VERSION=v1.5.0 +docker run tykio/tyk-sync:$SYNC_VERSION [command] [flag] +``` + +If you want to dump your API configurations to the local file system or sync configurations saved locally to Tyk, use Docker [bind mounts](https://docs.docker.com/storage/bind-mounts): + +```bash +docker run -v /path/to/local/directory:/app/data tykio/tyk-sync:$SYNC_VERSION [command] [flag] +``` +Replace [command] with the specific Tyk Sync command you want to execute. + + +### Specify target Tyk installation + +#### Tyk Dashboard +For Dashboard users, you can provide the necessary connection details using the `--dashboard` and `--secret` options. + +```bash +tyk-sync --dashboard --secret [command] [flags] +``` + +DASHBOARD_URL is the fully qualified dashboard target URL (e.g. `http://localhost:3000`) and SECRET refers to the API access key use to access your Dashboard API. For dashboard users, you can get it from the β€œUsers” page under β€œTyk Dashboard API Access Credentials”. + +If you prefer not to provide the secret via the command line, you can set the environment variable `TYKGIT_DB_SECRET` instead. This method keeps your secret secure and avoids exposure in command history. + +```bash +export TYKGIT_DB_SECRET= +tyk-sync --dashboard [command] [flags] +``` + +#### Open Source Gateway +For open source Gateway users, you can provide the necessary connection details using the `--gateway` and `--secret` options. + +```bash +tyk-sync --gateway --secret [command] [flags] +``` + +GATEWAY_URL is the fully qualified gateway target URL (e.g. `http://localhost:8080`) and SECRET refers to the API secret (`secret` parameter in your tyk.conf file) used to access your Gateway API. + +If you prefer not to provide the secret via the command line, you can set the environment variable `TYKGIT_GW_SECRET` instead. This method keeps your secret secure and avoids exposure in command history. + +```bash +export TYKGIT_GW_SECRET= +tyk-sync --gateway [command] [flags] +``` + +2. Export configurations from your development environment: + +```bash +tyk-sync dump -d http://localhost:3000 -s -t dev-backup +``` + +This command exports all configurations from your development Tyk Dashboard to a local directory named `dev-backup`. + +3. Import configurations to your staging environment: + +```bash +tyk-sync publish -d http://staging-dashboard:3000 -s -p dev-backup +``` + +This command imports the configurations from the `dev-backup` directory to your staging Tyk Dashboard. + +### Specify Source API Configurations +For the `sync`, `update`, and `publish` commands, you need to specify where Tyk Sync can get the source API configurations to update the target Tyk installation. You can store the source files either in a Git repository or the local file system. + +#### Working with Git +For any Tyk Sync command that requires Git repository access, specify the Git repository as the first argument after the command. By default, Tyk Sync reads from the `master` branch. To specify a different branch, use the `--branch` or `-b` flag. If the Git repository requires connection using Secure Shell Protocol (SSH), you can specify SSH keys with `--key` or `-k` flag. + +```bash +tyk-sync [command] https://github.com/your-repo --branch develop +``` + +#### Working with the local file system +To update API configurations from the local file system, use the `--path` or `-p` flag to specify the source directory for your API configuration files. + +```bash +tyk-sync [command] --path /path/to/local/directory +``` + +#### Index File Requirement +A `.tyk.json` index file is required at the root of the source Git repository or the specified path. This `.tyk.json` file lists all the files that should be processed by Tyk Sync. + +Example `.tyk.json`: +```json expandable +{ + "type": "apidef", + "files": [ + { + "file": "api1/api1.json" + }, + { + "file": "api2/api2.json" + }, + { + "file": "api3.json" + } + ], + "policies": [ + { + "file": "policy1.json" + } + ], + "assets": [ + { + "file": "template1.json" + } + ] +} +``` + + +## Automate API Configuration Management with Tyk Sync +By integrating GitHub Actions, teams can schedule backups to cloud storage, sync configurations from a Git repository, and update local API definitions directly to the Tyk Dashboard. These workflows ensure configurations are securely maintained, aligned across environments, and easily managed within the API lifecycle. + + +### Backup API Configurations with Github Actions +API platform teams can automate configuration backups using GitHub Actions. By setting up a scheduled GitHub Action, API configurations can be periodically exported and stored in cloud storage, like AWS S3. This approach ensures backups remain up-to-date, offering a reliable way to safeguard data and simplify restoration if needed. + + +#### Create a GitHub Action workflow + +1. In your repository, create a new file `.github/workflows/tyk-backup.yml`. +2. Add the following content to the `tyk-backup.yml` file: + +```yaml expandable +name: Tyk Backup + +on: + schedule: + - cron: '0 0 * * *' # Runs every day at midnight + +jobs: + backup: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Create Backup Directory + run: | + BACKUP_DIR="backup/$(date +%Y-%m-%d)" + mkdir -p $BACKUP_DIR + echo "BACKUP_DIR=$BACKUP_DIR" >> $GITHUB_ENV + + - name: Set Permissions for Backup Directory + run: | + sudo chown -R 1001:1001 ${{ github.workspace }}/backup + + - name: Dump API Configurations + run: | + docker run --user 1001:1001 -v ${{ github.workspace }}:/app/data tykio/tyk-sync:${TYK_SYNC_VERSION} dump --target /app/data/${{ env.BACKUP_DIR }} --dashboard ${TYK_DASHBOARD_URL} --secret ${TYK_DASHBOARD_SECRET} + env: + TYK_SYNC_VERSION: ${{ vars.TYK_SYNC_VERSION }} + TYK_DASHBOARD_URL: ${{ secrets.TYK_DASHBOARD_URL }} + TYK_DASHBOARD_SECRET: ${{ secrets.TYK_DASHBOARD_SECRET }} + + - name: Upload to S3 + uses: jakejarvis/s3-sync-action@v0.5.1 + with: + args: --acl private --follow-symlinks --delete + env: + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: 'us-east-1' # Change to your region + SOURCE_DIR: ${{ env.BACKUP_DIR }} +``` + +#### Set up secrets + +1. Go to your GitHub repository. +2. Navigate to Settings > Secrets and variables > Actions. +3. Add the following variable: + - `TYK_SYNC_VERSION`: The version of Tyk Sync you want to use. +4. Add the following secrets: + - `TYK_DASHBOARD_URL`: The URL of your Tyk Dashboard. + - `TYK_DASHBOARD_SECRET`: The secret key for your Tyk Dashboard. + - `AWS_S3_BUCKET`: The name of your AWS S3 bucket. + - `AWS_ACCESS_KEY_ID`: Your AWS access key ID. + - `AWS_SECRET_ACCESS_KEY`: Your AWS secret access key. + +#### Commit and push changes + +Commit the `tyk-backup.yml` file and push it to the main branch of your repository. + +#### Verify backups + +The GitHub Action will run every day at midnight, dumping API configurations into a backup directory and uploading them to your specified S3 bucket. + + +### Synchronize API configurations with GitHub Actions +API platform teams can use GitHub Actions to sync API configurations, policies, and templates from a Git repository to Tyk. Triggered by repository changes, the action generates a .tyk.json file and applies updates with the sync command, keeping the Tyk setup aligned with the repository. + +#### Setup GitHub repository +Organize your repository with the following structure: + +- `/apis/` for API definition files. +- `/policies/` for security policy files. +- `/assets/` for API template files. + +#### Create a GitHub Action workflow + +1. In your repository, create a new file `.github/workflows/tyk-sync.yml`. +2. Add the following content to the `tyk-sync.yml` file: + +```yaml expandable +name: Tyk Sync + +on: + push: + branches: + - main + +jobs: + sync: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Create .tyk.json + run: | + echo '{' > .tyk.json + echo ' "type": "apidef",' >> .tyk.json + echo ' "files": [' >> .tyk.json + find . -type f -name '*.json' -path './apis/*' -exec echo ' {"file": "{}"},' \; | sed '$ s/,$//' >> .tyk.json + echo ' ],' >> .tyk.json + echo ' "policies": [' >> .tyk.json + find . -type f -name '*.json' -path './policies/*' -exec echo ' {"file": "{}"},' \; | sed '$ s/,$//' >> .tyk.json + echo ' ],' >> .tyk.json + echo ' "assets": [' >> .tyk.json + find . -type f -name '*.json' -path './assets/*' -exec echo ' {"file": "{}"},' \; | sed '$ s/,$//' >> .tyk.json + echo ' ]' >> .tyk.json + echo '}' >> .tyk.json + cat .tyk.json + + - name: Sync with Tyk + run: | + docker run tykio/tyk-sync:${TYK_SYNC_VERSION} version + docker run -v ${{ github.workspace }}:/app/data tykio/tyk-sync:${TYK_SYNC_VERSION} sync --path /app/data --dashboard ${TYK_DASHBOARD_URL} --secret ${TYK_DASHBOARD_SECRET} + env: + TYK_SYNC_VERSION: ${{ vars.TYK_SYNC_VERSION }} + TYK_DASHBOARD_URL: ${{ secrets.TYK_DASHBOARD_URL }} + TYK_DASHBOARD_SECRET: ${{ secrets.TYK_DASHBOARD_SECRET }} +``` + +#### Set up secrets + +1. Go to your GitHub repository. +2. Navigate to Settings > Secrets and variables > Actions. +3. Add the following variable: + - `TYK_SYNC_VERSION`: The version of Tyk Sync you want to use (e.g., v2.0.0). +4. Add the following secrets: + - `TYK_DASHBOARD_URL`: The URL of your Tyk Dashboard. + - `TYK_DASHBOARD_SECRET`: The secret key for your Tyk Dashboard. + +#### Commit and push changes + +Commit the `tyk-sync.yml` file and push it to the main branch of your repository. + +#### Verify synchronisation + +Each time there is a change in the repository, the GitHub Action will be triggered. It will create the `.tyk.json` file including all JSON files in the repository and use the `sync` command to update the Tyk installation. + + +### Update API Definitions locally +For API developers managing definitions locally, Tyk Sync's publish or update commands can upload local API definitions directly to the Tyk Dashboard, streamlining updates and keeping definitions in sync during development. Follow these steps to update your API definitions locally. + +#### Prepare your API Definition + +Create your API definition file and save it locally. For example, save it as *api1.json* in a directory structure of your choice. + +#### Create a .tyk.json index file + +In the root directory of your API definitions, create a `.tyk.json` file to list all API definition files that Tyk Sync should process. + +Example `.tyk.json`: +```json expandable +{ + "type": "apidef", + "files": [ + { + "file": "api1.json" + } + ] +} +``` + +#### Install Tyk Sync via Docker + +If you haven't installed Tyk Sync, you can do so via Docker: + +```bash +docker pull tykio/tyk-sync:v2.0.0 +``` + +#### Publish API Definitions to Tyk + +Use the `publish` command to upload your local API definitions to Tyk. Use Docker bind mounts to access your local files. + +```bash +docker run -v /path/to/your/directory:/app/data tykio/tyk-sync:v2.0.0 publish \ + --path /app/data \ + --dashboard [DASHBOARD_URL] \ + --secret [SECRET] +``` + +#### Update API Definitions to Tyk + +Similarly, to update existing API definitions, use the update command. + +```bash +docker run -v /path/to/your/directory:/app/data tykio/tyk-sync:v2.0.0 update \ + --path /app/data \ + --dashboard [DASHBOARD_URL] \ + --secret [SECRET] +``` + +#### Verify the update + +Log in to your Tyk Dashboard to verify that the API definitions have been published or updated successfully. + + +## Tyk Sync Commands + +### Dump Command + +| Aspect | Details | +| :--------------- | :------------------------------------------------------------------------------------------------------ | +| **Command** | `tyk-sync dump` | +| **Usage** | ```tyk-sync dump -d DASHBOARD_URL [-s SECRET] [-t PATH]``` | expandable +| **Flags** | `-d, --dashboard DASHBOARD_URL`: Tyk Dashboard URL (required)
`-h, --help`: Help for the dump command
`-t, --target PATH`: Target directory for output files (optional)
`-s, --secret SECRET`: API secret for Dashboard access (optional)
`--apis IDS`: Specific API IDs to dump
`--oas-apis IDS`: Specific OAS API IDs to dump
`--policies IDS`: Specific policy IDs to dump
`--templates IDS`: Specific template IDs to dump | +| **Example** | ```tyk-sync dump --dashboard http://tyk-dashboard:3000 --secret your-secret ```| +| **Example** | ```tyk-sync dump --dashboard http://tyk-dashboard:3000 --secret your-secret --target /path/to/backup --apis c2ltcGxlLWdyYXBoLWRldi90eWthcGktc2NoZW1h,baa5d2b65f1b45385dac3aeb658fa04c ``` | + +### Examples Command + +| Aspect | Details | +| :--------------- | :------------------------------------------------------------------------------------------------------ | +| **Command** | `tyk-sync examples` | +| **Usage** | ```tyk-sync examples [flags]```
```tyk-sync examples [command]``` | +| **Subcommands**| `publish`: Publish a specific example
`show`: Show details of a specific example | +| **Flags** | `-h, --help`: Help for examples command | +| **Example** | ```tyk-sync examples ``` | + +### Examples Show Command + +| Aspect | Details | +| :--------------- | :------------------------------------------------------------------------------------------------------ | +| **Command** | ```tyk-sync examples show``` | +| **Usage** | ```tyk-sync examples show [flags]``` | +| **Flags** | `-h, --help`: Help for show command
`-l, --location string`: Location of the example | +| **Example** | ```tyk-sync examples show --location="udg/vat-checker" ``` | + +### Examples Publish Command + +| Aspect | Details | +| :--------------- | :------------------------------------------------------------------------------------------------------ | +| **Command** | ```tyk-sync examples publish``` | +| **Usage** | ```tyk-sync examples publish [flags]``` | +| **Flags** | `-b, --branch string`: Branch to use (default "refs/heads/main")
`-d, --dashboard string`: Dashboard target URL
`-g, --gateway string`: Gateway target URL
`-h, --help`: Help for publish command
`-k, --key string`: Key file location for auth
`-l, --location string`: Location of the example
`-s, --secret string`: API secret
`--test`: Use test publisher, output to stdio | +| **Example** | ```tyk-sync examples publish -d="http://localhost:3000" -s="b2d420ca5302442b6f20100f76de7d83" -l="udg/vat-checker" ``` | + +### Publish Command + +| Aspect | Details | +| :--------------- | :------------------------------------------------------------------------------------------------------ | +| **Command** | ```tyk-sync publish``` | +| **Usage** | ```tyk-sync publish {-d DASHBOARD_URL \| -g GATEWAY_URL} [-s SECRET] [-b BRANCH] [-k SSHKEY] [-o ORG_ID] REPOSITORY_URL```

```tyk-sync publish {-d DASHBOARD_URL \| -g GATEWAY_URL} [-s SECRET] [-o ORG_ID] -p PATH``` | +| **Flags** | `-b, --branch BRANCH`: Git branch (default "refs/heads/master")
`-d, --dashboard DASHBOARD_URL`: Dashboard URL
`-g, --gateway GATEWAY_URL`: Gateway URL
`-h, --help`: Help for publish command
`-k, --key SSHKEY`: SSH key file location
`-p, --path PATH`: Source file directory
`-s, --secret SECRET`: API secret
`--no-delete`: Skip deletion of resources during synchronisation
`--test`: Use test publisher
`--allow-duplicate-listenpaths`: Allow duplicate listen paths
`--apis IDS`: Specific API IDs to publish
`--oas-apis IDS`: Specific OAS API IDs to publish
`--policies IDS`: Specific policy IDs to publish
`--templates IDS`: Specific template IDs to publish | +| **Example** | ```tyk-sync publish -d http://tyk-dashboard:3000 -s your-secret -p /app/data --apis 726e705e6afc432742867e1bd898cb23 ```| +| **Example** | ```tyk-sync publish -d http://tyk-dashboard:3000 -s your-secret -b develop https://github.com/your-repo/your-apis ``` | + +### Sync Command + +| Aspect | Details | +| :--------------- | :------------------------------------------------------------------------------------------------------ | +| **Command** | `tyk-sync sync` | +| **Usage** | ```tyk-sync sync {-d DASHBOARD_URL \| -g GATEWAY_URL} [-s SECRET] [-b BRANCH] [-k SSHKEY] [-o ORG_ID] REPOSITORY_URL```

```tyk-sync sync {-d DASHBOARD_URL \| -g GATEWAY_URL} [-s SECRET] [-o ORG_ID] -p PATH``` | +| **Flags** | `-b, --branch BRANCH`: Git branch (default "refs/heads/master")
`-d, --dashboard DASHBOARD_URL`: Dashboard URL
`-g, --gateway GATEWAY_URL`: Gateway URL
`-h, --help`: Help for sync command
`-k, --key SSHKEY`: SSH key file location
`-o, --org ORG_ID`: Override organization ID
`-p, --path PATH`: Source file directory
`-s, --secret SECRET`: API secret
`--test`: Use test publisher
`--apis IDS`: Specific API IDs to sync (to be deprecated)
`--policies IDS`: Specific policy IDs to sync (to be deprecated) | +| **Example** | ```tyk-sync sync -d http://tyk-dashboard:3000 -s your-secret https://github.com/your-repo/your-apis ```| +| **Example** | ```tyk-sync sync -d http://tyk-dashboard:3000 -s your-secret -p /path/to/your/apis ``` | + +### Update Command + +| Aspect | Details | +| :--------------- | :------------------------------------------------------------------------------------------------------ | +| **Command** | `tyk-sync update` | +| **Usage** | ```tyk-sync update {-d DASHBOARD_URL \| -g GATEWAY_URL} [-s SECRET] [-b BRANCH] [-k SSHKEY] [-o ORG_ID] REPOSITORY_URL```

```tyk-sync update {-d DASHBOARD_URL \| -g GATEWAY_URL} [-s SECRET] [-o ORG_ID] -p PATH``` | +| **Flags** | `-b, --branch BRANCH`: Git branch (default "refs/heads/master")
`-d, --dashboard DASHBOARD_URL`: Dashboard URL
`-g, --gateway GATEWAY_URL`: Gateway URL
`-h, --help`: Help for update command
`-k, --key SSHKEY`: SSH key file location
`-p, --path PATH`: Source file directory
`-s, --secret SECRET`: API secret
`--test`: Use test publisher
`--apis IDS`: Specific API IDs to update
`--oas-apis IDS`: Specific OAS API IDs to update
`--policies IDS`: Specific policy IDs to update
`--templates IDS`: Specific template IDs to update | +| **Example** | ```tyk-sync update -d http://tyk-dashboard:3000 -s your-secret -p /app/data --apis 726e705e6afc432742867e1bd898cb23```| +| **Example** | ```tyk-sync update -d http://tyk-dashboard:3000 -s your-secret -b develop https://github.com/your-repo/your-apis ``` | + +## Troubleshooting and FAQ + +### How are Tyk configurations synchronized to Git? + +Tyk Sync allows you to dump configurations to a local directory, which can then be committed to a Git repository. This enables version control and easy synchronization across environments. + +For example: +1. Dump configurations: `tyk-sync dump -d http://dashboard:3000 -s secret -t ./configs` +2. Commit to Git: + ``` + cd configs + git add . + git commit -m "Update Tyk configurations" + git push + ``` + +### Can I sync multiple APIs to a single Git repository? + +Yes, you can store multiple API definitions, policies, and other Tyk resources in a single Git repository. Tyk Sync and Tyk Operator can work with multiple resources in the same directory. + +Your repository structure might look like this: +``` +tyk-configs/ +β”œβ”€β”€ apis/ +β”‚ β”œβ”€β”€ api1.yaml +β”‚ └── api2.yaml +β”œβ”€β”€ policies/ +β”‚ β”œβ”€β”€ policy1.yaml +β”‚ └── policy2.yaml +└── tyk-operator/ + └── operator-context.yaml +``` + +### How do I roll back changes made with Tyk Sync? + +To roll back changes made with Tyk Sync: + +1. If you're using Git, check out the previous version of your configurations: + ```bash + git checkout + ``` + +2. Use Tyk Sync to publish the previous version: + ```bash + tyk-sync sync -d http://dashboard:3000 -s -p ./ + ``` + +It's a good practice to maintain separate branches or tags for different environments to make rollbacks easier. + diff --git a/api-management/batch-processing.mdx b/api-management/batch-processing.mdx new file mode 100644 index 00000000..bf738417 --- /dev/null +++ b/api-management/batch-processing.mdx @@ -0,0 +1,234 @@ +--- +title: "Batch Processing" +'og:description': "Make multiple API requests in a single HTTP call using Batch Requests" +tags: ['Request Optimization', 'Optimization', 'Batched Requests', 'Batch', 'Batch Processing'] +sidebarTitle: "Batch Processing" +--- + +## Overview + +Batch Requests is a powerful Tyk Gateway feature that allows clients to make multiple requests to an API in a single HTTP call. Instead of sending numerous individual requests to the API, clients can bundle these requests together, reducing network overhead and improving performance. + +### What are Batch Requests? + +Batch Requests act as an aggregator for multiple API calls. When a client sends a batch request to Tyk, the Gateway processes each request in the batch individually (applying all relevant middleware, authentication, and rate limiting) and returns a combined response containing the results of all requests. The scope of a batch request is limited to a single API deployed on Tyk, though can comprise requests to different endpoints (method and path) defined for that API. + +### Key Benefits + +- Reduced Network Overhead: Minimize the number of HTTP connections required for multiple related API operations +- Improved Client Performance: Decrease latency by eliminating multiple round-trips to the server +- Simplified Error Handling: Process success and failure responses for multiple operations in a single place +- Maintained Security: Each individual request within a batch still goes through Tyk's full security pipeline +- Flexible Execution: Choose between parallel or sequential execution of requests + +### When to Use Batch Requests + +Batch Requests are ideal for scenarios such as: + +- Mobile applications that need to fetch data from multiple endpoints during startup +- Dashboard applications that need to populate multiple widgets with different API data +- Complex workflows that require data from several API endpoints to complete a single user action +- Integration scenarios where you need to synchronize operations across multiple services + +### How Batch Requests Work + +When Tyk receives a batch request, it: + +- Validates the batch request format +- Processes each request in the batch individually (applying all middleware, authentication, and quotas) +- Collects all responses +- Returns a single combined response to the client + +This process ensures that security is maintained while providing the performance benefits of batching. + +## Using Batch Requests + +### Configuration + +Batch Requests are disabled by default, so you need to enable batch request support in your API definition by setting `server.batchProcessing.enabled` in the Tyk Vendor Extension (Tyk Classic: `enable_batch_request_support`). + +### Batch Request Endpoint + +When batch requests are enabled, Tyk automatically creates an additional logical endpoint on the subrouter for the API. This won't appear in the API definition and so will not be added to the OpenAPI description. This `/tyk/batch/` endpoint accepts requests in a specific "batch" format and processes them as described in the next section. + +For example, if your API's listen path is `/myapi/` the batch request endpoint would be `/myapi/tyk/batch/`. + +Note that the trailing slash `/` at the end of the URL is required when calling this endpoint. + +### Batch Request Format + +Batch requests must be sent as HTTP `POST` requests with a JSON payload that follows this structure: + +```json expandable +{ + "requests": [ + { + "method": "GET", + "headers": { + "x-header-1": "value-1", + "authorization": "your-auth-token" + }, + "body": "", + "relative_url": "resource/123" + }, + { + "method": "POST", + "headers": { + "x-header-2": "value-2", + "authorization": "your-auth-token" + }, + "body": "{\"property\": \"value\"}", + "relative_url": "resource/create" + }, + { + "method": "GET", + "headers": { + "x-header-3": "value-3", + "authorization": "your-auth-token" + }, + "body": "", + "relative_url": "resource/invalid" + } + ], + "suppress_parallel_execution": false +} +``` + +Where: + +- `requests`: An array of individual requests to be processed + - `method`: The HTTP method for the individual request (`GET`, `POST`, `PUT`, `DELETE`, etc.) + - `headers`: Any HTTP headers to include with the request + - `body`: The request body (for `POST`, `PUT` requests) in the format prescribed by the API (e.g. JSON string) + - `relative_url`: The endpoint for the request, which can include query parameters +- `suppress_parallel_execution`: A boolean flag to control whether requests should be processed in parallel (`false`) or sequentially in the order that they appear in the array (`true`) + +In the example above, on receipt of a request to `POST /my-api/tyk/batch` with this payload, Tyk would process three requests in parallel: + +- `GET /my-api/resource/123` passing `x-header-1` and `Authorization` headers +- `POST /my-api/resource/create` passing `x-header-2` and `Authorization` headers and the payload descrbied in `body` +- `GET /my-api/resource/invalid` passing `x-header-3` and `Authorization` headers + +### Execution Order + +Tyk will work through the requests in the batch in the order that they are declared in the `requests` array. The `suppress_parallel_execution` setting is used to determine whether Tyk should wait for each request to complete before starting the next (`true`), or if it should issue all of the requests in parallel (`false`). + +If sequential execution is in use, Tyk will work through the entire `requests` array regardless of whether any requests return errors. All responses (success and failure) will be logged and returned to the client as described [below](/api-management/batch-processing#batch-response-format). + +### Batch Response Format + +When you send a batch request to Tyk, each individual request within the batch is processed independently. This means that some requests in a batch may succeed while others fail. Tyk provides detailed response information for each request in the batch to help you identify and handle errors appropriately. + +The response from a batch request is an array of response objects, each corresponding to one of the requests in the batch in the order that they appeared in the `requests` array: + +```json expandable +[ + { + "relative_url": "resource/123", + "code": 200, + "headers": { + "Content-Type": ["application/json"], + "Date": ["Wed, 15 Mar 2023 12:34:56 GMT"] + }, + "body": "{\"id\":\"123\",\"name\":\"Example Resource\"}" + }, + { + "relative_url": "resource/create", + "code": 201, + "headers": { + "Content-Type": ["application/json"], + "Date": ["Wed, 15 Mar 2023 12:34:56 GMT"] + }, + "body": "{\"id\":\"456\",\"name\":\"New Resource\",\"status\":\"created\"}" + }, + { + "relative_url": "resource/invalid", + "code": 404, + "headers": { + "Content-Type": ["application/json"], + "Date": ["Wed, 15 Mar 2023 12:34:56 GMT"] + }, + "body": "{\"error\":\"Resource not found\"}" + } +] +``` + +Each response object contains: + +- `relative_url`: The URL of the endpoint targeted by the request +- `code`: The HTTP status code returned from the individual request +- `headers`: The response headers +- `body`: The response body as a string + +### Response Status Codes + +The batch endpoint itself returns an `HTTP 200 OK` status code as long as the batch request was properly formatted and processed, regardless of whether individual requests within the batch succeeded or failed. + +To determine the success or failure of individual requests, you need to examine the status code for each request in the response array. + +In the previous example, we can see that the first two requests were successful, returning `HTTP 200 OK` and `HTTP 201 Created`, whereas the third failed returning `HTTP 404 Not found`. + +## Invoking Batch Requests from Custom JavaScript Middleware + +You can make requests to the logical batch request endpoint from within [custom JavaScript middleware](/api-management/plugins/javascript) via the `TykBatchRequest` function that is included in Tyk's [JavaScript API](/api-management/plugins/javascript#javascript-api). + +This integration enables you to: + +- Create batch requests programmatically +- Process batch responses with custom logic +- Implement advanced error handling specific to your use case + +## Security Considerations + +Requests to the `/tyk/batch/` endpoint do not require any authentication, however the requests within the batch (declared in the payload) do not bypass any security mechanisms. + +As this endpoint is keyless, no rate limiting is applied to the requests to `/tyk/batch/`. + +Each request in a batch is processed through Tyk's full security pipeline, including authentication and rate limiting, so API keys or other authentication credentials must be included in each individual request within the batch. + +Rate limiting and quotas are applied to each request in the batch individually - so a batch containing three requests using the same API key will add three to their rate limit and quota counts. This could lead to one or more of the batched requests being rejected. + +This means that, whilst anyone can make a request to the batch endpoint, they can only successfully execute requests within the batch by providing valid authentication credentials in those requests. + +This means that the batch endpoint could potentially be used for reconnaissance, as attackers might determine which APIs exist based on responses. If this is a concern then you could consider: + +- using IP allowlists to restrict access to your API +- using [Internal Looping](/advanced-configuration/transform-traffic/looping) to put the batch request API behind a protected API +- disabling batch requests entirely if you don't need this feature + +## Performance Considerations + +- Setting `suppress_parallel_execution` to `false` provides better performance but doesn't guarantee response order. +- For large batches, consider the impact on your upstream services +- Tyk applies rate limiting to each request in the batch, which may cause some requests to be rejected if limits are exceeded + +## Best Practices when using Tyk's Batch Request feature + +We recommend that you consider the following best practice guidelines when using batch requests: + +- Validate Before Sending: Perform client-side validation before including requests in a batch to minimize predictable errors. +- Implement Timeouts: Set appropriate timeouts for batch requests to prevent long-running operations from blocking your application. +- Log Detailed Errors: Log detailed error information for failed requests to facilitate debugging. +- Group Similar Requests: Group requests with similar authentication requirements and rate limits to minimize errors. +- Implement Circuit Breakers: Use circuit breaker patterns to prevent repeated failures when upstream services are experiencing issues. + +## Troubleshooting + +There are some common issues that can be encountered when using Tyk's batch requests feature. + +### Missed trailing slash + +When an API client makes a request to the logical `tyk/batch/` endpoint, it is essential that the trailing slash is included in the request, otherwise Tyk will return an `HTTP 404` error. + +### Custom domains + +Several specific issues can arise when using batch requests with custom domains: + +**DNS Resolution**: The Tyk Gateway needs to be able to resolve the custom domain internally. If the Gateway can't resolve the custom domain name, batch requests will fail with connection errors, even though external requests to the same API work fine. +**Solution**: Ensure that the Tyk Gateway host can resolve the custom domain, either through proper DNS configuration or by adding entries to the host's `/etc/hosts` file. + +**Internal vs. External Routing**: When a batch request is made to a custom domain, Tyk needs to route the individual requests within the batch correctly. If the custom domain is only configured for external access but not for internal routing, the batch requests may fail. +**Solution**: Configure your custom domain to work with both external and internal routing. + +**Certificate Validation**: If your custom domain uses HTTPS, certificate validation issues can occur during the internal processing of batch requests. +**Solution**: Ensure that the certificates for your custom domain are properly configured and trusted by the Tyk Gateway. \ No newline at end of file diff --git a/api-management/certificates.mdx b/api-management/certificates.mdx new file mode 100644 index 00000000..8347b645 --- /dev/null +++ b/api-management/certificates.mdx @@ -0,0 +1,454 @@ +--- +title: "Certificates - TLS and SSL" +'og:description': "How to enable SSL with the Tyk Gateway and Dashboard" +tags: ['TLS', 'SSL', 'Security', 'Certificate', 'Pinning'] +order: 2 +sidebarTitle: "Certificates in Tyk" +--- + +## Introduction + +Secure communication is essential in today's digital landscape. TLS/SSL protocol and Public Key Infrastructure (PKI) play a crucial role in ensuring encrypted and authenticated connections. This document provides a comprehensive walkthrough on configuring TLS/SSL, managing certificates for the Tyk Gateway and Dashboard. + +In this section, we delve into the following key topics: + +1. **[Enabling TLS in Tyk components](/api-management/certificates#enable-tlsssl-in-tyk-components)**: + Learn how to enable and configure TLS/SSL for Tyk Gateway and Dashboard to secure your communication. +2. **[TLS Support in Tyk](/api-management/certificates#tlsssl-configuration)**: + Understand the supported TLS versions, cipher suites, their configurations, and best practices for secure communication. +3. **[Configuring Tyk Certificate Storage](/api-management/certificates#using-tyk-certificate-storage)**: + Discover how to manage and store certificates for seamless TLS configuration in Tyk. + Explore advanced TLS settings for enhanced security. +4. **[Self Signed Certificates](/api-management/certificates#self-signed-certificates)**: + Learn how to configure and use self-signed certificates for secure communication in Tyk. +5. **[Configuring Internal Proxy Setup](/api-management/certificates#internal-proxy-setup)**: + Set up internal proxies with TLS to ensure secure communication within your architecture. + +### Certificates + +If you have had to configure an SSL server or SSH access, the following information below should be familiar to you. + +Let's start with certificate definition. Here is what [Wikipedia](https://en.wikipedia.org/wiki/Public_key_certificate) says: + +> In cryptography, a public key certificate, also known as a digital certificate or identity certificate, is an electronic document used to prove the ownership of a public key. The certificate includes information about the key, information about the identity of its owner (called the subject), and the digital signature of an entity that has verified the certificate's contents (called the issuer). If the signature is valid, and the software examining the certificate trusts the issuer, then it can use that key to communicate securely with the certificate's subject. + +When it comes to authorization, it is enough for the server that has a public client certificate in its trusted certificate storage to trust it. However, if you need to send a request to the server protected by mutual TLS, or need to configure the TLS server itself, you also need to have a private key, used while generating the certificate, to sign the request. + +Using Tyk, you have two main certificate use cases: + +1. Certificates without public keys used for [client authorization and authentication](/basic-config-and-security/security/mutual-tls/client-mtls#why-use-mutual-tls) +2. Certificates with private keys used for [upstream access](/api-management/upstream-authentication/mtls), and server certificates (in other words when we need to sign and encrypt the request or response). + +### PEM format + +Before a certificate can be used by Tyk, it must be encoded into **PEM format**. If you are using an `openssl` command to generate certificates, it should use PEM by default. A nice bonus of the PEM format is that it allows having multiple entries inside the same file. So in cases where a certificate also requires a private key, you can just concatenate the two files together. + +## Enable TLS/SSL in Tyk components + +TLS protocol is supported by all Tyk components. You can enable TLS in Tyk Gateway and Dashboard by modifying the `tyk.conf` and `tyk_analytics.conf` files. + +For self signed certificates additional consideration has to be taken place, [refer to the section below](#self-signed-certificates). + +
+ +### Gateway + +You'll need to add the following to your **tyk.conf** as the minimum to enable TLS for the Gateway: + +```json expandable +"http_server_options": { + "use_ssl": true, + "certificates": [ + { + "domain_name": "*.yoursite.com", + "cert_file": "./new.cert.cert", + "key_file": "./new.cert.key" + } + ] +} +``` + +### Dashboard + +You'll need to add the following to your **tyk_analytics.conf** as the minimum to enable TLS for the Dashboard: + +```json expandable +"http_server_options": { + "use_ssl": true, + "certificates": [ + { + "domain_name": "*.yoursite.com", + "cert_file": "./new.cert.cert", + "key_file": "./new.cert.key" + } + ] +} +``` + +Set the [host_config.generate_secure_paths](/tyk-dashboard/configuration#host_configgenerate_secure_paths) flag to `true` so that your Dashboard URL starts with HTTPS. + +If you are using self-signed certs or are in a test environment, [you can tell Tyk to ignore validation on certs Mutual TLS support](#self-signed-certificates) + +### Testing TLS/SSL Configuration + +Restart the servers/containers and they should now be using SSL: +```{.copyWrapper} +$ docker-compose up tyk-gateway tyk-dashboard +... +tyk-gateway_1 | time="Apr 24 18:30:47" level=info msg="--> Using TLS (https)" prefix=main +tyk-gateway_1 | time="Apr 24 18:30:47" level=warning msg="Starting HTTP server on:[::]:443" prefix=main +... +``` + +And then we can curl both servers: +```{.copyWrapper} +$ curl -k https://localhost:8080/hello +{"status":"pass","version":"v3.0.0","description":"Tyk GW","details":{"dashboard":{"status":"pass","componentType":"system","time":"2020-08-28T17:19:49+02:00"},"redis":{"status":"pass","componentType":"datastore","time":"2020-08-28T17:19:49+02:00"}}} + +$ curl -k https://localhost:3000 + +``` + +### MDCB + +Mutual TLS configuration in an MDCB environment has specific requirements. An MDCB environment consists of a Control Plane and multiple Data Planes that, using MDCB, sync configuration. +The Control Plane and Data Plane deployments usually do not share any secrets; thus a certificate with private keys encoded with secret in the Control Plane will not be accessible to Data Plane gateways. + +To solve this issue, you need to set `security.private_certificate_encoding_secret` in the MDCB configuration file to the same value as specified in your management Gateway configuration file. By knowing the original secret, MDCB will be able to decode private keys, and +send them to client without password. Using a secure connection between Data Plane Gateways and MDCB is required in this case. See MDCB setup page for use_ssl usage. + + +## TLS/SSL Configuration + +TLS is configured in the `http_server_options` section of your Gateway and Dashboard configuration files. This has the following structure, common to both components: + +```{.copyWrapper} expandable +"http_server_options": { + "use_ssl": true, + "server_name": "yoursite.com", + "min_version": 771, + "max_version": 772, + "ssl_ciphers": ["TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256", "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256"], + "certificates": [ + { + "domain_name": "*.yoursite.com", + "cert_file": "./new.cert.cert", + "key_file": "./new.cert.key" + } + ] +}, +``` + +- `min_version` and `max_version` are optional and allow you to configure the [versions of TLS](#supported-tls-versions) from which Tyk will accept connections +- `ssl_ciphers` allows you to select the [cipher suite](#supported-tls-cipher-suites) that will be used to negotiate connections +- you can enter multiple certificates to be used in the encryption that will be applied for different domain names, this enables you to have multiple TLS certs for your Gateways or Dashboard domains if they are providing access to different domains via the same IP + +### Supported TLS Versions + +You need to use the following values for setting the TLS `min_version` and `max_version`. The numbers associated with the TLS versions represent protocol version numbers used in the TLS protocol specification. These are standardized numerical values assigned by the Internet Engineering Task Force (IETF) to identify each TLS version during communication. + +| TLS Version | Value to Use | +| :----------------------- | :---------------- | +| 1.0 (see note) | 769 | +| 1.1 (see note) | 770 | +| 1.2 | 771 | +| 1.3 | 772 | + +If you do not configure minimum and maximum TLS versions, then Tyk Gateway will default to: + - minimum TLS version: 1.2 + - maximum TLS version: 1.3 + + + + + Tyk uses Golang libraries to provide TLS functionality, so the range of TLS versions supported by the Gateway is dependent upon the underlying library. Support for TLS 1.0 and 1.1 was removed in Go 1.22 (which was adopted in Tyk 5.3.6/5.6.0), so these are no longer supported by Tyk. + + + +### Supported TLS Cipher Suites + +The strength of encryption is determined by the cipher that is negotiated between client & server; each version of the TLS protocol provides a suite of available ciphers. + +TLS 1.3 protocol does not allow the setting of custom ciphers, and is designed to automatically pick the most secure cipher. + +When using earlier TLS protocols, you can deliberately choose the ciphers to be used using the `http_server_options` config option `ssl_ciphers` in `tyk.conf` and `tyk-analytics.conf`. This takes an array of strings as its value. Each string must be one of the allowed cipher suites as defined at https://golang.org/pkg/crypto/tls/#pkg-constants + +For example: + +```json +{ + "http_server_options": { + "ssl_ciphers": ["TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256", "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256"], + } +} +``` + +If no ciphers match, Tyk will default to golang crypto/tls standard ciphers. + +```text expandable +"ssl_ciphers": ["TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256", "TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256"] + +SSL-Session: + Protocol : TLSv1.2 + Cipher : ECDHE-RSA-AES128-SHA256 + Session-ID: 8246BAFF7396BEDE71FD5AABAD493A1DD2CAF4BD70BA9A816AD2969CFD3EAA98 + Session-ID-ctx: + Master-Key: 3BB6A2623FCCAD272AE0EADFA168F13FDAC83CEAFCA232BD8A8B68CEACA373552BE5340A78672A116A908E61EEF0AD29 +``` + +```text expandable +"ssl_ciphers": ["TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256"] + +2018/06/22 18:15:00 http: TLS handshake error from 127.0.0.1:51187: tls: no cipher suite supported by both client and server + +SSL-Session: + Protocol : TLSv1.2 + Cipher : 0000 + Session-ID: + Session-ID-ctx: + Master-Key: + Start Time: 1529687700 + Timeout : 7200 (sec) + Verify return code: 0 (ok) +``` + +```text expandable +"ssl_ciphers": ["junk or empty"] + +SSL-Session: + Protocol : TLSv1.2 + Cipher : ECDHE-RSA-AES256-GCM-SHA384 + Session-ID: A6CFF2DCCE2344A59D877872F89BDC9C9B2F15E1BBAE8C7926F32E15F957AA2B + Session-ID-ctx: + Master-Key: 88D36C895808BDF9A5481A8CFD68A0B821CF8E6A6B8C39B40DB22DA82F6E2E791C77A38FDF5DC6D21AAE3D09825E4A2A +``` + + +### Validate Hostname against Common Name + +From v2.9.3 you can force the validation of the hostname against the common name, both at the Gateway level via your `tyk.conf` and at the API level. + + + + +Set `ssl_force_common_name_check` to `true` in your `tyk.conf` file. + + + +Use `proxy.transport.ssl_force_common_name_check` in your API definition. + + + + +### Dynamically setting SSL certificates for custom domains + +If you include certificateID or certificate path to an API definition `certificates` field, Gateway will dynamically load this ceritficate for your custom domain, so you will not need to restart the process. You can do it from the Dashboard UI too, in the custom domain section. + + + + +Let say the domain certificate is stored in secret named `my-test-tls` in the same namespace as this ApiDefinition resource `httpbin`. You can provide the domain certificate in `certificate_secret_names` field. Tyk Operator will help you retrieve the certificate from secret and upload it to Tyk. + +```yaml{linenos=true, linenostart=1, hl_lines=["10-11"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin +spec: + name: httpbin + use_keyless: true + protocol: https + listen_port: 8443 + certificate_secret_names: + - my-test-tls + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true +``` + + + +You can also manage custom domain certificates using Kubernetes secrets in Tyk OAS. + +Example of Defining Custom Domain Certificates + +```yaml{linenos=true, linenostart=1, hl_lines=["50-51"]} expandable +# Secret is not created in this manifest. +# Please store custom domain certificate in a kubernetes TLS secret `custom-domain-secret`. +apiVersion: v1 +data: + test_oas.json: |- + { + "info": { + "title": "Petstore with custom domain", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "components": {}, + "paths": {}, + "x-tyk-api-gateway": { + "info": { + "name": "Petstore with custom domain", + "state": { + "active": true + } + }, + "upstream": { + "url": "https://petstore.swagger.io/v2" + }, + "server": { + "listenPath": { + "value": "/petstore/", + "strip": true + } + } + } + } +kind: ConfigMap +metadata: + name: cm + namespace: default +--- +apiVersion: tyk.tyk.io/v1alpha1 +kind: TykOasApiDefinition +metadata: + name: petstore-with-customdomain +spec: + tykOAS: + configmapRef: + name: cm + namespace: default + keyName: test_oas.json + customDomain: + enabled: true + name: "buraksekili.dev" + certificatesRef: + - custom-domain-secret +``` + +This example shows how to enable a custom domain (`buraksekili.dev`) with a TLS certificate stored in a Kubernetes secret (`custom-domain-secret`). + + + + + +## Certificate Management + +Tyk provides two options to manage certificates: plain files or certificate storage with a separate API. + +All configuration options, which require specifying certificates, support both plain file paths or certificate IDs. You are able to mix them up, and Tyk will automatically distinguish file names from certificate IDs. + +The Tyk Gateway and Dashboard Admin APIs provide endpoints to create, remove, list, and see information about certificates. For the Gateway, the endpoints are: + +* Create: `POST /tyk/certs` with PEM body. Returns `{"id": "", ... }` +* Delete: `DELETE /tyk/certs/` +* Get info: `GET /tyk/certs/`. Returns meta info about the certificate, something similar to: +```json expandable +{ + "id": "", + "fingerprint": , + "has_private_key": false, + "issuer": , + "subject": "", ... +} +``` +* Get info about multiple certificates: `GET /tyk/certs/,,`. +Returns array of meta info objects, similar to above. +* List all certificate IDs: `GET /tyk/certs`. Returns something similar to: + +```json +{ "certs": "", "", ... } +``` + +The Dashboard Admin API is very similar, except for a few minor differences: + +* Endpoints start with `/api` instead of `/tyk`, e.g. `/api/certs`, `/api/certs/`, etc. +* All certificates are managed in the context of the organization. In other words, certificates are not shared between organizations. + +Certificate storage uses a hex encoded certificate SHA256 fingerprint as its ID. When used with the Dashboard API, Tyk additionally appends the organization id to the certificate fingerprint. It means that certificate IDs are predictable, and you can check certificates by their IDs by manually +generating certificate SHA256 fingerprint using the following command: + +```{.copyWrapper} +openssl x509 -noout -fingerprint -sha256 -inform pem -in . +``` + +You may notice that you can't get the raw certificate back, only its meta information. This is to ensure security. Certificates with private keys have special treatment and are encoded before storing. If a private key is found it will be encrypted with AES256 algorithm 3 using the `security.private_certificate_encoding_secret` secret, defined in `tyk.conf` file. Otherwise, the certificate will use the [secret](/tyk-oss-gateway/configuration#secret) value in `tyk.conf`. + +### Using Tyk Certificate Storage + +In Tyk Gateway 2.4 and Tyk Dashboard 1.4 we added [Mutual TLS support](/basic-config-and-security/security/mutual-tls/client-mtls#why-use-mutual-tls) including special Certificate storage, which is used to store all kinds of certificates from public to server certificates with private keys. + +In order to add new server certificates to the Gateway: + +1. Ensure that both private key and certificates are in PEM format +2. Concatenate Cert and Key files to single file +3. Go to "Certificates" section of the Tyk Dashboard, upload certificate, and you will get a unique ID response +4. Set it to the Tyk Gateway using one of the approaches below: + + * Using your `tyk.conf`: + + ``` expandable + "http_server_options": { + "ssl_certificates": ["", ""] + } + ``` + + * Using environment variables (handy for Multi-Cloud installation and Docker in general): `TYK_GW_HTTPSERVEROPTIONS_SSLCERTIFICATES=` (if you want to set multiple certificates just separate them using a comma.) + + The Domain in this case will be extracted from standard certificate fields: `DNSNames`. + + + + + Prior to Tyk v5, the Domain could also be extracted from the now deprecated `Subject.CommonName` field. + + + +## Self Signed Certificates + +Self signed certificates can be managed in multiple ways. + +Best practice dictates that you store certificates in the standard certificate store on the local system, e.g. +`/etc/ssl/certs` + +For example, if you are using a self-signed cert on the Dashboard, in order for the Gateway to trust it, add it to the Gateway's certificate store in `/etc/ssl/certs` + +Alternatively, you can disable the verification of SSL certs in the component configurations below. **You shouln't do this in production!** + +### Creating a self-signed certificate pair +You can create self-signed client and server certificates with this command: +```{.copyWrapper} +openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes +``` + +For the server in `common name` specify a domain, or just pass `-subj "/CN=localhost"` to OpenSSL command. Then follow our [TLS and SSL Guide](/api-management/certificates). + +To get certificate SHA256 fingerprint use the following command: + +```{.copyWrapper} +openssl x509 -noout -fingerprint -sha256 -inform pem -in +``` + +If you are testing using cURL, your command will look like: + +```{.copyWrapper} +curl --cert client_cert.pem --key client_key.pem https://localhost:8181 +``` + +### Using self-signed certificates with Tyk Gateway + +You can set `http_server_options.ssl_insecure_skip_verify` to `true` in your tyk.conf to allow the use of self-signed certificates when connecting to the Gateway. + +### Using self-signed certificates with Tyk Dashboard + +You can set `http_server_options.ssl_insecure_skip_verify` to `true` in your tyk_analytics.conf to allow the use of self-signed certificates when connecting to the Dashboard. + + + +## Internal Proxy Setup + +From v2.9.3 you can also specify a custom proxy and set the minimum TLS versions and any SSL ciphers within your API definitions. See [Internal Proxy Setup](/api-management/gateway-config-tyk-classic#proxy-transport-settings) for more details. + + diff --git a/api-management/client-authentication.mdx b/api-management/client-authentication.mdx new file mode 100644 index 00000000..20fbe414 --- /dev/null +++ b/api-management/client-authentication.mdx @@ -0,0 +1,370 @@ +--- +title: "Client Authentication and Authorization" +'og:description': "Learn how to apply the most appropriate authentication method to secure access your APIs with Tyk. Here you will find everything there is to know about authenticating and authorizing API clients with Tyk." +sidebarTitle: "Overview" +tags: ['Authentication', 'Authorization', 'Tyk Authentication', 'Tyk Authorization', 'Secure APIs', 'client'] +--- + +import { ResponsiveGrid } from '/snippets/ResponsiveGrid.mdx'; + +## Introduction + +Tyk Gateway sits between your clients and your services, securely routing requests and responses. For each API proxy that you expose on Tyk, you can configure a range of different methods that clients must use to identify (authenticate) themselves to Tyk Gateway when making a request to access the API. + +*Authentication* and *Authorization* are the processes that you use to control access to your APIs and protect your upstream services. Each serves a distinct purpose: + +* **Authentication** (or **AuthN**) is the process of confirming the identity of the user or system making the API request. This step validates "who" is attempting to access the API, commonly using credentials such as tokens, passwords, or certificates. + +* **Authorization** (or **AuthZ**) is the process that determines if the user or system has the right permissions to perform the requested action. This step defines "what" they are allowed to do based on assigned roles, scopes, or policies. + +Whilst AuthN and AuthZ are separate actions with different standards, they are often considered together under the topic of *Securing the API*. Together, these processes allow API providers to control access, safeguard data integrity, and meet security and compliance standards, making them vital for any API management strategy. + +--- + +## How does Tyk Implement Authentication and Authorization? + +The API request processing flow within Tyk Gateway consists of a [chain of middleware](/api-management/traffic-transformation#request-middleware-chain) that perform different checks and transformations on the request (headers, parameters and payload). Several dedicated **authentication middleware** are provided and there is also support for user-provided **custom authentication plugins**. Multiple authentication middleware can be chained together if required by the API's access security needs. *Note that it is not possible to set the order of chained auth methods.* + +The OpenAPI description can contain a list of [securitySchemes](https://spec.openapis.org/oas/v3.0.3.html#security-scheme-object) which define the authentication methods to be used for the API; the detailed configuration of the Tyk authentication middleware is set in the [server.authentication](/api-management/gateway-config-tyk-oas#authentication) section of the Tyk Vendor Extension. + +You must enable client authentication using the `server.authentication.enabled` flag and then configure the appropriate authentication method as indicated in the relevant section of this document. When creating a Tyk OAS API from an OpenAPI description, Tyk can automatically enable authentication based upon the content of the OpenAPI description as described [here](/api-management/gateway-config-managing-oas#importing-an-openapi-description-to-create-an-api). + +When using Tyk Classic APIs, each authentication middleware has its own fields within the API definition + +### Managing authorization data + +The data that the client provides with the API request used to authenticate with Tyk and confirm that it is authorized to access the API is often of no use to the upstream service and, depending on your security governance, may even be prohibited from being made available to the upstream. + +Tyk offers a simple option, separately configurable for each API to remove, or "strip", the authentication/authorization date from the incoming request before proxying to the upstream. + +This is controlled using the [server.authentication.stripAuthorizationData](/api-management/gateway-config-tyk-oas#authentication) field in the Tyk Vendor Extension (Tyk Classic: `strip_auth_data`). + +## What does Tyk Support? + +Tyk includes support for various industry-standard methods to secure your APIs. This page provides an overview of the options available, helping you to choose and implement what works best for you. + +Use Ctrl+F or the sidebar to find specific topics, for example β€œJWT” for JSON Web Tokens or β€œmTLS” for mutual TLS. + +You can also use the links below to jump directly to the appropriate sections to learn how to secure your APIs using Tyk. + + + + +Delegate authentication using one of the most widely used open standard protocols + + + +Securely transmit information between parties. + + + +Secure APIs with username and password credentials. + + + +Implement token-based authentication for API access. + + + +Establish secure channels with two-way certificate verification. + + + +Verify message integrity using shared secret keys. + + +{/* To be added + +Verify message integrity using shared secret certificates. + */} + + +Create custom plugins to implement specific authentication requirements. + + + +Allow unrestricted access for public APIs. + + + + + +--- + +## Other Authentication Methods + +### Integrate with External Authorization Server (deprecated) + + + +Tyk has previously offered two types of OAuth authentication flow; [Tyk as the authorization server]() and Tyk connecting to an external *auth server* via a dedicated *External OAuth* option. The dedicated external *auth server* option was deprecated in Tyk 5.7.0. +
+ +For third-party OAuth integration we recommend using the JSON Web Token (JWT) middleware which is described [above](/basic-config-and-security/security/authentication-authorization/json-web-tokens), which offers the same functionality with a more streamlined setup and reduced risk of misconfiguration. +
+ +The remainder of this section is left for reference and is not maintained. + + + +To call an API that is protected by OAuth, you need to have an access token from the third party IDP (it could be an opaque token or a JWT). + +For subsequent calls the access token is provided alongside the API call and needs to be validated. With JWT, Tyk can confirm the validity of the JWT with the secret provided in your config. The secret signs the JWT when created and confirms that none of its contents has changed. + +For this reason, information like the expiry date which are often set within the JWT cannot be changed after the JWT has been initially created and signed. This means you are not able to revoke a token before the expiry set in the JWT with the standard JWT flow. With OAuth you can use [OAuth introspection](https://www.rfc-editor.org/rfc/rfc7662) to overcome this. With introspection, you can validate the access token via an introspection endpoint that validates the token. + +Let’s see how external OAuth middleware is configured. + +#### OAS contract + +```yaml expandable +externalOAuthServer: + enabled: true, + providers: # only one item in the array for now (we're going to support just one IDP config in the first iteration) + - jwt: #validate JWTs generated by 3rd party Oauth servers (like Okta) + enabled: true + signingMethod: HMAC/RSA/ECDSA # to verify signing method used in jwt + source: key # secret to verify signature + issuedAtValidationSkew: 0 + notBeforeValidationSkew: 0 + expiresAtValidationSkew: 0 + identityBaseField: # identity claimName + introspection: # array for introspection details + enabled: true/false + clientID: # for introspection request + clientSecret: # for introspection request, if empty will use oAuth.secret + url: # token introspection endpoint + cache: # Tyk will cache the introspection response when `cache.enabled` is set to `true` + enabled: true/false, + timeout: 0 # The duration (in seconds) for which Tyk will retain the introspection outcome in its cache. If the value is "0", it indicates that the introspection outcome will be stored in the cache until the token's expiration. + identityBaseField: # identity claimName +``` + +#### Tyk Classic API definition contract + +```yaml expandable +"external_oauth": { + "enabled": true, + "providers": [ + { + "jwt": { + "enabled": false, + "signing_method": rsa/ecdsa/hmac, + "source": # jwk url/ base64 encoded static secret / base64 encoded jwk url + "identity_base_field": # identity claim name + "expires_at_validation_skew": # validation skew config for exp + "not_before_validation_skew": # validation skew config for nbf + "issued_at_validation_skew" : # validation skew config for iat + }, + "introspection": { + "enabled": true, + "url": # introspection endpoint url + "client_id": # client Id used for introspection + "client_secret": # client secret to be filled here (plain text for now, TODO: decide on a more secure mechanism) + "identity_base_field": # identity claim name + "cache": { + "enabled": true, + "timeout": # timeout in seconds + } + } + } + ] +} +``` +- `externalOAuthServer` set `enabled` to `true` to enable the middleware. +- `providers` is an array of multiple IDP configurations, with each IDP config being an element in the `providers` array. +- You can use this config to use JWT self validation using `jwt` or use introspection via `instropection` in the `providers` section . + + + + + For now, you’ll be limiting `providers` to have only one element, ie one IDP configured. + + + +#### JWT + +There could be cases when you don’t need to introspect a JWT access token from a third party IDP, and instead you can just validate the JWT. This is similar to existing JWT middleware, adding it in External OAuth middleware for semantic reasons. + +- `enabled` - enables JWT validation. +- `signingMethod` - specifies the signing method used to sign the JWT. +- `source` - the secret source, it can be one of: + - a base64 encoded static secret + - a valid JWK url in plain text + - a valid JWK url in base64 encoded format +- `issuedAtValidationSkew` , `notBeforeValidationSkew`, `expiresAtValidationSkew` can be used to [configure clock skew](/basic-config-and-security/security/authentication-authorization/json-web-tokens#adjust-jwt-clock-skew-configuration) for json web token validation. +- `identityBaseField` - the identity key name for claims. If empty it will default to `sub`. + +##### Example: Tyk OAS API definition with JWT validation enabled + +```json expandable +"securitySchemes": { + "external_jwt": { + "enabled": true, + "header": { + "enabled": true, + "name": "Authorization" + }, + "providers": [ + { + "jwt": { + "enabled": true, + "signingMethod": "hmac", + "source": "dHlrLTEyMw==", + "identityBaseField": "sub" + } + } + ] + } +} +``` + +##### Example: Tyk Classic API definition with JWT validation enabled + +```json expandable +"external_oauth": { + "enabled": true, + "providers": [ + { + "jwt": { + "enabled": true, + "signing_method": "hmac", + "source": "dHlrLTEyMw==", + "issued_at_validation_skew": 0, + "not_before_validation_skew": 0, + "expires_at_validation_skew": 0, + "identity_base_field": "sub" + }, + "introspection": { + "enabled": false, + "url": "", + "client_id": "", + "client_secret": "", + "identity_base_field": "", + "cache": { + "enabled": false, + "timeout": 0 + } + } + } + ] +} +``` +#### Introspection + +For cases where you need to introspect the OAuth access token, Tyk uses the information in the `provider.introspection` section of the contract. This makes a network call to the configured introspection endpoint with the provided `clientID` and `clientSecret` to introspect the access token. + +- `enabled` - enables OAuth introspection +- `clientID` - clientID used for OAuth introspection, available from IDP +- `clientSecret` - secret used to authenticate introspection call, available from IDP +- `url` - endpoint URL to make the introspection call +- `identityBaseField` - the identity key name for claims. If empty it will default to `sub`. + +##### Caching + +Introspection via a third party IdP is a network call. Sometimes it may be inefficient to call the introspection endpoint every time an API is called. Caching is the solution for this situation. Tyk caches the introspection response when `enabled` is set to `true` inside the `cache` configuration of `introspection`. Then it retrieves the value from the cache until the `timeout` value finishes. However, there is a trade-off here. When the timeout is long, it may result in accessing the upstream with a revoked access token. When it is short, the cache is not used as much resulting in more network calls. + +The recommended way to handle this balance is to never set the `timeout` value beyond the expiration time of the token, which would have been returned in the `exp` parameter of the introspection response. + +See the example introspection cache configuration: + +```yaml expandable +"introspection": { + ... + "cache": { + "enabled": true, + "timeout": 60 // in seconds + } +} +``` +##### Example: Tyk OAS API definition external OAuth introspection enabled + +```json expandable +"securitySchemes": { + "keycloak_oauth": { + "enabled": true, + "header": { + "enabled": true, + "name": "Authorization" + }, + "providers": [ + { + "introspection": { + "enabled": true, + "url": "http://localhost:8080/realms/tyk/protocol/openid-connect/token/introspect", + "clientId": "introspection-client", + "clientSecret": "DKyFN0WXu7IXWzR05QZOnnSnK8uAAZ3U", + "identityBaseField": "sub", + "cache": { + "enabled": true, + "timeout": 3 + } + } + } + ] + } +} +``` +##### Example: Tyk Classic API definition with external OAuth introspection enabled + +```json expandable +"external_oauth": { + "enabled": true, + "providers": [ + { + "jwt": { + "enabled": false, + "signing_method": "", + "source": "", + "issued_at_validation_skew": 0, + "not_before_validation_skew": 0, + "expires_at_validation_skew": 0, + "identity_base_field": "" + }, + "introspection": { + "enabled": true, + "url": "http://localhost:8080/realms/tyk/protocol/openid-connect/token/introspect", + "client_id": "introspection-client", + "client_secret": "DKyFN0WXu7IXWzR05QZOnnSnK8uAAZ3U", + "identity_base_field": "sub", + "cache": { + "enabled": true, + "timeout": 3 + } + } + } + ] +} +``` + +### Integrate with OpenID Connect (deprecated) + + + +Tyk has previously offered a dedicated OpenID Connect option for client authentication, but this was not straightforward to use and was deprecated in Tyk 5.7.0. +
+ +For integration with a third-party OIDC provider we recommend using the JSON Web Token (JWT) middleware which is described [above](/basic-config-and-security/security/authentication-authorization/json-web-tokens), which offers the same functionality with a more streamlined setup and reduced risk of misconfiguration. +
+ +The remainder of this section is left for reference and is not maintained. + + + + +[OpenID Connect](https://openid.net/developers/how-connect-works) (OIDC) builds on top of OAuth 2.0, adding authentication. You can secure your APIs on Tyk by integrating with any standards compliant OIDC provider using [JSON Web Tokens](/basic-config-and-security/security/authentication-authorization/json-web-tokens) (JWTs). +JWTs offer a simple way to use the third-party Identity Provider (IdP) without needing any direct integration between the Tyk and 3rd-party systems. + +To integrate a 3rd party OAuth2/OIDC IdP with Tyk, all you will need to do is ensure that your IdP can issue OAuth2 JWT access tokens as opposed to opaque tokens. + +The client application authenticates with the IdP which then provides an access token that is accepted by Tyk. Tyk will take care of the rest, ensuring that the rate limits and quotas of the underlying identity of the bearer are maintained across JWT token re-issues, so long as the "sub" (or whichever identity claim you chose to use) is available and consistent throughout and the policy that underpins the security clearance of the token exists too. + + + +## Conclusion + +Securing your APIs is a foundational step toward managing data integrity and access control effectively. Now that you've configured authentication and authorization, the next steps in your API journey with Tyk should involve: + +Defining Access Policies: Use Tyk’s policies to refine API access controls, rate limits, and quotas. This lets you align your security model with business needs and enhance user experience through granular permissions. You can learn more about policies [here](/api-management/policies). + +Exploring API Analytics: Leverage Tyk’s analytics to monitor access patterns, track usage, and gain insights into potential security risks or high-demand endpoints. Understanding usage data can help in optimizing API performance and enhancing security measures. You can learn more about analytics [here](/tyk-dashboard-analytics). \ No newline at end of file diff --git a/api-management/dashboard-configuration.mdx b/api-management/dashboard-configuration.mdx new file mode 100644 index 00000000..e30f3048 --- /dev/null +++ b/api-management/dashboard-configuration.mdx @@ -0,0 +1,5514 @@ +--- +title: "Manage the Tyk Dashboard" +'og:description': "How to manage users, teams, permissions, RBAC in Tyk Dashboard" +tags: ['Manage Tyk Dashboard', 'User Management', 'RBAC', 'Role Based Access Control', 'User Groups', 'Teams', 'Permissions', 'API Ownership', 'SSO', 'Single Sign On', 'Multi Tenancy'] +sidebarTitle: "Manage Tyk Dashboard" +--- + +import MongodbVersionsInclude from '/snippets/mongodb-versions-include.mdx'; +import SqlVersionsInclude from '/snippets/sql-versions-include.mdx'; +import { ButtonLeft } from '/snippets/ButtonLeft.mdx'; + +## Introduction + +Tyk Dashboard diagram + +The Tyk Dashboard is a powerful web-based interface that serves as the **central management hub for your API ecosystem**. It provides a user-friendly Graphical User Interface (GUI) for configuring, monitoring, and analyzing your APIs managed by Tyk. + +The Dashboard also exposes a **REST API**, allowing for programmatic control and integration with other tools and workflows. + +This page introduces general features of dashboard and how to configure them. If you are looking for global configurations of the Dashboard deployment refer this [config file](/tyk-dashboard/configuration). + +We will delve into the following key topics: + +1. **[Exploring the Dasbhoard UI](#exploring-the-dashboard-ui)**: A tour of the Dashboard UI. + +2. **[Exploring the Dasbhoard API](#exploring-the-dashboard-api)**: Explore the Dashboard APIs, including their classification, authentication methods, and usage examples with Swagger and Postman collections. + +3. **[API Management using API Endpoint Designer](#exploring-api-endpoint-designer)**: A graphical environment for configuring your Tyk APIs. + +4. **[Monitoring and Traffic Analytics](#traffic-analytics)**: Exploration of Tyk's traffic analytics capabilities, including logging mechanisms, error tracking, endpoint analysis, and various activity type measurements. + +5. **[API Governance using API Templates and API Categories](#governance-using-api-categories)** + +6. **[System Management](#system-administration)**: Detailed overview of Tyk's system management capabilities, including Admin API functionalities, organization management and configuting audit logs. + +7. **[Supported Database](#supported-database)**: We will examine Dashboard's storage requirement, compatible database versions and how to configure them. + +7. **[Exploring Data Storage Solution](#data-storage-solutions)**: We will explore Dashboard's multi-layered storage architecture and understand how to configure each storage tier effectively. + +## Exploring the Dashboard UI + +To get a tour of the Dashboard UI, refer to this [document](/getting-started/using-tyk-dashboard). + +## Exploring the Dashboard API + +The Dashboard is a large, granular REST API with a thin-client web front-end, and if being deployed as part of a Tyk install, serves as the main integration point instead of the Gateway API. + +API Overview + +**The Dashboard API is a superset of the Gateway API**, providing the same functionality, with additional features (anything that can be done in the Dashboard has an API endpoint), and offers some additional advantages: + - The Dashboard API has a granular structure, you can create separate clients easily. + - The API features read/write permissions on a per-endpoint level to have extra control over integrations. + - The API enforces a schema that can be modified and hardened depending on your usage requirements. + +### Types of Dashboard API + +The Dashboard exposes two APIs: + - **Dashboard API**: Is used for operational management of Tyk resources (APIs, policies, keys, etc.). This API offers granular permissions based on user roles. + + To know more about Dashboard APIs, refer the following documents: + - [Postman / Swagger / Open API specification](/tyk-dashboard-api) + - [Dashboard API Usage Examples](#dashboard-api-resources-and-usage) + + - **Dashboard Admin API**: Is used for system-level administration and initial setup tasks like managing organizations, initial user creation, backups/migrations and SSO setup. + + To know more about Dashboard Admin APIs, refer the following documents: + - [Postman / Swagger / Open API specification](/dashboard-admin-api) + - [Dashboard Admin API Usage Examples](#dashboard-admin-api-resources-and-usage) + +### Authenticating with Dashboard APIs + +**Dashboard API** + +The [Tyk Dashboard API](/tyk-dashboard-api) is secured using an `Authorization` header that must be added to each request that is made. The **Tyk Dashboard API Access Credentials** `Authorization` key can be found within the Dashboard UI at the bottom of the **Edit User** section for a user. + +**Dashboard Admin API** + +The Tyk Dashboard Admin API is secured using a shared secret that is set in the `tyk_analytics.conf` file. Calls to the Admin API require the `admin-auth` header to be provided, to differentiate the call from a regular Dashboard API call. + +## Dashboard API Resources and Usage + +### Overview + +The [Tyk Dashboard API](/tyk-dashboard-api) is a superset of the Tyk Gateway API, enabling (almost) all of the core features and adding many more. The Dashboard API is also more granular and supports [Role Based Access Control](/api-management/user-management#) (RBAC) on both a multi-tenant, and user basis. + +Using the Dashboard API it is possible to set Read / Write / ReadWrite / Deny access to sections of the API on a user by user basis, and also segregate User / Key / API Ownership by organization. + +The availability of RBAC varies depending on the license or subscription. For further information, please check our [price comparison](https://tyk.io/price-comparison/) or consult our sales and expert engineers + + + +For optimal results, it is advisable to exclusively employ the Tyk Dashboard API (avoiding direct calls to the Tyk Gateway API) within a Self-Managed setup, enabling the Dashboard to manage the Tyk API gateways cluster. + + + + +Tyk Dashboard API security + +### Pagination + +Selected Dashboard APIs can be paginated. + +You can select the number of result pages to return by adding a parameter `p` which starts at `1`. At the default page size, this returns items 1-10. Setting `p` to `2` returns items 11-20 and so on. Alternatively, passing `0` or lower as a parameter will return all items. + +The default page size is 10. You can overwrite the default page size in your `tyk_analytics.conf` using the `page_size` key. It's suggested you do not modify it as it will affect the performance of the Dashboard. + +**Sample Request:** + +```http +GET /api/apis/?p=1 HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response:** + +```yaml expandable +{ + "apis": [ + { ... }, + { ... }, + { ... } + ], + "pages": 1 +} +``` + +

Manage APIs - API Definition

+ + +See [API Definition Objects](/api-management/gateway-config-tyk-classic) section for an explanation of each field in the request & response. + + + +#### Get List of APIs + +| **Property** | **Description** | +| :------------ | :--------------- | +| Resource URL | `/api/apis/` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +GET /api/apis?p=0 HTTP/1.1 +Host: localhost:3000 +authorization: 7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response** + +```yaml expandable +{ + "apis": [ + { + "api_model": {}, + "api_definition": { + "id": "54b53e47eba6db5c70000002", + "name": "Nitrous Test", + "api_id": "39d2c98be05c424371c600bd8b3e2242", + "org_id": "54b53d3aeba6db5c35000002", + "use_keyless": false, + "use_oauth2": false, + "oauth_meta": { + "allowed_access_types": [], + "allowed_authorize_types": [ + "token" + ], + "auth_login_redirect": "" + }, + "auth": { + "auth_header_name": "authorization" + }, + "use_basic_auth": false, + "notifications": { + "shared_secret": "", + "oauth_on_keychange_url": "" + }, + "enable_signature_checking": false, + "definition": { + "location": "header", + "key": "" + }, + "version_data": { + "not_versioned": true, + "versions": { + "Default": { + "name": "Default", + "expires": "", + "paths": { + "ignored": [], + "white_list": [], + "black_list": [] + }, + "use_extended_paths": false, + "extended_paths": { + "ignored": [], + "white_list": [], + "black_list": [] + } + } + } + }, + "proxy": { + "listen_path": "/39d2c98be05c424371c600bd8b3e2242/", + "target_url": "http://tyk.io", + "strip_listen_path": true + }, + "custom_middleware": { + "pre": null, + "post": null + }, + "session_lifetime": 0, + "active": true, + "auth_provider": { + "name": "", + "storage_engine": "", + "meta": null + }, + "session_provider": { + "name": "", + "storage_engine": "", + "meta": null + }, + "event_handlers": { + "events": {} + }, + "enable_batch_request_support": false, + "enable_ip_whitelisting": false, + "allowed_ips": [], + "expire_analytics_after": 0 + }, + "hook_references": [] + } + ... + ], + "pages": 0 +} +``` + +#### Search APIs by name + +| **Property** | **Description** | +| :------------ | :------------------ | +| Resource URL | `/api/apis/search` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +GET /api/apis?q=Some+Name HTTP/1.1 +Host: localhost:3000 +authorization: 7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response** + +Similar to API list response + +#### Retrieve a single API by ID + +| **Property** | **Description** | +| :------------ | :---------------- | +| Resource URL | `/api/apis/{id}` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + + + +`{id}` can either be the internal or public ID ( see `api_id` in the sample response ) + + + +**Sample request** + +```http +GET /api/apis/54c24242eba6db1c9a000002 HTTP/1.1 +Host: localhost +authorization: 7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response** + +```json expandable +{ + "api_model": {}, + "api_definition": { + "id": "54c24242eba6db1c9a000002", + "name": "Test", + "api_id": "bc2f8cfb7ab241504d9f3574fe407499", + "org_id": "54b53d3aeba6db5c35000002", + "use_keyless": false, + "use_oauth2": false, + "oauth_meta": { + "allowed_access_types": [], + "allowed_authorize_types": [ + "token" + ], + "auth_login_redirect": "" + }, + "auth": { + "auth_header_name": "authorization" + }, + "use_basic_auth": false, + "notifications": { + "shared_secret": "", + "oauth_on_keychange_url": "" + }, + "enable_signature_checking": false, + "definition": { + "location": "header", + "key": "" + }, + "version_data": { + "not_versioned": true, + "versions": { + "Default": { + "name": "Default", + "expires": "", + "paths": { + "ignored": [], + "white_list": [], + "black_list": [] + }, + "use_extended_paths": true, + "extended_paths": { + "ignored": [ + { + "path": "/test-path/", + "method_actions": { + "GET": { + "action": "no_action", + "code": 200, + "data": "", + "headers": {} + } + } + }, + { + "path": "/test-path/reply", + "method_actions": { + "GET": { + "action": "reply", + "code": 200, + "data": "{\"foo\":\"bar\"}", + "headers": { + "x-test": "test" + } + } + } + } + ], + "white_list": [], + "black_list": [] + } + } + } + }, + "proxy": { + "listen_path": "/bc2f8cfb7ab241504d9f3574fe407499/", + "target_url": "http://httpbin.org/", + "strip_listen_path": true + }, + "custom_middleware": { + "pre": [], + "post": [] + }, + "session_lifetime": 0, + "active": true, + "auth_provider": { + "name": "", + "storage_engine": "", + "meta": null + }, + "session_provider": { + "name": "", + "storage_engine": "", + "meta": null + }, + "event_handlers": { + "events": { + "QuotaExceeded": [ + { + "handler_name": "eh_web_hook_handler", + "handler_meta": { + "_id": "54be6c0beba6db07a6000002", + "event_timeout": 60, + "header_map": { + "x-tyk-test": "123456" + }, + "method": "POST", + "name": "Test Post", + "org_id": "54b53d3aeba6db5c35000002", + "target_path": "http://httpbin.org/post", + "template_path": "" + } + } + ] + } + }, + "enable_batch_request_support": true, + "enable_ip_whitelisting": true, + "allowed_ips": [ + "127.0.0.1" + ], + "expire_analytics_after": 0 + }, + "hook_references": [ + { + "event_name": "QuotaExceeded", + "event_timeout": 60, + "hook": { + "api_model": {}, + "id": "54be6c0beba6db07a6000002", + "org_id": "54b53d3aeba6db5c35000002", + "name": "Test Post", + "method": "POST", + "target_path": "http://httpbin.org/post", + "template_path": "", + "header_map": { + "x-tyk-test": "123456" + }, + "event_timeout": 0 + } + } + ] +} +``` + + +#### Delete API by ID + +**Sample Request** + +```http +DELETE /api/apis/54c24242eba6db1c9a000002 HTTP/1.1 +Host: localhost +Authorization: 7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response** + +```json +{ + "Status":"OK", + "Message":"API deleted", + "Meta":null +} +``` + +#### Create API Definition + +Creating API definitions is slightly different to the core API, API definitions are wrapped inside an `api_definition` field and event handlers, such as webhooks are not embedded in the main `api_defintion` object (though they can be), webhooks are instead appended as references into the `hook_references` field, the API will embed the correct webhook data into the event handler interface. + +Please note that ID's (both `id` and `api_id`) are auto-generated by Tyk and cannot be set by the user. In Self-Managed installations `api_id` can be overwritten with a call to the Update API Definition endpoint, but this is currently not possible when the Dashboard resides in Tyk Cloud. + +| **Property** | **Description** | +| :------------ | :----------------------- | +| Resource URL | `/api/apis/` | +| Method | POST | +| Type | None | +| Body | Advanced API Definition | +| Param | None | + +**Sample Request** + +```http expandable +POST /api/apis HTTP/1.1 +Host: localhost:3000 +Connection: keep-alive +Content-Type: application/json +Content-Length: 1356 +authorization: 7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "api_definition": { + "name": "Test", + "auth": { + "auth_header_name": "authorization" + }, + "definition": { + "location": "header", + "key": "" + }, + "proxy": { + "target_url": "http://httpbin.org/" + }, + "version_data": { + "use_extended_paths": true, + "not_versioned": true, + "versions": { + "Default": { + "expires": "", + "name": "Default", + "paths": { + "ignored": [], + "white_list": [], + "black_list": [] + }, + "extended_paths": { + "ignored": [ + { + "path": "/test-path/", + "method_actions": { + "GET": { + "action": "no_action", + "code": 200, + "data": "", + "headers": {} + } + } + }, + { + "path": "/test-path/reply", + "method_actions": { + "GET": { + "action": "reply", + "code": 200, + "data": "{\"foo\":\"bar\"}", + "headers": { + "x-test": "test" + } + } + } + } + ], + "white_list": [], + "black_list": [] + }, + "use_extended_paths": true + } + } + }, + "use_oauth2": false, + "oauth_meta": { + "auth_login_redirect": "", + "allowed_access_types": [], + "allowed_authorize_types": [ + "token" + ] + }, + "notifications": { + "shared_secret": "", + "oauth_on_keychange_url": "" + }, + "enable_ip_whitelisting": true, + "allowed_ips": [ + "127.0.0.1" + ], + "use_keyless": false, + "enable_signature_checking": false, + "use_basic_auth": false, + "active": true, + "enable_batch_request_support": true + }, + "hook_references": [ + { + "event_name": "QuotaExceeded", + "hook": { + "api_model": {}, + "id": "54be6c0beba6db07a6000002", + "org_id": "54b53d3aeba6db5c35000002", + "name": "Test Post", + "method": "POST", + "target_path": "http://httpbin.org/post", + "template_path": "", + "header_map": { + "x-tyk-test": "123456" + }, + "event_timeout": 0 + }, + "event_timeout": 60 + } + ] +} +``` + +**Sample Response** + +```json +{ + "Status": "OK", + "Message": "API created", + "Meta": "54c24242eba6db1c9a000002" +} +``` + +Please note that Tyk matches the Ignored paths in the order in which they are specified in the `ignored` array. Subpaths of a route are matched automatically and so should be placed above parent routes if they need to be matched individually. + +#### Update API Definition + +APIs that are created using the advanced Dashboard API are referenced by their internal ID instead of their API-ID. + +Please note that whilst `api_id` can be updated for Self-Managed installations, this is currently not possible when the Dashboard resides in Tyk Cloud. Updates to `api_id` in Tyk Cloud will be ignored. + +| **Property** | **Description** | +| :------------ | :------------------------------------- | +| Resource URL | `/api/apis/{internal_or_external_id}` | +| Method | PUT | +| Type | None | +| Body | Advanced API Definition | +| Param | None | + +**Sample Request** + +```http expandable +PUT /api/apis/54c24242eba6db1c9a000002 HTTP/1.1 +Host: localhost:3000 +Connection: keep-alive +Content-Type: application/json +Content-Length: 1356 +authorization: 7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "api_definition": { + "id": "54c24242eba6db1c9a000002", + "api_id": "bc2f8cfb7ab241504d9f3574fe407499", + "name": "Test", + "auth": { + "auth_header_name": "authorization" + }, + "definition": { + "location": "header", + "key": "" + }, + "proxy": { + "target_url": "http://httpbin.org/" + }, + "version_data": { + "use_extended_paths": true, + "not_versioned": true, + "versions": { + "Default": { + "expires": "", + "name": "Default", + "paths": { + "ignored": [], + "white_list": [], + "black_list": [] + }, + "extended_paths": { + "ignored": [ + { + "path": "/test-path/", + "method_actions": { + "GET": { + "action": "no_action", + "code": 200, + "data": "", + "headers": {} + } + } + }, + { + "path": "/test-path/reply", + "method_actions": { + "GET": { + "action": "reply", + "code": 200, + "data": "{\"foo\":\"bar\"}", + "headers": { + "x-test": "test" + } + } + } + } + ], + "white_list": [], + "black_list": [] + }, + "use_extended_paths": true + } + } + }, + "use_oauth2": false, + "oauth_meta": { + "auth_login_redirect": "", + "allowed_access_types": [], + "allowed_authorize_types": [ + "token" + ] + }, + "notifications": { + "shared_secret": "", + "oauth_on_keychange_url": "" + }, + "enable_ip_whitelisting": true, + "allowed_ips": [ + "127.0.0.1" + ], + "use_keyless": false, + "enable_signature_checking": false, + "use_basic_auth": false, + "active": true, + "enable_batch_request_support": true + }, + "hook_references": [ + { + "event_name": "QuotaExceeded", + "hook": { + "api_model": {}, + "id": "54be6c0beba6db07a6000002", + "org_id": "54b53d3aeba6db5c35000002", + "name": "Test Post", + "method": "POST", + "target_path": "http://httpbin.org/post", + "template_path": "", + "header_map": { + "x-tyk-test": "123456" + }, + "event_timeout": 0 + }, + "event_timeout": 60 + } + ] +} +``` + +**Sample Response** + +```json +{ + "Status": "OK", + "Message": "Api updated", + "Meta": "" +} +``` + +### Data Graphs API + +Currently `/api/data-graphs/` has only one endpoint called `/data-sources` with only a `POST` HTTP method. + +The Dashboard exposes the `/api/data-graphs/data-sources/import` endpoint which allows you to import an [AsyncAPI](https://www.asyncapi.com/docs/reference/specification/v3.0.0) or [OpenAPI](https://swagger.io/specification/) document. + +#### Supported AsyncAPI versions +* 2.0.0 +* 2.1.0 +* 2.2.0 +* 2.3.0 +* 2.4.0 + +#### Supported OpenAPI versions +* 3.0.0 + +#### Import a document from a remote resource + +| **Property** | **Description** | +| :-------------- | :-------------------------------------------- | +| Resource URL | `/api/data-graphs/data-sources/import` | +| Method | `POST` | +| Content-Type | `application/json` | +| Body | `{`
` "url": "resource URL" `
`}` | + +The fetched document can be an OpenAPI or AsyncAPI document. The format will be detected automatically. The data source import API only checks the fetched data and tries to determine the document format, the status codes are ignored. +It returns an error if it fails to determine the format and the document type. HTTP 500 is returned if a programming or network error occurs. If the fetched request body is malformed then HTTP 400 is returned. + +#### Import an OpenAPI document + +The data source import API supports importing OpenAPI documents. The document can be used as a request body. + +| **Property** | **Description** | +| :-------------- | :------------------------------------------- | +| Resource URL | `/api/data-graphs/data-sources/import` | +| Method | `POST` | +| Content-Type | `application/vnd.tyk.udg.v2.openapi` | +| Body | `` | + + +The document can be in JSON or YAML format. The import API can determine the type and parse it. + +#### Import an AsyncAPI document + +The data source import API supports importing AsyncAPI documents. The document can be used as a request body. + +| **Property** | **Description** | +| :-------------- | :---------------------------------------- | +| Resource URL | `/api/data-graphs/data-sources/import` | +| Method | `POST` | +| Content-Type | `application/vnd.tyk.udg.v2.asyncapi` | +| Body | `` | + +The document can be in JSON or YAML format. The import API can determine the type and parse it. + +#### Response Structure + +The response structure is consistent with other endpoints, as shown in the table below: + +| **Property** | **Description** | +| :-------------- | :------------------------------------------------------- | +| Status | `Error` or `OK` | +| Message | Verbal explanation | +| Meta | API ID for success and `null` with error (not in use) | + +**Sample Response** + +```json +{ + "Status": "OK", + "Message": "Data source imported", + "Meta": "64102568f2c734bd2c0b8f99" +} +``` + +### Analytics API + + + +Below APIs returns data only if you have Pump 1.7.0 + + + +#### Analytics of API Key +| **Property** | **Description** | +| :------------ | :--------------- | +| Resource URL | `/api/activity/keys/endpoint/{keyHash}/{startDay}/{startMonth}/{startYear}/{EndDay}/{EndMonth}/{EndYear}` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +It returns analytics of the endpoints of all APIs called using KEY between start and end date. + +**Sample Request** +To get analytics of all endpoints called using the key `7f3c3ca87376cabe` between October 13th 2020 and October 14th 2020, make the following call: + +```http +GET api/activity/keys/endpoint/7f3c3ca87376cabe/13/10/2020/14/10/2020 HTTP/1.1 +Host: localhost:3000 +authorization: 7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response** + +```json expandable +{ + "data": [ + { + "id": { + "day": 0, + "month": 0, + "year": 0, + "hour": 0, + "code": 0, + "path": "/anything", + "key": "", + "alias": "", + "url": "", + "iso_country": "", + "api_id": "41351a6a94094da05f75146a695a16f6", + "api_name": "" + }, + "hits": 1, + "success": 1, + "error": 0, + "last_hit": "2020-10-13T13:22:49.667+05:30", + "request_time": 0, + "latency": 217, + "upstream_latency": 217, + "max_upstream_latency": 217, + "min_upstream_latency": 217, + "max_latency": 217, + "min_latency": 217 + }, + { + "id": { + "day": 0, + "month": 0, + "year": 0, + "hour": 0, + "code": 0, + "path": "/anything", + "key": "", + "alias": "", + "url": "", + "iso_country": "", + "api_id": "1793db2cbb724ad54da582ce3191d383", + "api_name": "" + }, + "hits": 1, + "success": 1, + "error": 0, + "last_hit": "2020-10-13T13:22:20.534+05:30", + "request_time": 568, + "latency": 568, + "upstream_latency": 568, + "max_upstream_latency": 568, + "min_upstream_latency": 568, + "max_latency": 568, + "min_latency": 568 + }, + ], + "pages": 1 +} +``` + + +#### Analytics of OAuth Client +| **Property** | **Description** | +| :------------ | :--------------- | +| Resource URL | `/api/activity/oauthid/endpoint/{OAuthClientID}/{startDay}/{startMonth}/{startYear}/{EndDay}/{EndMonth}/{EndYear}` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +It returns analytics of the all endpoints called using the given OAuth Client ID. + +**Sample Request** +To get activity of all endpoints which used OAuth client `27b35a9ed46e429eb2361e440cc4005c` between October 13th 2020 and October 14th 2020, make the following call: + +```http +GET /api/activity/oauthid/endpoint/27b35a9ed46e429eb2361e440cc4005c/13/10/2020/14/10/2020 HTTP/1.1 +Host: localhost:3000 +authorization: 7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response** +``` expandable +{ + "data": [ + { + "id": { + "day": 0, + "month": 0, + "year": 0, + "hour": 0, + "code": 0, + "path": "/get", + "key": "", + "alias": "", + "url": "", + "iso_country": "", + "api_id": "79fc7cb80df940cc5089772200bd4926", + "api_name": "" + }, + "hits": 2, + "success": 1, + "error": 1, + "last_hit": "2020-10-13T14:48:51.582+05:30", + "request_time": 498, + "latency": 498, + "upstream_latency": 497.5, + "max_upstream_latency": 747, + "min_upstream_latency": 248, + "max_latency": 748, + "min_latency": 248 + }, + { + "id": { + "day": 0, + "month": 0, + "year": 0, + "hour": 0, + "code": 0, + "path": "/post", + "key": "", + "alias": "", + "url": "", + "iso_country": "", + "api_id": "79fc7cb80df940cc5089772200bd4926", + "api_name": "" + }, + "hits": 1, + "success": 1, + "error": 0, + "last_hit": "2020-10-13T14:49:31.541+05:30", + "request_time": 0, + "latency": 241, + "upstream_latency": 239, + "max_upstream_latency": 239, + "min_upstream_latency": 239, + "max_latency": 241, + "min_latency": 241 + } + ], + "pages": 1 +} +``` + +### Users API + + + +`USER_ID` is a placeholder for your User ID value. + + + + +#### List Users + +| **Property** | **Description** | +| :------------ | :--------------- | +| Resource URL | `/api/users` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +GET /api/users HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response** + +``` expandable +{ + "users": [ + { + "api_model": {}, + "first_name": "John", + "last_name": "Smith", + "email_address": "john@jive.ly", + "password": "$2a$10$mRVfrAf72N66anVNhA1KVuYaOwOrXhFzxyg6bwgZemUeVo2MNOpIa", + "org_id": "54b53d3aeba6db5c35000002", + "active": true, + "id": "54b53d4bf25b920f09361526", + "access_key": "0cf5e6c37add465a406f19807c081765", + "user_permissions": { + "IsAdmin": "admin", + "ResetPassword": "admin" + } + }, + { + "api_model": {}, + "first_name": "Test", + "last_name": "User", + "email_address": "banana@test.com", + "password": "", + "org_id": "54b53d3aeba6db5c35000002", + "active": true, + "id": "54bd0ad9ff4329b88985aafb", + "access_key": "f81ee6f0c8f2467d539c132c8a422346", + "user_permissions": { + "user_groups": "read", + "users": "read" + } + } + ], + "pages": 0 +} +``` + +#### Get User + +| **Property** | **Description** | +| :------------ | :----------------------- | +| Resource URL | `/api/users/{USER_ID}` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +GET /api/users/54bd0ad9ff4329b88985aafb HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response** + +``` expandable +{ + "api_model": {}, + "first_name": "Test", + "last_name": "User", + "email_address": "banana@test.com", + "password": "", + "org_id": "54b53d3aeba6db5c35000002", + "active": true, + "id": "54bd0ad9ff4329b88985aafb", + "access_key": "f81ee6f0c8f2467d539c132c8a422346" +} +``` + +#### Add User + + + +You can add a user via the API without a password by leaving out the `password` field. You then use [Set User Password](#set-user-password) request to add a password. + + + +You need to have the `users` [Permission object](/api-management/user-management#user-permissions) set to write to use **Add User**. + +If you do set a password, you need to keep a record of it, to enable the password to be reset in the future. + +| **Property** | **Description** | +| :------------ | :--------------- | +| Resource URL | `/api/users` | +| Method | POST | +| Type | None | +| Body | User Object | +| Param | None | + +**Sample Request** + +```http expandable +POST /api/users HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "first_name": "Jason", + "last_name": "Jasonson", + "email_address": "jason@jasonsonson.com", + "active": true, + "password": "thisisatest", + "user_permissions": { "IsAdmin": "admin" } +} +``` + +**Sample Response** + +``` +{ + "Status": "OK", + "Message": "User created", + "Meta": "" +} +``` + +#### Set User Password + +If a user is created with a blank password, you will need to add a password in a second API call to set a password. In this scenario, the `current_password` field is not required. To change an current password, you need to know the existing password set in **Add User**. + +You need to have the `users` [Permission object](/api-management/user-management#user-permissions) set to **read** to use **Set User Password**. + +| **Property** | **Description** | +| :------------ | :------------------------------------- | +| Resource URL | `/api/users/{USER_ID}/actions/reset` | +| Method | POST | +| Type | None | +| Body | Password Object | +| Param | None | + +**Sample Request** + +```http expandable +POST /api/users/54c25e845d932847067402e2/actions/reset HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "current_password": "12345", + "new_password":"test123456", + "user_permissions": { "IsAdmin": "admin" } +} +``` + +**Sample Response** + +``` +{ + "Status": "OK", + "Message": "User password updated", + "Meta": "" +} +``` + +#### Allow Reset Password + +| **Property** | **Description** | +| :------------ | :------------------------------------------------------ | +| Resource URL | `/admin/users/{USER_ID}/actions/allow_reset_passwords`| +| Method | PUT | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** +```http +PUT -H "admin-auth: " http:///admin/users/{USER_ID}/actions/allow_reset_passwords +``` + +**Sample Response** +``` expandable +{ + "Status": "OK", + "Message": "User updated", + "Meta": + { …user object payload …} +} +``` + +#### Disallow Reset Password + +| **Property** | **Description** | +| :------------ | :---------------------------------------------------------- | +| Resource URL | `/admin/users/{USER_ID}/actions/disallow_reset_passwords` | +| Method | PUT | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** +```http +PUT -H "admin-auth: " http:///admin/users/{USER_ID}/actions/disallow_reset_passwords +``` + +**Sample Response** + +``` expandable +{ + "Status": "OK", + "Message": "User updated", + "Meta": + { …user object payload …} +} +``` + +#### Update User + +You need to have the `users` [Permission object](/api-management/user-management#user-permissions) set to write to use **Update User**. + +| **Property** | **Description** | +| :------------ | :----------------------- | +| Resource URL | `/api/users/{USER_ID}` | +| Method | PUT | +| Type | None | +| Body | User Object | +| Param | None | + +**Sample Request** + +```http expandable +PUT /api/users/54c25e845d932847067402e2 HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "first_name": "Jason", + "last_name": "File", + "email_address": "jason.file@jasonsonson.com", + "active": true, + "user_permissions": { "IsAdmin": "admin" } +} +``` + +**Sample Response** + +``` +{ + "Status": "OK", + "Message": "User updated", + "Meta": null +} +``` + +##### Reset User Session + +This call allows you to reset a user's current Dashboard session. + +You need to have the `users` [Permission object](/api-management/user-management#user-permissions) set to write to use this call. + + + +This also resets the user's Dashboard API credentials. + + + +| **Property** | **Description** | +| :------------ | :------------------------------------------ | +| Resource URL | `/api/users/{USER_ID}/actions/key/reset` | +| Method | PUT | +| Type | None | +| Body | `{"userId":"{USER_ID}"}` | +| Param | None | + +**Sample Request** + +```http expandable +PUT /api/users/54c25e845d932847067402e2/actions/key/reset HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +{ + "userId":"{USER_ID}" +} +``` + +**Sample Response** + +```http +{ + "Status": "OK", + "Message": "User session renewed", + "Meta": null +} +``` + +#### Delete User + +| **Property** | **Description** | +| :------------ | :----------------------- | +| Resource URL | `/api/users/{USER_ID}` | +| Method | DELETE | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +DELETE /api/users/54c25e845d932847067402e2 HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response** + +``` +{ + "Status": "OK", + "Message": "User deleted", + "Meta": "" +} +``` + +### User Groups API + +#### List User Groups + +| **Property** | **Description** | +| :------------ | :--------------- | +| Resource URL | `/api/usergroups` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +GET /api/usergroups HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response** + +``` expandable +{ + "groups": [ + { + "org_id": "54b53d3aeba6db5c35000002", + "id": "54b53d4bf25b920f09361526", + "name": "Analytics team", + "description": "Only access to analytics pages", + "active": true, + "user_permissions": { "analytics": "read" } + }, + { + "org_id": "54b53d3aeba6db5c35000003", + "id": "54b53d4bf25b920f09361527", + "name": "Certificates team", + "description": "Team to manage certificates", + "active": true, + "user_permissions": { "certificates": "write" } + } + ], + "pages": 0 +} +``` + +#### Get User Group + +| **Property** | **Description** | +| :------------ | :----------------------- | +| Resource URL | `/api/usergroups/{user_group-id}` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +GET /api/usergroups/54bd0ad9ff4329b88985aafb HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response** + +``` expandable +{ + "org_id": "54b53d3aeba6db5c35000002", + "id": "54b53d4bf25b920f09361526", + "name": "Certificates team", + "description": "Team to manage certificates", + "active": true, + "user_permissions": { "certificates": "write" } +} +``` + +#### Add User Group + + + +| **Property** | **Description** | +| :------------ | :--------------- | +| Resource URL | `/api/usergroups` | +| Method | POST | +| Type | None | +| Body | User Object | +| Param | None | + +**Sample Request** + +```http expandable +POST /api/usergroups HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "name": "Logs team", + "description": "Logs team description", + "user_permissions": { "logs": "read" } +} +``` + +**Sample Response** + +``` +{ + "Status": "OK", + "Message": "User group created", + "Meta": "" +} +``` + + + +#### Update User Group + +| **Property** | **Description** | +| :------------ | :----------------------- | +| Resource URL | `/api/usergroups/{user_group-id}` | +| Method | PUT | +| Type | None | +| Body | User Group Object | +| Param | None | + +**Sample Request** + +```http expandable +PUT /api/usergroups/54c25e845d932847067402e2 HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "name": "Certificates Team 2", + "description": "Another certificates team", +} +``` + +**Sample Response** + +``` +{ + "Status": "OK", + "Message": "User group updated", + "Meta": null +} +``` + +#### Delete User Group + +| **Property** | **Description** | +| :------------ | :----------------------- | +| Resource URL | `/api/usergroups/{user_group-id}` | +| Method | DELETE | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +DELETE /api/usergroups/54c25e845d932847067402e2 HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response** + +``` +{ + "Status": "OK", + "Message": "User group deleted", + "Meta": "" +} +``` + +### Additional Permissions API + + + +This API helps you to add and delete (CRUD) a list of additional (custom) permissions for your Dashboard users. +Once created, a custom permission will be added to standard list of user permissions. +
+Only Admin Dashboard users will be authorized to use this API. + + + + + +#### List Additional Permissions + +This API returns by default the initial set of additional permissions defined in your Tyk Dashboard configuration, under [security.additional_permissions](/tyk-dashboard/configuration#securityadditional_permissions). + +Once you update the permissions via the API, they will be stored at organization level. + +| **Property** | **Description** | +| :------------ | :--------------------- | +| Resource URL | `/api/org/permissions`| +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +GET /api/org/permissions HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response** + +``` expandable +{ + "additional_permissions": { + "api_developer": "API Developer", + "api_manager": "API Manager" + } +} +``` + +#### Add/Delete/Update Additional Permission + + + +Whenever you want to add/update/delete an additional permission, just send back the updated list of permissions, through a PUT request to the API. + + + + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/org/permission` | +| Method | PUT | +| Type | None | +| Body | Permissions Object | +| Param | None | + +**Sample Request** + +Let's imagine we have already defined two additional permissions: `api_developer` and `api_manager`. In order to add a new permission to this list, just send +an updated list of permissions by appending the values you want. In this example we will add a `custom_permission` permission. + +```http expandable +PUT /api/org/permissions HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "additional_permissions": { + "api_developer": "API Developer", + "api_manager": "API Manager", + "custom_permission": "Custom Permission" + } +} +``` + +**Sample Response** + +``` +{ + "Status": "OK", + "Message": "Additional Permissions updated in org level", + "Meta": null +} +``` + +### Access Keys Management API + + + +`{api-id}` can either be the internal or external API id. + + + +#### Get a list of Keys + +**Note:** This will not work with a hashed key set. + +| **Property** | **Description** | +| :------------ | :-------------------------- | +| Resource URL | `/api/apis/{api-id}/keys` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request:** + +```http +GET /api/apis/39d2c98be05c424371c600bd8b3e2242/keys HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response:** + +```yalm expandable +{ + "data": { + "keys": [ + "54b53d3aeba6db5c3500000289a8fbc2bbba4ebc4934bb113588c792", + "54b53d3aeba6db5c3500000230459d8568ec4bbf675bda2ff05e9293", + "54b53d3aeba6db5c35000002ec9a2b1aca7b495771273a0895cb3627", + "54b53d3aeba6db5c3500000272d97a10538248e9523ca09e425090b8", + "54b53d3aeba6db5c3500000252b5c56c61ad42fe765101f6d70cf9c6" + ] + }, + "pages": 0 +} +``` + +#### Get a specific key + +| **Property** | **Description** | +| :------------ | :----------------------------------- | +| Resource URL | `/api/apis/{api-id}/keys/{key-id}` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +GET /api/apis/39d2c98be05c424371c600bd8b3e2242/keys/54b53d3aeba6db5c3500000289a8fbc2bbba4ebc4934bb113588c792 HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response:** + +``` expandable +{ + "api_model": {}, + "key_id": "54b53d3aeba6db5c3500000289a8fbc2bbba4ebc4934bb113588c792", + "data": { + "last_check": 1421674410, + "allowance": 1000, + "rate": 1000, + "per": 60, + "expires": 1423684135, + "quota_max": -1, + "quota_renews": 1421164189, + "quota_remaining": -1, + "quota_renewal_rate": 60, + "access_rights": { + "39d2c98be05c424371c600bd8b3e2242": { + "api_name": "Nitrous Test", + "api_id": "39d2c98be05c424371c600bd8b3e2242", + "versions": [ + "Default" + ] + } + }, + "org_id": "54b53d3aeba6db5c35000002", + "oauth_client_id": "", + "basic_auth_data": { + "password": "" + }, + "hmac_enabled": true, + "hmac_string": "" + } +} +``` + + +#### Create a custom key + +| **Property** | **Description** | +| :------------ | :--------------- | +| Resource URL | `/api/keys/{custom-key-id}` | +| Method | POST | +| Type | None | +| Body | Session Object | +| Param | None | + +**Sample Request** + +```http expandable +POST /api/keys/my-custom-key HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "apply_policies": ["5ecc0b91081ac40001ed261c"], + "org_id" : "5eb06f441fe4c4000147476e", + + // Below gets overwritten by the Policy, required nonetheless + "expires": 0, + "allowance": 0, + "per": 0, + "quota_max": 0, + "rate": 0, + "access_rights": { + "b742100081764ff06b00f75733145614": { + "api_name": "", + "api_id": "b742100081764ff06b00f75733145614", + "versions": [ + "Default" + ] + } + } +} +``` + +You might be wondering why `access_rights` is necessary, as we are adding a security policy and inheriting the access rights from there. That's because of legacy functionality. We need to add any APIs `api_id` to the key of the access_rights map, as well as the `api_id` value of that key. This will all get overwritten by the policy, but we need to add it. + +**Sample Response:** + + +``` expandable +{ + "api_model": {}, + "key_id": "eyJvcmciOiI1ZTlkOTU0NGExZGNkNjAwMDFkMGVkMjAiLCJpZCI6ImhlbGxvLXdvcmxkIiwiaCI6Im11cm11cjY0In0=", + "data": { + ... + }, + "key_hash": "567b9a5419c3a9ef" +} +``` + +You can now use `my-custom-key` as a key to access the API. Furthermore, you can use it to lookup the key in the Dashboard as well as the generated `key_hash` in the response. + +Let's try curling it: + +```curl +$ curl localhost:8080/my-api/users/1 --header "Authorization: my-custom-key" +{ + "response" : "hello world" +} +``` + +#### Generate a key + +| **Property** | **Description** | +| :------------ | :--------------- | +| Resource URL | `/api/keys` | +| Method | POST | +| Type | None | +| Body | Session Object | +| Param | None | + +**Sample Request** + +```http expandable +POST /api/keys HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "last_check": 0, + "allowance": 1000, + "rate": 1000, + "per": 60, + "expires": 0, + "quota_max": 10000, + "quota_renews": 1424543479, + "quota_remaining": 10000, + "quota_renewal_rate": 2520000, + "access_rights": { + "bc2f8cfb7ab241504d9f3574fe407499": { + "api_id": "bc2f8cfb7ab241504d9f3574fe407499", + "api_name": "Test", + "versions": [ + "Default" + ] + } + } +} +``` + +**Sample Response:** + +``` expandable +{ + "api_model": {}, + "key_id": "54b53d3aeba6db5c3500000216d056646b4b4ffe4e54f5b07d658f8a", + "data": { + "last_check": 0, + "allowance": 1000, + "rate": 1000, + "per": 60, + "expires": 0, + "quota_max": 10000, + "quota_renews": 1424543479, + "quota_remaining": 10000, + "quota_renewal_rate": 2520000, + "access_rights": { + "bc2f8cfb7ab241504d9f3574fe407499": { + "api_name": "Test", + "api_id": "bc2f8cfb7ab241504d9f3574fe407499", + "versions": [ + "Default" + ] + } + }, + "org_id": "54b53d3aeba6db5c35000002", + "oauth_client_id": "", + "basic_auth_data": { + "password": "" + }, + "hmac_enabled": false, + "hmac_string": "" + } +} +``` + +#### Update a key + +| **Property** | **Description** | +| :------------ | :------------------------------------ | +| Resource URL | `/api/apis/{api-id}/keys/{keyId}` | +| Method | PUT | +| Type | None | +| Body | Session Object | +| Param | None | + +**Sample Request** + +```http expandable +PUT /api/apis/39d2c98be05c424371c600bd8b3e2242/keys/54b53d3aeba6db5c3500000272d97a10538248e9523ca09e425090b8 HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "last_check": 0, + "allowance": 1000, + "rate": 1000, + "per": 60, + "expires": 1422113671, + "quota_max": -1, + "quota_renews": 1421675253, + "quota_remaining": -1, + "quota_renewal_rate": 60, + "access_rights": { + "39d2c98be05c424371c600bd8b3e2242": { + "api_id": "39d2c98be05c424371c600bd8b3e2242", + "api_name": "Nitrous Test", + "versions": [ + "Default" + ] + } + }, + "org_id": "54b53d3aeba6db5c35000002", + "oauth_client_id": "", + "basic_auth_data": { + "password": "" + }, + "hmac_enabled": false, + "hmac_string": "" +} +``` + +**Sample Response:** + +``` +{ + "Status": "OK", + "Message": "Key updated", + "Meta": "" +} +``` + +#### Delete a key + +| **Property** | **Description** | +| :------------ | :--------------------------------- | +| Resource URL | `/api/apis/{api-id}/keys/{keyId}` | +| Method | DELETE | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +DELETE /api/apis/39d2c98be05c424371c600bd8b3e2242/keys/54b53d3aeba6db5c3500000272d97a10538248e9523ca09e425090b8 HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response:** + +``` +{ + "Status": "OK", + "Message": "Key deleted succesfully", + "Meta": "" +} +``` + +#### Graphql API + +Presently, the Tyk Dashboard uses the GraphQL API for keys. + +| **Method** | **URL** | **Description** | +| :---------- | :------------- | :--------------------------- | +| POST | `/graphql` | GraphQL query endpoint | +| GET | `/playground` | Dashboard Graphql Playground - where you could see docs and run queries | + + +### Basic Authentication API + +Basic Auth users are essentially a form of API token, just with a customized, pre-set organization-specific ID instead of a generated one. To interact with basic auth users, you can use the API Token API calls (list, get delete etc.) + +#### Create a user + +| **Property** | **Description** | +| :------------ | :--------------------------------- | +| Resource URL | `/api/apis/keys/basic/{username}` | +| Method | POST | +| Type | None | +| Body | Session Object | +| Param | None | + +**Sample Request** + +```http expandable +POST /api/apis/keys/basic/test-user HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "last_check": 0, + "allowance": 1000, + "rate": 1000, + "per": 60, + "expires": 0, + "quota_max": 10000, + "quota_renews": 1424543479, + "quota_remaining": 10000, + "quota_renewal_rate": 2520000, + "access_rights": { + "bc2f8cfb7ab241504d9f3574fe407499": { + "api_id": "bc2f8cfb7ab241504d9f3574fe407499", + "api_name": "Test", + "versions": [ + "Default" + ] + } + }, + "basic_auth_data": { + "password": "test123" + } +} +``` + +**Sample Response** + +``` expandable +{ + "api_model": {}, + "key_id": "54b53d3aeba6db5c3500000test-user", + "data": { + "last_check": 0, + "allowance": 1000, + "rate": 1000, + "per": 60, + "expires": 0, + "quota_max": 10000, + "quota_renews": 1424543479, + "quota_remaining": 10000, + "quota_renewal_rate": 2520000, + "access_rights": { + "bc2f8cfb7ab241504d9f3574fe407499": { + "api_name": "Test", + "api_id": "bc2f8cfb7ab241504d9f3574fe407499", + "versions": [ + "Default" + ] + } + }, + "org_id": "54b53d3aeba6db5c35000002", + "oauth_client_id": "", + "basic_auth_data": { + "password": "" + }, + "hmac_enabled": false, + "hmac_string": "" + } +} +``` + +### OAuth Key Management API + +#### Create a new OAuth2.0 Client + +Any OAuth keys must be generated under an API in the Dashboard. Any POST requests made should contain the API's ID in the URL. + +| **Property** | **Description** | +| :------------ | :---------------------------- | +| Resource URL | `/api/apis/oauth/{{api-id}}` | +| Method | POST | +| Type | JSON | +| Body | Client Object | + + +**Sample Request** + +```curl + curl -vX POST -H "Authorization: {{API Access Credentials}}" \ + -H "Content-Type: application/json" \ + -d '{"redirect_uri": "", "policy_id": "{{policy_id}}"}' http://{{dasboard-hostname}}/api/apis/oauth/{{api-id}} +``` + +**Sample Response** + +```yaml expandable +{ + "client_id": "72083e90e9b044c57e2667d49effff78", + "secret": "YWUxZTM2ODItOTJjYS00MmIyLTQxZGEtZTE0M2MzNmYwMDI2", + "redirect_uri": "", + "policy_id": "57f7b07647e0800001aa2320" +} +``` + +#### List OAuth Clients + +| **Property** | **Description** | +| :------------ | :---------------------------- | +| Resource URL | `/api/apis/oauth/{{api-id}}` | +| Method | GET | +| Type | JSON | +| Body | NONE | + + +**Sample Request** + +```http +curl -vX GET -H "Authorization: {{API Access Credentials}}" \ + -H "Content-Type: application/json" \ + http://{{dashboard-hostname}}/api/apis/oauth/{{api-id}} +``` + +**Sample Response** + +```yaml expandable +{ + "apps": [ + { + "client_id": "7dce7fc297424fd65596b51c214666a4", + "secret":"Yzg0ZDRjZTctMzUxNy00YmQ5LTRkM2UtMDdmODQ3MTNjNWM5", + "redirect_uri": "/cats", + "policy_id": "57f7b07647e0800001aa2320" + }, + { + "client_id": "72083e90e9b044c57e2667d49effff78", + "secret": "YWUxZTM2ODItOTJjYS00MmIyLTQxZGEtZTE0M2MzNmYwMDI2", + "redirect_uri": "", + "policy_id": "57f7b07647e0800001aa2320" + } + ], + "pages":0 +} +``` + +#### Get an OAuth2.0 Client + +| **Property** | **Description** | +| :------------ | :------------------------------------------ | +| Resource URL | `/api/apis/oauth/{{api-id}}/{{client_id}}` | +| Method | GET | +| Type | JSON | +| Body | NONE | + + +**Sample Request** + +```curl +curl -vX GET -H "Authorization: {{API Access Credentials}}" \ + -H "Content-Type: application/json" \ + http://localhost:3000/api/apis/oauth/{{api-id}}/{{client_id}} +``` + +**Sample Response** + +```yaml expandable +{ + "client_id": "7dce7fc297424fd65596b51c214666a4", + "secret": "Yzg0ZDRjZTctMzUxNy00YmQ5LTRkM2UtMDdmODQ3MTNjNWM5", + "redirect_uri": "/cats", + "policy_id": "57f7b07647e0800001aa2320" +} +``` + +#### Delete OAuth Client + +You can delete an OAuth client using a simple DELETE method. Please note that tokens issued with the client ID will still be valid until they expire. + +| **Property** | **Description** | +| :------------ | :------------------------------------------ | +| Resource URL | `/api/apis/oauth/{{api-id}}/{{client_id}}` | +| Method | DELETE | +| Type | JSON | +| Body | NONE | + + +**Sample Request** + +```curl +curl -vX DELETE -H "Authorization: {{API Access Credentials}}" \ + -H "Content-Type: application/json" \ + http://{{dashboard-hostname}}/api/apis/oauth/{{api-id}}/{{client_id}} +``` + +**Sample Response** + +```json +{ + "Status": "OK", + "Message": "OAuth Client deleted successfully", + "Meta": null +} +``` + +#### Retrieve All Current Tokens for Specified OAuth2.0 Client + +This endpoint allows you to retrieve a list of all current tokens and their expiry date for a provided API ID and OAuth-client ID in the following format. This endpoint will work only for newly created tokens. + + + +This option is available from v2.6.0 onwards. + + + + +| **Property** | **Description** | +| :------------ | :---------------------------------------------------- | +| Resource URL | `/api/apis/oauth/{apiID}/{oauthClientId}/tokens` | +| Method | GET | +| Type | | +| Body | NONE | + +**Sample Request** +```http +GET /api/apis/oauth/528a67c1ac9940964f9a41ae79235fcc/25348e8cf157409b52e39357fd9578f1/tokens HTTP/1.1 +Host: localhost:3000 +Authorization: {{API Access Credentials}} +Cache-Control: no-cache +``` + +**Sample Response** +```yaml expandable +[ + { + "code": "5a7d110be6355b0c071cc339327563cb45174ae387f52f87a80d2496", + "expires": 1518158407 + }, + { + "code": "5a7d110be6355b0c071cc33988884222b0cf436eba7979c6c51d6dbd", + "expires": 1518158594 + }, + { + "code": "5a7d110be6355b0c071cc33990bac8b5261041c5a7d585bff291fec4", + "expires": 1518158638 + }, + { + "code": "5a7d110be6355b0c071cc339a66afe75521f49388065a106ef45af54", + "expires": 1518159792 + } +] +``` + +You can control how long you want to store expired tokens in this list using `oauth_token_expired_retain_period` which specifies retain period for expired tokens stored in Redis. By default expired token not get removed. See [here](/tyk-oss-gateway/configuration#oauth_token_expired_retain_period) for more details. + +#### Revoke a Single OAuth Client Token + +| **Property** | **Description** | +| :------------ | :---------------------------------------------- | +| Resource URL | `/api/apis/oauth/{oauthClientId}/revoke` | +| Method | POST | +| Type | JSON | +| Body | Client Object | +| Param | None | + + +**Sample Request** + +```http expandable +POST /api/apis/oauth/411f0800957c4a3e81fe181141dbc22a/revoke +Host: localhost +Authorization 64c8e662f6924c4f55e94a873d75e44d +Body: { + "token": "eyJvcmciOiI1ZTIwOTFjNGQ0YWVmY2U2MGMwNGZiOTIiLCJpZCI6IjIyODQ1NmFjNmJlMjRiMzI5MTIyOTdlODQ5NTc4NjJhIiwiaCI6Im11cm11cjY0In0=", + "token_type_hint": "access_token" +} +``` +**Sample Response** + +```json +{ + "Status": "OK", + "Message": "token revoked successfully", + "Meta": null +} +``` +#### Revoke all OAuth Client Tokens + +| **Property** | **Description** | +| :------------ | :---------------------------------------------- | +| Resource URL | `/api/apis/oauth/{oauthClientId}/revoke_all` | +| Method | POST | +| Type | JSON | +| Body | Client Object | +| Param | None | + +**Sample Request** + +```http expandable +POST /api/apis/oauth/411f0800957c4a3e81fe181141dbc22a/revoke_all +Host: localhost +Authorization: 64c8e662f6924c4f55e94a873d75e44d +Body: { + "client_secret":"MzUyNDliNzItMDhlNy00MzM3LTk1NWUtMWQyODMyMjkwZTc0" +} +``` + +**Sample Response** + +```json +{ + "Status": "OK", + "Message": "tokens revoked successfully", + "Meta": null +} +``` + +#### OAuth2.0 Authorization Code + +This endpoint is used in the [Authorization Code Grant](/api-management/authentication/oauth-2#using-the-authorization-code-grant) flow, generating an authorization code that can be used by the client to request an access token. + +| **Property** | **Description** | +| :------------ | :---------------------------------------------- | +| Resource URL | `/api/apis/oauth/{{api_id}}/authorize-client/` | +| Method | POST | +| Type | Form-Encoded | +| Body | Fields (see below) | + +* `api_id`: Unlike the other requests on this page, this must be the `api_id` value and **NOT** the API's `id` value. +* `response_type`: Should be provided by requesting client as part of authorization request, this should be either `code` or `token` depending on the methods you have specified for the API. +* `client_id`: Should be provided by requesting client as part of authorization request. The Client ID that is making the request. +* `redirect_uri`: Should be provided by requesting client as part of authorization request. Must match with the record stored with Tyk. +* `key_rules`: A string representation of a Session Object (form-encoded). *This should be provided by your application in order to apply any quotas or rules to the key.* + +Note that in the following example, the `policy_id` isn't included in the request as these are optional. OAuth2.0 Flow also supports callbacks which can be added to the `key_rules` in the payload in requests that don't include the `policy_id`. + + +**Sample Request** + +```curl +curl -vX POST -H "Authorization: {{API Access Credentials}}" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d 'response_type=code&client_id={{client_id}}&redirect_uri=http%3A%2F%2Foauth.com%2Fredirect&key_rules=%7B+++++%22allowance%22%3A+999%2C+++++%22rate%22%3A+1000%2C+++++%22per%22%3A+60%2C+++++%22expires%22%3A+0%2C+++++%22quota_max%22%3A+-1%2C+++++%22quota_renews%22%3A+1406121006%2C+++++%22quota_remaining%22%3A+0%2C+++++%22quota_renewal_rate%22%3A+60%2C+++++%22access_rights%22%3A+%7B+++++++++%22528a67c1ac9940964f9a41ae79235fcc%22%3A+%7B+++++++++++++%22api_name%22%3A+%22{{api_name}}%22%2C+++++++++++++%22api_id%22%3A+%{{api_id}}%22%2C+++++++++++++%22versions%22%3A+%5B+++++++++++++++++%22Default%22+++++++++++++%5D+++++++++%7D+++++%7D%2C+++++%22org_id%22%3A+%22{{org_id}}%22+%7D' +http://{{dashboard-hostname}}/api/apis/oauth/{{api_id}}/authorize-client +``` + +**Sample Response** + +``` +{ + "code": "MWY0ZDRkMzktOTYwNi00NDRiLTk2YmQtOWQxOGQ3Mjc5Yzdk", + "redirect_to": "http://localhost:3000/oauth-redirect/?code=MWY0ZDRkMzktOTYwNi00NDRiLTk2YmQtOWQxOGQ3Mjc5Yzdk" +} +``` + +### Single Sign On API + + + +This functionality is available from [v2.9.0](/developer-support/release-notes/archived#single-sign-on-for-the-tyk-saas). If you have an older version please using the [admin api](/api-management/dashboard-configuration#single-sign-on-api-1) + + + + +The Dashboard SSO API allows you to implement custom authentication schemes for the Dashboard and Portal. +Our Tyk Identity Broker (TIB) internally also uses this API. + +#### Generate authentication token + +The Dashboard exposes the `/api/sso` Dashboard API which allows you to generate a temporary authentication token, valid for 60 seconds. + +You should provide JSON payload with the following data: + +* `ForSection` - scope with possible values of `"dashboard"` or `"portal"` only. +* `OrgID` - organization id +* `EmailAddress` - user email +* `GroupID` - user group id ( it is the mongo id and you can can find it in the url when opening a user group via Tyk- Dashboard UI or if you call Tyk-Dashboard REST API `/api/usergroups` ) + + +| **Property** | **Description** | +| :------------ | :---------------------------- | +| Resource URL | `/api/sso` | +| Method | POST | +| Body | `{"ForSection":"", "OrgID": "", "EmailAddress": "", "GroupID": ""}` | + +**Sample Request** + +```http expandable +POST /api/sso HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "ForSection": "dashboard", + "OrgID": "588b4f0bb275ff0001cc7471", + "EmailAddress": "name@somewhere.com", + "GroupID": "" +} +``` + +**Sample Response:** +```json +{ + "Status": "OK", + "Message": "SSO Nonce created", + "Meta": "YTNiOGUzZjctYWZkYi00OTNhLTYwODItZTAzMDI3MjM0OTEw" +} +``` + +See [Single Sign On](/api-management/external-service-integration#single-sign-on-sso) documentation for how to use this token and more details. + +### Web Hooks API + +#### List web hooks + +| **Property** | **Description** | +| :------------ | :--------------- | +| Resource URL | `/api/hooks` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +GET /api/hooks HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response** + +``` expandable +{ + "hooks": [ + { + "api_model": {}, + "id": "54be6c0beba6db07a6000002", + "org_id": "54b53d3aeba6db5c35000002", + "name": "Test Post", + "method": "POST", + "target_path": "http://httpbin.org/post", + "template_path": "", + "header_map": { + "x-tyk-test": "123456" + }, + "event_timeout": 0 + } + ], + "pages": 0 +} +``` + +#### Get single web hook + +| **Property** | **Description** | +| :------------ | :---------------------- | +| Resource URL | `/api/hooks/{hook-id}` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +GET /api/hooks/54be6c0beba6db07a6000002 HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response** + +``` expandable +{ + "api_model": {}, + "id": "54be6c0beba6db07a6000002", + "org_id": "54b53d3aeba6db5c35000002", + "name": "Test Post", + "method": "POST", + "target_path": "http://httpbin.org/post", + "template_path": "", + "header_map": { + "x-tyk-test": "123456" + }, + "event_timeout": 0 +} +``` + +#### Add hook + +| **Property** | **Description** | +| :------------ | :--------------- | +| Resource URL | `/api/hooks` | +| Method | POST | +| Type | None | +| Body | Hook object | +| Param | None | + +**Sample Request** + +```http expandable +POST /api/hooks HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "name": "New Post Test", + "method": "POST", + "target_path": "http://httpbin.org/post", + "header_map": { + "x-test": "y-answer" + } +} +``` + +**Sample Response** + +``` +{ + "Status": "OK", + "Message": "Webhook created", + "Meta": "" +} +``` + +#### Update hook + +| **Property** | **Description** | +| :------------ | :---------------------- | +| Resource URL | `/api/hooks/{hook-id}` | +| Method | PUT | +| Type | None | +| Body | Hook object | +| Param | None | + +**Sample Request** + +```http expandable +PUT /api/hooks/54c2617aeba6db1edc000003 HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "api_model": {}, + "id": "54c2617aeba6db1edc000003", + "org_id": "54b53d3aeba6db5c35000002", + "name": "New Post Test", + "method": "PUT", + "target_path": "http://httpbin.org/post", + "template_path": "", + "header_map": { + "x-test": "y-answer" + }, + "event_timeout": 0 +} +``` + +**Sample Response** + +``` +{ + "Status": "OK", + "Message": "Webhook updated", + "Meta": "" +} +``` + +#### Delete web hook + +| **Property** | **Description** | +| :------------ | :------------------------- | +| Resource URL | `/api/hooks/{hook-id}` | +| Method | DELETE | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +DELETE /api/hooks/54c2617aeba6db1edc000003 HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response** + +``` +{ + "Status": "OK", + "Message": "Webhook deleted", + "Meta": "" +} +``` + +### Open Policy Agent API + + + +The Open Policy Agent API helps you to manage (CRUD) the OPA (Open Policy Agent) rules that are being applied to the Tyk Dashboard. You can also change the OPA settings, such as to enable/disable it or enable/disable the debug mode. + +Only Admin role Dashboard users are authorized to use it. + + + +For more information on how to configure OPA see [Open Policy Agent](/api-management/dashboard-configuration#extend-permissions-using-open-policy-agent-opa). +#### List OPA rules and settings + +This endpoint returns by defaul the initial set of OPA rules defined in your Tyk Dashboard, which are located in [schema/dashboard.rego](/api-management/dashboard-configuration#dashboard-opa-rules) (accessible in Self-Managed installations). + +Once you update the rules via the API, the OPA rules will be stored at the organization level. + +| **Property** | **Description** | +| :------------ | :--------------------- | +| Resource URL | `/api/org/opa `| +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +GET /api/org/opa HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +**Sample Response** + +``` expandable +{ + "open_policy": { + "enabled": true, + "rules": "default hello = false\r\n\r\nhello {\r\n m := input.message\r\n m == \"world\"\r\n}" + } +} +``` +#### Update OPA rules and settings + + + +Whenever you want to update OPA rules or its settings, send the updated value of the OPA rules or changed values for the settings (`enabled`) via a PUT request to the `permissions` endpoint. + + + + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/org/permission` | +| Method | PUT | +| Type | None | +| Body | Permissions Object | +| Param | None | + +**Sample Request** + +```http +PUT /api/org/opa HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +``` expandable +{ + "open_policy": { + "enabled": false, + "rules": "default hello = false\r\n\r\nhello {\r\n m := input.message\r\n m == \"world\"\r\n}" + } +} +``` + +**Sample Response** + +``` +{ + "Status": "OK", + "Message": "OPA rules has been updated on org level", + "Meta": null +} +``` + +## Dashboard Admin API Resources and Usage + +### API Usage Instructions + + + +**Important Note on Spelling:** + +While our documentation now uses American English [(en-us)](https://www.andiamo.co.uk/resources/iso-language-codes/), the product itself, including all user interfaces, configuration +fields, environment variables, and APIs, continues to use British English spellings. When interacting with the product, +please continue using the British English (en-gb) spellings as appear in the interface and API. This change does not affect +how you use the product; all functionality remains the same. + +
+ +**Example:** The API endpoint `/organisations` as shown throughout this page uses British spelling (with an 's' not 'z'). +In all other instances, such as when describing or referring to this object in the documentation, we will use the +American spelling β€œorganization” with a 'z'. + + + + + +In a production environment, you must change the default `admin_Secret` in the`tyk_analytics.conf` file. Admin APIs use this value for authentication, and you should set it in the `admin-auth` header while sending the request. +
+ + + +For the official Tyk Dashboard Admin API Reference, please visit our [API Documentation](/dashboard-admin-api). + +### Organizations API + +#### Retrieve a single Organization + +| **Property** | **Description** | +| :------------ | :--------------------------------- | +| Resource URL | `/admin/organisations/{org-id}` | +| Method | GET | +| Type | None | +| Body | Organization Object | +| Param | None | + +**Sample Request** + +```http +GET /admin/organisations/{ORG_ID} HTTP/1.1 +Host: localhost:3000 +admin-auth: 12345 +``` + +**Sample Response** + +```json expandable +{ + "id": "5cc03283d07e7f00019404b3", + "owner_name": "TestOrg5 Ltd.", + "owner_slug": "testorg", + "cname_enabled": true, + "cname": "www.tyk-portal-test.com", + "apis": [ + { + "api_human_name": "First API #Test", + "api_id": "5508bd9429434d5768c423a04db259ea" + } + ], + "developer_quota": 0, + "developer_count": 0, + "event_options": {}, + "hybrid_enabled": false, + "ui": { + "languages": {}, + "hide_help": false, + "default_lang": "", + "login_page": {}, + "nav": {}, + "uptime": {}, + "portal_section": {}, + "designer": {}, + "dont_show_admin_sockets": false, + "dont_allow_license_management": false, + "dont_allow_license_management_view": false, + "cloud": false + }, + "org_options_meta": {} +} +``` + + +#### Retrieve all Organizations + +| **Property** | **Description** | +| :------------ | :------------------------- | +| Resource URL | `/admin/organisations/' | +| Method | GET | +| Type | None | +| Body | Organization Object | +| Param | None | + +**Sample Request** + +```http +GET /admin/organisations/ HTTP/1.1 +Host: localhost:3000 +admin-auth: 12345 +``` + +**Sample Response** + +```json expandable +{ + "organisations": [ + { + "id": "5cc03283d07e7f00019404b3", + "owner_name": "TestOrg5 Ltd.", + "owner_slug": "testorg", + "cname_enabled": true, + "cname": "www.tyk-portal-test.com", + "apis": [ + { + "api_human_name": "First API #Test", + "api_id": "5508bd9429434d5768c423a04db259ea" + } + ], + "developer_quota": 0, + "developer_count": 0, + "event_options": {}, + "hybrid_enabled": false, + "ui": { + "languages": {}, + "hide_help": false, + "default_lang": "", + "login_page": {}, + "nav": {}, + "uptime": {}, + "portal_section": {}, + "designer": {}, + "dont_show_admin_sockets": false, + "dont_allow_license_management": false, + "dont_allow_license_management_view": false, + "cloud": false + }, + "org_options_meta": {} + }, + { + "id": "5ccae84aa402ce00018b5435", + "owner_name": "Jively", + "owner_slug": "", + "cname_enabled": true, + "cname": "jive.ly", + "apis": [], + "developer_quota": 0, + "developer_count": 0, + "event_options": {}, + "hybrid_enabled": false, + "ui": { + "languages": {}, + "hide_help": false, + "default_lang": "", + "login_page": {}, + "nav": {}, + "uptime": {}, + "portal_section": {}, + "designer": {}, + "dont_show_admin_sockets": false, + "dont_allow_license_management": false, + "dont_allow_license_management_view": false, + "cloud": false + }, + "org_options_meta": {} + } + ], + "pages": 0 +} +``` + +#### Create an Organization + +| **Property** | **Description** | +| :------------ | :------------------------- | +| Resource URL | `/admin/organisations/` | +| Method | POST | +| Type | None | +| Body | Organization Object | +| Param | None | + +**Sample Request** + +```http expandable +POST /admin/organisations/ HTTP/1.1 +Host: localhost:3000 +admin-auth: 12345 + +{ + "owner_name": "Jively", + "cname": "jive.ly", + "cname_enabled": true +} +``` + +**Sample response** + +```json +{ + "Status":"OK", + "Message":"Org created", + "Meta":"54b53d3aeba6db5c35000002" +} +``` + +#### Update an Organization + +| **Property** | **Description** | +| :------------ | :----------------------------- | +| Resource URL | `/admin/organisations/{id}` | +| Method | PUT | +| Type | None | +| Body | Organization Object | +| Param | None | + +**Sample Request** + +```http expandable +PUT /admin/organisations/54b53d3aeba6db5c35000002 HTTP/1.1 +Host: localhost:3000 +admin-auth: 12345 + +{ + "owner_name": "Jively", + "cname": "jive.ly", + "cname_enabled": true +} +``` + +**Sample Response** + +```json +{ + "Status":"OK", + "Message":"Org updated", + "Meta":"" +} +``` + +#### Delete an Organization + +| **Property** | **Description** | +| :------------ | :----------------------------- | +| Resource URL | `/admin/organisations/{id}` | +| Method | DELETE | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +DELETE /admin/organisations/54b53d3aeba6db5c35000002 HTTP/1.1 +Host: localhost:3000 +admin-auth: 12345 +``` + +**Sample Response** + +```json +{ + "Status":"OK", + "Message":"Org deleted", + "Meta":"" +} +``` + +### Users API + +#### Get User + +| **Property** | **Description** | +| :------------ | :------------------------- | +| Resource URL | `/admin/users/{USER_ID}` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +GET /admin/users/54bd0ad9ff4329b88985aafb HTTP/1.1 +Host: localhost:3000 +admin-auth: 12345 +``` + +**Sample Response** + + +```json expandable +{ + "api_model": {}, + "first_name": "Test", + "last_name": "User", + "email_address": "banana@test.com", + "password": "", + "org_id": "54b53d3aeba6db5c35000002", + "active": true, + "id": "54bd0ad9ff4329b88985aafb", + "access_key": "f81ee6f0c8f2467d539c132c8a422346" +} +``` + +#### Add user + +When you add a new user, they are created without a password being set. After adding a user, you need to use the [Set Password](#set-user-password) call to set a password using the `user-id` created. + +| **Property** | **Description** | +| :------------ | :--------------- | +| Resource URL | `/admin/users` | +| Method | POST | +| Type | None | +| Body | User Object | +| Param | None | + +**Sample Request** + +```http expandable +POST /admin/users HTTP/1.1 +Host: localhost:3000 +admin-auth: 12345 + +{ + "org_id": "5d15d3068ba30a0001621bfe", + "first_name": "Jason", + "last_name": "Jasonson", + "email_address": "jason@jasonsonson.com", + "active": true, + "user_permissions": { "IsAdmin": "admin" } +} +``` + + + +You can also create a user without an `org_id`. This will create a "Super User", who has global access to all APIs, Policies, etc, for all organizations created within Tyk. + + + + +**Sample Response** + + +``` expandable +{ + "Status": "OK", + "Message": "e5485fa02e12425974e1220e1636e4d0", + "Meta": { + "api_model": {}, + "first_name": "Jason", + "last_name": "user", + "email_address": "jason@jasonsonson.com", + "org_id": "", + "active": true, + "id": "5d55378edd4b9e9c308e87da", + "access_key": "e5485fa02e12425974e1220e1636e4d0", + "user_permissions": { + "IsAdmin": "admin" + }, + "group_id": "", + "password_max_days": 0, + "password_updated": "0001-01-01T00:00:00Z", + "PWHistory": [], + "created_at": "2019-08-15T10:44:30.784Z" + } +} +``` + + +#### Update User + +You need to have the `users` [Permission object](/api-management/user-management#user-permissions) set to write to use **Update User**. + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/admin/users/{USER_ID}` | +| Method | PUT | +| Type | None | +| Body | User Object | +| Param | None | + + +**Sample Request** + +```http expandable +PUT /admin/users/54c25e845d932847067402e2 HTTP/1.1 +Host: localhost:3000 +admin-auth: 12345 + +{ + "access_key": "3a8c1cea90af485575bb5455c2e9fb68", + "first_name": "Jason", + "last_name": "File", + "email_address": "jason.file@jasonsonson.com", + "active": true, + "password": "plaintext_password", + "user_permissions": { "IsAdmin": "admin" } +} +``` + + + +If you are modifying a user password, you will need to include an access_key in the body of your request. This can be obtained from doing a GET to the same Resource URL. + + + +**Sample Response** + + +```json +{ + "Status": "OK", + "Message": "User updated", + "Meta": "" +} +``` + +### Single Sign On API + +The Dashboard Admin SSO API endpoint allows you to implement custom authentication schemes for the Dashboard and Portal. Our Tyk Identity Broker (TIB) internally also uses this API. See [Single Sign On](/api-management/external-service-integration#single-sign-on-sso) for more details. + +#### Generate authentication token + +The Dashboard exposes the `/admin/sso` Admin API which allows you to generate a temporary authentication token, valid for 60 seconds. + +You should provide JSON payload with the following data: + +* `ForSection` - scope with possible values of `"dashboard"` or `"portal"` +* `OrgID` - with your organization id. +* `GroupID` - the group id +* `EmailAddress` - user email + + +| **Property** | **Description** | +| :------------ | :---------------------------- | +| Resource URL | `/admin/sso` | +| Method | POST | +| Body | `{"ForSection":"", "OrgID": "", "GroupID": ""}` | + +**Sample Request** + +```http expandable +POST /admin/sso HTTP/1.1 +Host: localhost:3000 +admin-auth: 12345 + +{ + "ForSection": "dashboard", + "OrgID": "588b4f0bb275ff0001cc7471", + "EmailAddress": "name@somewhere.com", + "GroupID": "" +} +``` + +**Sample Response:** + +```json +{ + "Status": "OK", + "Message": "SSO Nonce created", + "Meta": "YTNiOGUzZjctYWZkYi00OTNhLTYwODItZTAzMDI3MjM0OTEw" +} +``` + +See [Single Sign On](/api-management/external-service-integration#single-sign-on-sso) documentation for how to use this token and more details. + +### URL Reload API + +Since the Dashboard can have multiple URLs associated with it. It is possible to force a URL reload by calling an API endpoint of the Dashboard API. + +#### Reload the Dashboard URLs + +| **Property** | **Description** | +| :------------ | :---------------------- | +| Resource URL | `/admin/system/reload` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +GET /admin/system/reload HTTP/1.1 +Host: localhost:3000 +admin-auth:12345 +``` + +**Sample Response** + +```json +{ + "status": "ok" +} +``` + +### Export Assets API + +To make Tyk installations more portable, the Export API enables you to export key configuration objects required to back-up and re-deploy a basic Tyk Pro installation. + + + +To enable this feature, the minimum required versions for the Gateway and Dashboard are v2.3 and v1.3.1.2, respectively. + + + +#### Export Organizations + +The organization object is the most fundamental object in a Tyk setup, all other ownership properties hang off the relationship between an organization and its APIs, Policies and API Tokens. + +| **Property** | **Description** | +| :------------ | :------------------------------- | +| Resource URL | `/admin/organisations/{ORG-ID}` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http +GET /admin/organisations/54bd0ad9ff4329b88985aafb HTTP/1.1 +Host: localhost:3000 +admin-auth: 12345 +``` + +**Sample Response** + + +```json expandable +{ + "id": "53ac07777cbb8c2d53000002", + "owner_name": "Test", + "owner_slug": "test", + "cname_enabled": true, + "cname": "my.domain.com", + "apis": [{ + "api_human_name": "API 2", + "api_id": "5fa2db834e07444f760b7ceb314209fb" + }, { + "api_human_name": "API 1", + "api_id": "7a6ddeca9244448a4233866938a0d6e2" + }, { + "api_human_name": "API 3", + "api_id": "109eacaa50b24b64651a1d4dce8ec385" + }], + "developer_quota": 123, + "developer_count": 21, + "event_options": { + "key_event": { + "webhook": "", + "email": "", + "redis": true + }, + "key_request_event": { + "webhook": "", + "email": "", + "redis": false + } + }, + "hybrid_enabled": false, + "ui": { + "languages": {}, + "hide_help": false, + "default_lang": "", + "login_page": {}, + "nav": {}, + "uptime": {}, + "portal_section": {}, + "designer": {}, + "dont_show_admin_sockets": false, + "dont_allow_license_management": false, + "dont_allow_license_management_view": false + } +} +``` + +#### Export APIs and Policies + +To export APIs and Policies you should use the standard `GET APIS` endpoint and `GET POLICIES` list endpoints. The output from these endpoints can be used by the [Import API](#import-assets-api). + +### Import Assets API + +The import API enables you to add *Organizations*, *APIs* and *Policies* back into a Tyk installation while retaining their base IDs so that they work together. + + + +To enable this feature, the minimum required versions for the Gateway and Dashboard are v2.3 and v1.3.1.2, respectively. + + + +#### Import Organization + +The [Organization object](#organizations) is the most fundamental object in a Tyk setup, all other ownership properties hang off the relationship between an Organization and its APIs, Policies and API Tokens. + +| **Property** | **Description** | +| :------------ | :---------------------------- | +| Resource URL | `admin/organisations/import` | +| Method | POST | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http expandable +POST /admin/organisations/import HTTP/1.1 +Host: localhost:3000 +admin-auth: 12345 + +{ + "id": "53ac07777cbb8c2d53000002", + "owner_name": "Test", + "owner_slug": "test", + "cname_enabled": true, + "cname": "my.domain.com", + "apis": [{ + "api_human_name": "API 2", + "api_id": "5fa2db834e07444f760b7ceb314209fb" + }, { + "api_human_name": "API 1", + "api_id": "7a6ddeca9244448a4233866938a0d6e2" + }, { + "api_human_name": "API 3", + "api_id": "109eacaa50b24b64651a1d4dce8ec385" + }], + "developer_quota": 123, + "developer_count": 21, + "event_options": { + "key_event": { + "webhook": "", + "email": "", + "redis": true + }, + "key_request_event": { + "webhook": "", + "email": "", + "redis": false + } + }, + "hybrid_enabled": false, + "ui": { + "languages": {}, + "hide_help": false, + "default_lang": "", + "login_page": {}, + "nav": {}, + "uptime": {}, + "portal_section": {}, + "designer": {}, + "dont_show_admin_sockets": false, + "dont_allow_license_management": false, + "dont_allow_license_management_view": false + } +} +``` + +#### Import APIs + +The import APIs operates on *lists* of APIs. + +| **Property** | **Description** | +| :------------ | :------------------- | +| Resource URL | `admin/apis/import` | +| Method | POST | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http expandable +POST /admin/apis/import HTTP/1.1 +Host: localhost:3000 +admin-auth: 12345 + +{ + "apis": [{ + "api_model": {}, + "api_definition": {...}, + "hook_references": [], + "is_site": false, + "sort_by": 0 +}, { + "api_model": {}, + "api_definition": {...}, + "hook_references": [], + "is_site": false, + "sort_by": 0 +}] +} +``` + +#### Import Policies + +The import Policies operates on *lists* of Policies. + +| **Property** | **Description** | +| :------------ | :----------------------- | +| Resource URL | `admin/policies/import` | +| Method | POST | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```http expandable +POST /admin/policies/import HTTP/1.1 +Host: localhost:3000 +admin-auth: 12345 + +{ + "Data": [{ + "_id": "57ed12fc30c55e6b890d37d8", + "access_rights": { + "5fa2db834e07444f760b7ceb314209fb": { + "allowed_urls": [], + "api_id": "5fa2db834e07444f760b7ceb314209fb", + "api_name": "New API 1", + "versions": ["Default"] + }, + "7a6ddeca9244448a4233866938a0d6e2": { + "allowed_urls": [], + "api_id": "7a6ddeca9244448a4233866938a0d6e2", + "api_name": "API1", + "versions": ["Default"] + } + }, + "active": true, + "date_created": "0001-01-01T00:00:00Z", + "hmac_enabled": false, + "is_inactive": false, + "key_expires_in": 0, + "last_updated": "1478791603", + "name": "Default", + "org_id": "53ac07777cbb8c2d53000002", + "partitions": { + "acl": false, + "quota": false, + "rate_limit": false + }, + "per": 60, + "quota_max": -1, + "quota_renewal_rate": 3600, + "rate": 1000, + "tags": [] + }, { + "_id": "5824343b30c55e52d5e6cfde", + "access_rights": { + "7a6ddeca9244448a4233866938a0d6e2": { + "allowed_urls": [], + "api_id": "7a6ddeca9244448a4233866938a0d6e2", + "api_name": "API 1", + "versions": ["Default"] + } + }, + "active": true, + "date_created": "0001-01-01T00:00:00Z", + "hmac_enabled": false, + "is_inactive": false, + "key_expires_in": 0, + "last_updated": "1478791761", + "name": "Test Policy", + "org_id": "53ac07777cbb8c2d53000002", + "partitions": { + "acl": false, + "quota": false, + "rate_limit": false + }, + "per": 1, + "quota_max": 100, + "quota_renewal_rate": 90000, + "rate": 10, + "tags": [] + }, { + "_id": "58172a2330c55e22afcd59af", + "access_rights": { + "109eacaa50b24b64651a1d4dce8ec385": { + "allowed_urls": [], + "api_id": "109eacaa50b24b64651a1d4dce8ec385", + "api_name": "API 3", + "versions": ["Default"] + }, + "5fa2db834e07444f760b7ceb314209fb": { + "allowed_urls": [], + "api_id": "5fa2db834e07444f760b7ceb314209fb", + "api_name": "API 2", + "versions": ["Default"] + }, + "7a6ddeca9244448a4233866938a0d6e2": { + "allowed_urls": [], + "api_id": "7a6ddeca9244448a4233866938a0d6e2", + "api_name": "API 1", + "versions": ["Default"] + }, + "d023576f836a4e407153009e8d95cf73": { + "allowed_urls": [], + "api_id": "d023576f836a4e407153009e8d95cf73", + "api_name": "Test API", + "versions": ["Default"] + } + }, + "active": true, + "date_created": "0001-01-01T00:00:00Z", + "hmac_enabled": false, + "is_inactive": false, + "key_expires_in": 2592000, + "last_updated": "1477913123", + "name": "Big Policy", + "org_id": "53ac07777cbb8c2d53000002", + "partitions": { + "acl": false, + "quota": false, + "rate_limit": false + }, + "per": 1, + "quota_max": 6000, + "quota_renewal_rate": 3600, + "rate": 10, + "tags": ["TEST-1", "TEST-2"] +}], + "Pages": 0 +} +``` + +## Exploring API Endpoint Designer + +### Classic APIs + +Tyk Dashboard's Endpoint Designer provides a graphical environment for the creation and update of your Tyk Classic APIs. + +The Endpoint Designer allows to configure all elements of your Tyk Classic API and consists of several tabs, plus a **Raw Definition** view which allows you to directly edit the Tyk Classic API Definition (in JSON format). Note that + +### Core Settings + +The Tyk Classic Endpoint Designer - Core Settings tab + +The **Core Settings** tab provides access to configure basic settings for the API: +- [Detailed logging](/api-management/logs-metrics#capturing-detailed-logs) +- API Settings including + - Listen path + - [API Categories](#governance-using-api-categories) +- Upstream settings including + - Upstream service (target) URL + - [Service Discovery](/planning-for-production/ensure-high-availability/service-discovery) +- [API Ownership](/api-management/user-management#api-ownership) +- [API level rate limiting](/api-management/rate-limit#configuring-the-rate-limiter-at-the-api-level) +- [Authentication](/api-management/client-authentication) + +### Versions + +The Tyk Classic Endpoint Designer - Versions tab + +The **Versions** tab allows you to create and manage [API versioning](/api-management/gateway-config-tyk-classic#tyk-classic-api-versioning) for the API. + +At the top of the Endpoint Designer, you can see which version you are currently editing. If you have more than one option, selecting it from the drop-down will load its endpoint configuration into the editor. + +### Endpoint Designer + +The Tyk Classic Endpoint Designer - Endpoint Designer tab + +The **Endpoint Designer** is where you can define endpoints for your API so that you can enable and configure Tyk middleware to [perform checks and transformations](/api-management/traffic-transformation) on the API traffic. + +In some cases, you will want to set global settings that affect all paths that are managed by Tyk. The **Global Version Settings** section will enable you to configure API-level [request](/api-management/traffic-transformation/request-headers#tyk-classic-api) and [response](/api-management/traffic-transformation/request-headers#tyk-classic-api) header transformation. + +### Advanced Options + +The Tyk Classic Endpoint Designer - Advanced Options tab + +The **Advanced Options** tab is where you can configure Tyk's other powerful features including: +- Upstream certificate management +- [API-level caching](/api-management/response-caching#configuring-the-cache-via-the-dashboard) including a button to invalidate (flush) the cache for the API +- [CORS](/api-management/gateway-config-tyk-classic#cross-origin-resource-sharing-cors) +- Add custom attributes to the API definition as *config data* that can be accessed by middleware +- Enable [context variables](/api-management/traffic-transformation/request-context-variables) so that they are extracted from requests and made available to middleware +- Manage *segment tags* if you are working with [sharded gateways](/api-management/multiple-environments#gateway-sharding) +- Manage client IP address [allow](/api-management/gateway-config-tyk-classic#ip-access-control) and [block](/api-management/gateway-config-tyk-classic#ip-access-control) lists +- Attach [webhooks](/api-management/gateway-events#event-handling-with-webhooks) that will be triggered for different events + +### Uptime Tests + +The Tyk Classic Endpoint Designer - Uptime Tests tab + +In the **Uptime Tests** tab you can configure Tyk's [Uptime Test](/api-management/gateway-config-tyk-classic#uptime-tests) functionality + +### Debugging + +The Tyk Classic Endpoint Designer - Debugging tab + +The **Debugging** tab allows you to test your endpoints before you publish or update them. You can also use it for testing any middleware plugins you have implemented. Any debugging you create will persist while still in the current API, enabling you to make changes in the rest of the API settings without losing the debugging scenario. + +The Debugging tab consists of the following sections: + +- Request +- Response +- Logs + +##### Request + +Debugging Request + +In this section, you can enter the following information: + +- Method - select the method for your test from the drop-down list +- Path - your endpoint to test +- Headers/Body - enter any header information, such as Authorization, etc. Enter any body information. For example, entering user information if creating/updating a user. + +Once you have entered all the requested information, click **Run**. Debugging Response and Log information will be displayed: + +##### Response + +Debugging Response + +The Response section shows the JSON response to your request. + +##### Logs + +Debugging Logs + +The debugging level is set to **debug** for the request. This outputs all logging information in the Endpoint Designer. In the Tyk Gateway logs you will see a single request. Any Error messages will be displayed at the bottom of the Logs output. + +## Traffic Analytics + +The Tyk Dashboard provides a full set of analytics functions and graphs that you can use to segment and view your API traffic and activity. The Dashboard offers a great way for you to debug your APIs and quickly pin down where errors might be cropping up and for which clients. + +[User Owned Analytics](/api-management/user-management#user-permissions), introduced in Tyk v5.1, can be used to limit the visibility of aggregate statistics to users when API Ownership is enabled. Due to the way that the analytics data are aggregated, not all statistics can be filtered by API and so may be inaccessible to users with the Owned Analytics permission. + + + +For the Tyk Dashboard's analytics functionality to work, you must configure both per-request and aggregated pumps for the database platform that you are using. For more details see the [Setup Dashboard Analytics](/api-management/tyk-pump#setup-dashboard-analytics) section. + + + + +## Analyzing API Traffic Activity + +### API Activity Dashboard + +The first screen (and main view) of the Tyk Dashboard will show you an overview of the aggregate usage of your APIs, this view includes the number of hits, the number of errors and the average latency over time for all of your APIs as an average: + +API Activity Dashboard + +You can toggle the graphs by clicking the circular toggles above the graph to isolate only the stats you want to see. + +Use the Start and End dates to set the range of the graph, and the version drop-down to select the API and version you wish to see traffic for. + +You can change the granularity of the data by selecting the granularity drop down (in the above screenshot: it is set to β€œDay”). + +The filter by tag option, in a graph view, will enable you to see the graph filtered by any tags you add to the search. + +Below the aggregate graph, you’ll see an error breakdown and endpoint popularity chart. These charts will show you the overall error type (and code) for your APIs as an aggregate and the popularity of the endpoints that are being targeted by your clients: + +Error Breakdown and Endpoints + + + +From Tyk v5.1 (and LTS patches v4.0.14 and v5.0.3) the Error Breakdown and Endpoint Popularity charts will not be visible to a user if they are assigned the [Owned Analytics](/api-management/user-management#user-permissions) permission. + + + +### Activity Logs + +When you look through your Dashboard and your error breakdown statistics, you'll find that you will want to drill down to the root cause of the errors. This is what the Log Browser is for. + +The Log Browser will isolate individual log lines in your analytics data set and allow you to filter them by: + +* API Name +* Token ID (hashed) +* Errors Only +* By Status Code + +You will be presented with a list of requests, and their metadata: + +Log Viewer + +Click a request to view its details. + +Log Viewer Details + +#### Self-Managed Installations Option + +In an Self-Managed installation, if you have request and response logging enabled, then you can also view the request payload and the response if it is available. +To enable request and response logging, please take a look at [useful debug modes](/api-management/troubleshooting-debugging#capturing-detailed-logs) . + +**A warning on detailed logging:** This mode generates a very large amount of data, and that data exponentially increases the size of your log data set, and may cause problems with delivering analytics in bulk to your MongoDB instances. This mode should only be used to debug your APIs for short periods of time. + + + +Detailed logging is not available for Tyk Cloud Classic customers. + + + +### Activity by API + +To get a tabular view of how your API traffic is performing, you can select the **Activity by API** option in the navigation and see a tabular view of your APIs. This table will list out your APIs by their traffic volume and you'll be able to see when they were last accessed: + +Activity per API + +You can use the same range selectors as with the Dashboard view to modify how you see the data. However, granularity and tag views will not work since they do not apply to a tabulated view. + +If you select an API name, you will be taken to the drill-down view for that specific API, here you will have a similar Dashboard as you do with the aggregate API Dashboard that you first visit on log in, but the whole view will be constrained to just the single API in question: + +Traffic per API: CLosed graph + +You will also see an error breakdown and the endpoint popularity stats for the API: + +API error breakdown pie chart + +Tyk will try to normalize endpoint metrics by identifying IDs and UUIDs in a URL string and replacing them with normalized tags, this can help make your analytics more useful. It is possible to configure custom tags in the configuration file of your Tyk Self-Managed or Multi-Cloud installation. + + + +From Tyk v5.1 (and LTS patches v4.0.14 and v5.0.3) the Error Breakdown and Endpoint Popularity charts will not be visible to a user if they are assigned the [Owned Analytics](/api-management/user-management#user-permissions) permission. + + + +### Activity by Key + +You will often want to see what individual keys are up to in Tyk, and you can do this with the **Activity per Key** section of your analytics Dashboard. This view will show a tabular layout of all keys that Tyk has seen in the range period and provide analytics for them: + +Activity per Token + +You'll notice in the screenshot above that the keys look completely different to the ones you can generate in the key designer (or via the API), this is because, by default, Tyk will hash all keys once they are created in order for them to not be snooped should your key-store be breached. + +This poses a problem though, and that is that the keys also no longer have any meaning as analytics entries. You'll notice in the screenshot above, one of the keys is appended by the text **TEST_ALIAS_KEY**. This is what we call an Alias, and you can add an alias to any key you generate and that information will be transposed into your analytics to make the information more human-readable. + +The key `00000000` is an empty token, or an open-request. If you have an API that is open, or a request generates an error before we can identify the API key, then it will be automatically assigned this nil value. + +If you select a key, you can get a drill down view of the activity of that key, and the errors and codes that the token has generated: + +Traffic activity by key graph + +Errors by Key + +(The filters in this view will not be of any use except to filter by API Version). + + + +From Tyk v5.1 (and LTS patches v4.0.14 and v5.0.3) the Traffic per Key screen will not be visible to a user if they are assigned the [Owned Analytics](/api-management/user-management#user-permissions) permission. + + + +### Activity by endpoint + +To get a tabular view of how your API traffic is performing at the endpoint level, you can select the Activity by Endpoint option in the navigation and see a tabular view of your API endpoints. This table will list your API endpoints by their traffic volume and you’ll be able to see when they were last accessed: + +Activity by endpoint + +#### Controlling which endpoints appear in the analytics data + +The aggregate pumps have an option to `track_all_paths` which will ensure that all analytics records generated by the Tyk Gateway will be included in the aggregated statistics on the Endpoint Popularity screen. Set this to `true` to capture all endpoints in the aggregated data and subsequently on the Dashboard page. + +You can alternatively select only a subset of the endpoints to include in the aggregated data by setting `track_all_paths` to `false` and identifying specific endpoints to be "tracked". These are identified by the `TrackPath` [flag](/api-management/tyk-pump#trackpath) being set to `true` in the record. In this configuration, the Pump will only include transaction records from "tracked" endpoints in the aggregated data. + +Tyk Gateway will set `TrackPath` to `true` in transaction records generated for endpoints that have the track endpoint middleware enabled. + + + +The *track endpoint* middleware only affects the inclusion of endpoints in the per-endpoint aggregates, it does not have any impact on other [aggregated data](/api-management/logs-metrics#aggregated-analytics) nor the [per-request data](/api-management/dashboard-configuration#activity-logs). + + + +#### Selecting Tyk OAS APIs endpoints to be tracked + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. The `path` can contain wildcards in the form of any string bracketed by curly braces, for example `{user_id}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +The track endpoint middleware (`trackEndpoint`) can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + +The `trackEndpoint` object has the following configuration: + - `enabled`: enable the middleware for the endpoint + +For example: +```json {hl_lines=["39-41"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-track-endpoint", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/anything": { + "get": { + "operationId": "anythingget", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-track-endpoint", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-track-endpoint/", + "strip": true + } + }, + "middleware": { + "operations": { + "anythingget": { + "trackEndpoint": { + "enabled": true + } + } + } + } + } +} +``` + +In this example the track endpoint middleware has been configured for requests to the `GET /anything` endpoint. These requests will appear in the Endpoint Popularity analytics screen, located within the API Usage section of Tyk Dashboard. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the track endpoint middleware. + +##### Configuring the middleware in the API Designer + +Adding the track endpoint middleware to your API endpoints is easy when using the API Designer in the Tyk Dashboard, simply follow these steps: + +1. **Add an endpoint** + + From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + + Tyk OAS API Designer showing no endpoints created + + Adding an endpoint to an API using the Tyk OAS API Designer + + Tyk OAS API Designer showing no middleware enabled on endpoint + +2. **Select the Track Endpoint middleware** + + Select **ADD MIDDLEWARE** and choose the **Track Endpoint** middleware from the *Add Middleware* screen. + + Adding the Track Endpoint middleware + +3. **Save the API** + + Select **SAVE API** to apply the changes to your API. + +#### Selecting Tyk Classic API endpoints to be tracked +If you are working with Tyk Classic APIs then you must add a new `track_endpoints` object to the `extended_paths` section of your API definition. + +The `track_endpoints` object has the following configuration: +- `path`: the endpoint path +- `method`: the endpoint HTTP method + +For example: +```.json {linenos=true, linenostart=1} expandable +{ + "extended_paths": { + "track_endpoints": [ + { + "disabled": false, + "path": "/anything", + "method": "GET", + } + ] + } +} +``` + +In this example the track endpoint middleware has been configured for HTTP `GET` requests to the `/anything` endpoint. These requests will appear in the Endpoint Popularity analytics screen, located within the API Usage section of Tyk Dashboard. + +##### Configuring the middleware in the API Designer + +You can use the API Designer in the Tyk Dashboard to configure the track endpoint middleware for your Tyk Classic API by following these steps. + +1. **Add an endpoint for the path and select the plugin** + + From the **Endpoint Designer** add an endpoint that matches the path for which you want to allow access. Select the **Track endpoint** plugin. + + Select the middleware + +2. **Save the API** + + Use the *save* or *create* buttons to save the changes and activate the middleware for the selected endpoint. + +### Activity by Location + +Tyk will attempt to record GeoIP based information based on your inbound traffic. This requires a MaxMind IP database to be available to Tyk and is limited to the accuracy of that database. + +You can view the overview of what the traffic breakdown looks like per country, and then drill down into the per-country traffic view by selecting a country code from the list: + +Geographic Distribution + + + +From Tyk v5.1 (and LTS patches v4.0.14 and v5.0.3) the Geographic Distribution screen will not be visible to a user if they are assigned the [Owned Analytics](/api-management/user-management#user-permissions) permission. + + + +**MaxMind Settings** + +To use a MaxMind database, see [MaxMind Database Settings](/tyk-oss-gateway/configuration#analytics_configenable_geo_ip) in the Tyk Gateway Configuration Options. + +### Activity by Error + +The error overview page limits the analytics down to errors only, and gives you a detailed look over the range of the number of errors that your APIs have generated. This view is very similar to the Dashboard, but will provide more detail on the error types: + +Error Overview + + + +From Tyk v5.1 (and LTS patches v4.0.14 and v5.0.3) the Errors by Category data will not be visible to a user if they are assigned the [Owned Analytics](/api-management/user-management#user-permissions) permission. + + + +### Activity by Oauth Client + +Traffic statistics are available on a per OAuth Client ID basis if you are using the OAuth mode for one of your APIs. To get a breakdown view of traffic aggregated to a Client ID, you will need to go to the **System Management -> APIs** section and then under the **OAuth API**, there will be a button called **OAuth API**. Selecting an OAuth client will then show its aggregate activity + +OAuth Client + +In the API list view – an **OAuth Clients** button will appear for OAuth enabled APIs, use this to browse to the Client ID and the associated analytics for that client ID: + +OAuth Client Analytics Data + +You can view the analytics of individual tokens generated by this Client ID in the regular token view. + + + +From Tyk v5.1 (and LTS patches v4.0.14 and v5.0.3) the Traffic per OAuth Client ID charts will not be visible to a user if they are assigned the [Owned Analytics](/api-management/user-management#user-permissions) permission. + + + +--- + +## Governance using API Categories + +API categorization is a governance feature provided within the Tyk Dashboard that helps you to manage a portfolio of APIs. You can filter the list of APIs visible in the Dashboard UI or to be returned by the Dashboard API by category. You can assign an API to any number of categories and any number of APIs to a category. All category names are entirely user defined. + +### When to use API categories +#### Managing a large portfolio of APIs +As a platform manager looking after a large portfolio of APIs, if I need to make changes to a sub-set of APIs, it's cumbersome having to identify which APIs they are and then to find them one-by-one in the list. If I have assigned categories to my APIs then I can filter quickly and easily to work with that sub-set. What's really powerful is that an API can appear in as many different categories as I like. + +#### Multi-tenant deployment +Multi-tenant deployments with [role-based access control](/api-management/user-management#) enabled allows an admin user to give different users or groups access to a sub-set of the entire API portfolio. Categories can be aligned with the API ownership rules that you have deployed to allow filtering the list of APIs for those visible to each separate user group/team. + +### How API categories work +API categories with Tyk are a very simple concept - you can define any string as a category and then tag the relevant APIs with that string. + +Categories might refer to the API's general focus (e.g. 'weather' or 'share prices'); they might relate to geographic location (e.g. 'APAC' or 'EMEA'); they might refer to technical markers (e.g. 'dev', 'test'); or anything else you might need. It's completely up to you. + +Categories can be defined, added to and removed from APIs without limitation. + +#### Tyk OAS APIs +When a Tyk OAS API is assigned to a category, the category name (string) is appended to a list in the database object where the API definition is stored by Tyk Dashboard. No change is made to the API definition itself. + +#### Tyk Classic APIs +When a Tyk Classic API is assigned to a category, the category name (string) is appended to the `name` field in the API definition using a `#` qualifier. For example, let's say you have an API with this (partial) API definition: + +```json +{ + "name": "my-classic-api" +} +``` +You can add it to the `global` and `staging` categories by updating the API definition to: + +```json +{ + "name": "my-classic-api #global #staging" +} +``` +When a Tyk Classic API is migrated from one environment to another using Tyk Sync, it will retain any category labels that it has been assigned. + + + +The use of the `#` qualifier to identify a category prevents the use of `#` in your API names; this is not an issue when working with Tyk OAS APIs. + + + +### Using API categories +API categories can be added and removed from APIs within the [API Designer](#api-designer), via the [Tyk Dashboard API](#tyk-dashboard-api), or via [Tyk Operator](/api-management/automations/operator#what-is-tyk-operator). + +#### API Designer +The API Designer in the Tyk Dashboard UI provides a simple method for assigning APIs to categories, removing categories and filtering the API list by category. + +##### Managing categories with Tyk OAS APIs +When working with Tyk OAS APIs, the API Designer has a separate **Access** tab where you can configure the categories to which the API is assigned. +Tyk OAS API Designer + +You can choose existing categories from the drop-down or define new categories simply by typing in the box. You can also remove the API from a category by clicking on the `x` or deleting the category from the box. +Managing categories for a Tyk OAS API + +##### Managing categories with Tyk Classic APIs +When working with Tyk Classic APIs, the API Designer has a box in the **API Settings** section where you can configure the categories to which the API is assigned. +Tyk Classic API Designer + +You can choose existing categories from the list that appears when you click in the box or you can define new categories simply by typing in the box. You can also remove the API from a category by clicking on the `x` or deleting the category from the box. +Managing categories for a Tyk Classic API + +##### Filtering the API list +When you have APIs assigned to categories, you can choose to view only the APIs in a specific category by using the **FILTER BY API CATEGORY** drop-down on the **Created APIs** screen. +View APIs in a category + +#### Tyk Dashboard API +The [Tyk Dashboard API](/tyk-dashboard-api) provides endpoints to manage categories directly, if you are not using the API Designer. + +When working with Tyk OAS APIs, you can manage categories for an API using these endpoints: + +| Method | Endpoint path | Action | +| :-------- | :-------------------------------------- | :---------------------------------------------------------------------------------------- | +| `PUT` | `/api/apis/oas/{apiID}/categories` | Assign a list of categories to the specified API +| `GET` | `/api/apis/oas/{apiID}/categories` | Retrieve the list of categories assigned to the specified API | + +When working with Tyk Classic APIs, you manage categories for an API by modifying the `name` field in the API definition and then updating the API in Tyk with that using these endpoints: + +| Method | Endpoint | Action | +| :-------- | :-------------------------------------- | :---------------------------------------------------------------------------------------- | +| `PUT` | `/api/apis/{apiID}` | Update the API definition for the specified API - CRUD category tags in the `name` field | +| `GET` | `/api/apis/{apiID}` | Retrieve the API definition for the specified API - category tags in `name` field | + +These endpoints will return information for categories across all APIs in the system (both Tyk OAS and Tyk Classic): + +| Method | Endpoint path | Action | +| :-------- | :-------------------------------------- | :---------------------------------------------------------------------------------------- | +| `GET` | `/api/apis/categories` | Retrieve a list of all categories defined in the system and the number of APIs in each | +| `GET` | `/api/apis?category={category_name}` | Retrieve a list of all APIs assigned to the specified category | + +#### Tyk Operator + +You can manage categories using Tyk Operator custom resources. Please refer to [Tyk Operator](/api-management/automations/operator#api-categories) documentation to see how to manage API categories for Tyk OAS APIs and Tyk Classic APIs. + +## Governance using API Templates + +API Templates are an API governance feature provided to streamline the process of creating Tyk OAS APIs. An API template is an asset managed by Tyk Dashboard that is used as the starting point - a blueprint - from which you can create a new Tyk OAS API definition. + +The default template is a blank API definition; your custom templates will contain some configuration, for example cache configuration or default endpoints with pre-configured middleware. When you create a new API using a custom template, whether importing an OpenAPI document or building the API from scratch in the Tyk API Designer, those elements of the API configuration included in the template will be pre-configured for you. + + + +API Templates are exclusive to [Tyk OAS APIs](/api-management/gateway-config-introduction#api-definitions) and can be managed via the Tyk Dashboard API or within the Tyk Dashboard UI. + + + +### When to use API templates +#### Gateway agnostic API design +When working with OpenAPI described upstream service APIs, your service developers do not need to learn about Tyk. You can create and maintain a suitable suite of templates that contain the Tyk-specific configuration (`x-tyk-api-gateway`) that you require for your externally published API portfolio. Creating an API on Tyk is as simple as importing the OpenAPI document and selecting the correct template. Tyk will combine the OpenAPI description with the template to produce a valid Tyk OAS API. + +#### Standardizing API configuration +If you have specific requirements for your external facing APIs - for example authentication, caching or even a healthcheck endpoint - you can define the appropriate API templates so that when APIs are created on Tyk these fields are automatically and correctly configured. + +### How API templating works +An API template is a blueprint from which you can build new APIs - it is an incomplete JSON representation of a Tyk OAS API definition that you can use as the starting point when creating a new API on Tyk. There is no limit to how much or how little of the API definition is pre-configured in the template (such that when you choose to create a new API without choosing a template, the blank API definition that you start from is itself a template). + +Templates are used only during the creation of an API, they cannot be applied later. Before you can use a template as the basis for an API, you must register the template with Tyk Dashboard. + +#### Structure of an API template +An API template asset has the following structure: + - `id`: a unique string type identifier for the template + - `kind`: the asset type, which is set to `oas-template` + - `name`: human-readable name for the template + - `description`: a short description of the template, that could be used for example to indicate the configuration held within the template + - `data`: a Tyk OAS API definition, the content of which will be used for templating APIs + - `_id`: a unique identifier assigned by Tyk when the template is registered in the Dashboard database + +#### Creating an API from a template +When you use a template during the [creation](/api-management/gateway-config-managing-oas#creating-an-api) of an API, the fields configured in `data` will be pre-set in your new API. You are able to modify these during and after creation of the template. No link is created between the API and the template, so changes made to the API will not impact the template. + +#### Merging with an OpenAPI description or Tyk OAS API definition +When you use a template during the creation of an API where you [import](/api-management/gateway-config-managing-oas#importing-an-openapi-description-to-create-an-api) the OpenAPI document or a full Tyk OAS API definition, the template is combined with the imported OAS description. If the `x-tyk-api-gateway` extension exists in the template, it will be applied to the newly created API. + +Where there are clashes between configuration in the OpenAPI description and the template: + - for maps, such as `paths` and `components`, new keys will be added alongside any existing ones from the template + - if a key in the OpenAPI description matches one in the template, the OpenAPI description takes precedence + - for array properties, such as `servers` and `tags`, values in the OpenAPI description will replace those in the template + +
+ +If you're using the API Designer in the Tyk Dashboard UI, then you can find details and examples of how to work with API templates [here](#working-with-api-templates-using-the-template-designer). + +If you're using the Tyk Dashboard API, then you can find details and examples of how to work with API templates [here](#working-with-api-templates-using-the-dashboard-api). + +### Working with API Templates using the Template Designer + +[API Templates](#governance-using-api-templates) are an API governance feature provided to streamline the process of creating Tyk OAS APIs. An API template is an asset managed by Tyk Dashboard that is used as the starting point - a blueprint - from which you can create a new Tyk OAS API definition. + +The Tyk Dashboard UI provides the following functionality to support working with API templates: + - Creating templates + - [new template](#creating-a-new-api-template) + - [from an existing API](#creating-a-template-from-an-existing-api) + - Using templates + - [when creating an API](#using-a-template-when-creating-a-new-api) + - [when importing an OpenAPI description or API definition](#using-a-template-when-importing-an-openapi-description-or-api-definition) + - [Managing templates](#managing-templates) + +API Templates can be found in the **API Templates** section of the **API Management** menu in the Tyk Dashboard. This screen lists all the templates currently registered with Tyk and displays their names and short descriptions. It also gives access to options to create and manage templates. + +API Templates + + + +API Templates are exclusive to [Tyk OAS APIs](/api-management/gateway-config-introduction#api-definitions). + + + +#### Creating templates +API templates can be created starting from a blank template or from an existing API + +##### Creating a new API template +To create a template, simply visit the **API Templates** section of the Tyk Dashboard and select **ADD TEMPLATE**. + +This will take you to the **Create API Template** screen, where you can configure all aspects of the template. + +The template does not need to be a complete or valid API definition however as a minimum: + - you must give the template a **Name** + - you must give the template a **Description** + +In this example, we have configured just the Name, Description, Gateway Status and Access settings: + +Configure the template + +When you have configured all of the API-level and endpoint-level settings you require, select **SAVE TEMPLATE** to create and register the template with Tyk. + +Returning to the **API Template** screen you will see your new template has been added to the list and assigned a unique `id` that can be used to access the template from the [Tyk Dashboard API](#structure-of-an-api-template): + +Template has been successfully created + +##### Creating a template from an existing API +You can use an existing API deployed on Tyk as the basis for a new API template - this is a great way to build up a portfolio of standardized APIs once you've got your first one correctly configured. + +From the **Created APIs** screen within the **APIs** section of the Tyk Dashboard, select the API that you wish to use as your starting point. In the **ACTIONS** drop-down select the **CREATE API TEMPLATE** option. + +Select Create API Template + +This will take you to the **Create API Template** screen, where you can configure all aspects of the template. + +The template does not need to be a complete or valid API definition however as a minimum: + - you must give the template a **Name** + - you must give the template a **Description** + +In this example, we have configured the Name and Description. The base API included response header transformation middleware on the `/anything` endpoint and API-level cache configuration, all of which will be configured within the template. + +Configure the template +Cache settings inherited from base API +Endpoint settings inherited from base API + +When you have configured all of the API-level and endpoint-level settings you require, select **SAVE TEMPLATE** to create and register the template with Tyk. + +Returning to the **API Template** screen you will see your new template has been added to the list and assigned a unique `id` that can be used to access the template from the [Tyk Dashboard API](#structure-of-an-api-template). + +Template has been successfully created + +#### Using templates +API templates are used as the starting point during the creation of a new API. They can be applied in all of the methods supported by Tyk for creating new APIs. + +##### Using a template when creating a new API +There are two ways to base a new API, created entirely within the Tyk Dashboard's API Designer, on a template that you've created and registered with Tyk. + +You can go from the **API Template** screen - for the template you want to use, select **CREATE API FROM TEMPLATE** from the **ACTIONS** menu: +Select Create API from template + +Or, from the **Created APIs** screen, select **ADD NEW API** as normal and then select the template you want to use from the **API Template** section: +Select the template you want to use + +Both of these routes will take you through to the API Designer, where the settings from your API template will be pre-configured. + +In this example, we applied "My first template" that we created [here](#creating-a-new-api-template). You can see that the Gateway Status and Access fields have been configured: +The API with template applied + +##### Using a template when importing an OpenAPI description or API definition +From the **Import API** screen, if you select the OpenAPI **type** then you can create an API from an OpenAPI description or Tyk OAS API definition; choose the appropriate method to provide this to the Dashboard: +- paste the JSON into the text editor +- provide a plain text file containing the JSON +- provide a URL to the JSON +Options when importing an OpenAPI description + +After pasting the JSON or locating the file, you can select the template you want to use from the **API Template** section: +Select the template you want to use + +In this example we used this simple OpenAPI description and selected "My second template" that we created [here](#creating-a-template-from-an-existing-api): + +``` json {linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "my-open-api-document", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "servers": [ + { + "url": "http://httpbin.org" + } + ], + "paths": { + "/xml": { + "get": { + "operationId": "xmlget", + "responses": { + "200": { + "description": "" + } + } + } + } + } +} +``` +The API that is created has both `/xml` and `/anything` endpoints defined, with API-level caching configured. You can see the API definition [here](https://gist.github.com/andyo-tyk/5d5cfeda404ce1ba498bbf4b9c105cf0). + +#### Managing templates +The Dashboard UI allows you to edit and delete templates after they have been created and registered with the Tyk Dashboard + +##### Editing a template +You can make changes to a template that has been registered with Tyk from the **API Templates** screen. For the template that you want to modify, simply select **EDIT TEMPLATE** from the **ACTIONS** menu: +Accessing the API template + +This will take you to the **API Template Details** screen where you can view the current template configuration. If you want to make changes, simply select **EDIT** to make the fields editable: +Modifying the API template + +Alternatively you can view and modify the raw JSON for the template by selecting **VIEW RAW TEMPLATE** from the **ACTIONS** menu: +Modifying the API template JSON + +You'll need to select **SAVE TEMPLATE** to apply your changes from either screen. + +##### Deleting a template +You can delete a template from your Tyk Dashboard from the **API Template Details** screen. This screen can be accessed by selecting the template from the **API Templates** screen (either by clicking on the template name, or selecting **EDIT TEMPLATE** from the **ACTIONS** menu): +Accessing the API template +Accessing the API template + +From the **API Template Details** screen you can select **DELETE TEMPLATE** from the **ACTIONS** menu: +Deleting the API template + + + +You will be asked to confirm the deletion, because this is irrevocable. Once confirmed, the template will be removed from the database and cannot be recovered. + + + +### Working with API Templates using the Dashboard API + +[API Templates](#governance-using-api-templates) are an API governance feature provided to streamline the process of creating Tyk OAS APIs. An API template is an asset managed by Tyk Dashboard that is used as the starting point - a blueprint - from which you can create a new Tyk OAS API definition. + +The Tyk Dashboard API provides the following functionality to support working with API templates: + - [registering a template with Tyk Dashboard](#registering-a-template-with-tyk-dashboard) + - [applying a template when creating an API from an OpenAPI document](#applying-a-template-when-creating-an-api-from-an-openapi-document) + - [applying a template when creating an API from a Tyk OAS API definition](#applying-a-template-when-creating-an-api-from-a-tyk-oas-api-definition) + + + + + API Templates are exclusive to [Tyk OAS APIs](/api-management/gateway-config-introduction#api-definitions). + + + +#### Structure of an API template +An API template asset has the following structure: + - `id`: a unique string type identifier for the template + - `kind`: the asset type, which is set to `oas-template` + - `name`: human-readable name for the template + - `description`: a short description of the template, that could be used for example to indicate the configuration held within the template + - `data`: a Tyk OAS API definition, the content of which will be used for templating APIs + - `_id`: a unique identifier assigned by Tyk when the template is registered in the Dashboard database + +#### Registering a template with Tyk Dashboard +To register an API template with Tyk, you pass the asset in the body of a `POST` request to the dashboard's `/api/assets` endpoint. + +For example, if you send this command to the endpoint: +``` bash {linenos=true, linenostart=1} expandable +curl --location 'http://localhost:3000/api/assets' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer d9957aff302b4f5e5596c86a685e63d8' \ +--data '{ + "kind": "oas-template", + "name": "my-template", + "description": "My first template", + "id": "my-unique-template-id", + "data": { + "info": { + "title": "", + "version": "" + }, + "openapi": "3.0.3", + "paths": { + "/anything": { + "post": { + "operationId": "anythingpost", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "middleware": { + "global": { + "cache": { + "enabled": true, + "timeout": 5, + "cacheAllSafeRequests": true + } + }, + "operations": { + "anythingpost": { + "requestSizeLimit": { + "enabled": true, + "value": 100 + } + } + } + } + } + } +}' +``` + +Tyk will respond with `HTTP 201 Created` and will provide this payload in response: +``` json expandable +{ + "Status": "success", + "Message": "asset created", + "Meta": "65e8c352cb71918520ff660c", + "ID": "my-unique-template-id" +} +``` + +Here `Meta` contains the database ID (where the asset has been registered in the persistent storage) and `ID` contains the unique identifier for the template. This unique identifier will be automatically generated by Tyk if none was provided in the `id` of the template asset provided in the `curl` request. + +#### Applying a template when creating an API from an OpenAPI document +When creating an API on Tyk using an OpenAPI document describing your upstream service, you can use the `/apis/oas/import` endpoint to import the OpenAPI description and apply it to your API. + +If you have a template registered with your Dashboard, you can use this as the starting point for your new API. Tyk will combine the OpenAPI document with the template, automating the configuration of any element in the Tyk OAS API definition as defined in your chosen template. + +You'll need to identify the template to be used during the import. You can use either its unique `id` or the database ID that was assigned when the template was [registered with Tyk Dashboard](#registering-a-template-with-tyk-dashboard). You provide either the `id` or `_id ` in the `templateID` query parameter in the call to `/oapis/oas/import`. + +For example: +``` bash {linenos=true, linenostart=1} expandable +curl --location 'http://localhost:3000/api/apis/oas/import?templateID=my-unique-template-id' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer ' \ +--data '{ + "components": {}, + "info": { + "title": "my-open-api-document", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "servers": [ + { + "url": "http://httpbin.org" + } + ], + "paths": { + "/xml": { + "get": { + "operationId": "xmlget", + "responses": { + "200": { + "description": "" + } + } + } + } + } +}' +``` +Tyk will respond with `HTTP 200 OK` and will provide this payload in response: +``` json expandable +{ + "Status": "OK", + "Message": "API created", + "Meta": "65e8c4f4cb71918520ff660d", + "ID": "970560005b564c4755f1db51ca5660e6" +} +``` + +Here `Meta` contains the database ID (where the API has been registered in the persistent storage) and `ID` contains the unique identifier for the API. This unique identifier will be automatically generated by Tyk as none was provided in the `id` field of the `x-tyk-api-gateway.info` field provided in the `curl` request. + +The new Tyk OAS API will have this definition, combining the OpenAPI description provided in the body of the `curl` request with the template with Id `my-unique-template-id`: +``` json {linenos=true, linenostart=1} expandable +{ + "info": { + "title": "my-open-api-document", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "servers": [ + { + "url": "http://localhost:8181/" + }, + { + "url": "http://httpbin.org" + } + ], + "security": [], + "paths": { + "/anything": { + "post": { + "operationId": "anythingpost", + "responses": { + "200": { + "description": "" + } + } + } + }, + "/xml": { + "get": { + "operationId": "xmlget", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "components": { + "securitySchemes": {} + }, + "x-tyk-api-gateway": { + "info": { + "dbId": "65e8c4f4cb71918520ff660d", + "id": "970560005b564c4755f1db51ca5660e6", + "orgId": "65d635966ec69461e0e7ee52", + "name": "my-open-api-document", + "state": { + "active": true, + "internal": false + } + }, + "middleware": { + "global": { + "cache": { + "cacheResponseCodes": [], + "cacheByHeaders": [], + "timeout": 5, + "cacheAllSafeRequests": true, + "enabled": true + } + }, + "operations": { + "anythingpost": { + "requestSizeLimit": { + "enabled": true, + "value": 100 + } + } + } + }, + "server": { + "listenPath": { + "strip": true, + "value": "/" + } + }, + "upstream": { + "url": "http://httpbin.org" + } + } +} +``` +Note that the `GET /xml` endpoint from the OpenAPI description and the `POST /anything` endpoint from the template (complete with `requestSizeLimit` middleware) have both been defined in the API definition. API-level caching has been enabled, as configured in the template. Tyk has included the `server` entry from the OpenAPI description (which points to the upstream server) and added the API URL on Tyk Gateway ([as explained here](/api-management/gateway-config-tyk-oas#modifying-the-openapi-description)). + +#### Applying a template when creating an API from a Tyk OAS API definition +When creating an API using a complete Tyk OAS API definition (which includes `x-tyk-api-gateway`), you can use the `/apis/oas` endpoint to import the API defintiion. + +If you have a template registered with your Dashboard, you can use this as the starting point for your new API. Tyk will combine the API definition with the template, automating the configuration of any element defined in your chosen template. + +You'll need to identify the template to be used during the import. You can use either its unique `id` or the database ID that was assigned when the template was [registered with Tyk Dashboard](#registering-a-template-with-tyk-dashboard). You provide either the `id` or `_id` in the `templateID` query parameter in the call to `/apis/oas`. + +For example: +``` bash {linenos=true, linenostart=1} expandable +curl --location 'http://localhost:3000/api/apis/oas?templateID=my-unique-template-id' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer ' \ +--data '{ + "components": {}, + "info": { + "title": "example-api", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/json": { + "get": { + "operationId": "jsonget", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-api", + "state": { + "active": true, + "internal": false + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "strip": true, + "value": "/example-api/" + } + }, + "middleware": { + "operations": { + "jsonget": { + "transformResponseHeaders": { + "enabled": true, + "add": [ + { + "name": "X-Foo", + "value": "bar" + } + ] + } + } + } + } + } +}' +``` +Tyk will respond with `HTTP 200 OK` and will provide this payload in response: +``` json expandable +{ + "Status": "OK", + "Message": "API created", + "Meta": "65e98ca5cb71918520ff6616", + "ID": "b8b693c5e28a49154659232ca615a7e8" +} +``` + +Here `Meta` contains the database ID (where the API has been registered in the persistent storage) and `ID` contains the unique identifier for the API. This unique identifier will be automatically generated by Tyk as none was provided in the `id` field of the `x-tyk-api-gateway.info` field provided in the `curl` request. + +The new Tyk OAS API will have this definition, combining the Tyk OAS API definition provided in the body of the `curl` request with the template with Id `my-unique-template-id`: + +``` json {linenos=true, linenostart=1} expandable +{ + "info": { + "title": "example-api", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "servers": [ + { + "url": "http://localhost:8181/example-api/" + } + ], + "security": [], + "paths": { + "/anything": { + "post": { + "operationId": "anythingpost", + "responses": { + "200": { + "description": "" + } + } + } + }, + "/json": { + "get": { + "operationId": "jsonget", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "components": { + "securitySchemes": {} + }, + "x-tyk-api-gateway": { + "info": { + "dbId": "65e98ca5cb71918520ff6616", + "id": "b8b693c5e28a49154659232ca615a7e8", + "orgId": "65d635966ec69461e0e7ee52", + "name": "example-api", + "state": { + "active": true, + "internal": false + } + }, + "middleware": { + "global": { + "cache": { + "cacheResponseCodes": [], + "cacheByHeaders": [], + "timeout": 5, + "cacheAllSafeRequests": true, + "enabled": true + } + }, + "operations": { + "anythingpost": { + "requestSizeLimit": { + "enabled": true, + "value": 100 + } + }, + "jsonget": { + "transformResponseHeaders": { + "enabled": true, + "add": [ + { + "name": "X-Foo", + "value": "bar" + } + ] + } + } + } + }, + "server": { + "listenPath": { + "strip": true, + "value": "/example-api/" + } + }, + "upstream": { + "url": "http://httpbin.org/" + } + } +} +``` +Note that the `GET /json` endpoint from the OpenAPI description and the `POST /anything` endpoint from the template (complete with `requestSizeLimit` middleware) have both been defined in the API definition. API-level caching has been enabled, as configured in the template. + +## Extend Permissions using Open Policy Agent (OPA) + +### Overview + +The Tyk Dashboard permission system can be extended by writing custom rules using an Open Policy Agent (OPA). The rules engine works on top of your Dashboard API, which means you can control not only access rules, but also behavior of all Dashboard APIs (except your public developer portal). + +To give you some inspiration here are some ideas of the rules you can implement now: + +* Enforce HTTP proxy option for all APIs for which the target URL does not point at the internal domain +* Control access for individual fields. For example, do not allow changing the API "active" status (e.g. deploy), unless you have a specific permission set (and make new permissions to be available to the Dashboard/API). Custom permissions can be creating using the [Additional Permissions API](/api-management/dashboard-configuration#additional-permissions-api) +* Have a user(or group) which has read access to one APIs and write to another +OPA rule engine put on top of Dashboard API, which means you can control the behavior of all APIs (except public developer portal) + +We have a video that demonstrates how our Open Policy Agent enables you to add custom permissions. + + + +#### Configuration + +By default the Dashboard OPA engine is turned off, and you need to explicitly enable it via your Dashboard `tyk_analytics.conf` file. +You can then control OPA functionality on a global level via your `tyk_analytics.conf` file, or at an organization level using either the [OPA API](/api-management/dashboard-configuration#open-policy-agent-api) or the [Dashboard](#using-the-open-policy-agent-in-the-dashboard). + +| Key | Type | Description | Example | +| :------------------------------------- | :--------------- | :------------------------------------------------------------------------------------------------------------------------ | :----------------------------- | +| security.open_policy.enabled | boolean | Toggle support for OPA | false | +| security.open_policy.debug | boolean | Enable debugging mode, prints a lot of information to the console | false | +| security.open_policy.enable_api | boolean | Enable access to the OPA API, even for users with Admin role | false | +| security.additional_permissions | string map | Add custom user/user_group permissions. You can use them in your rules, and they will be displayed in the Dashboard | `{"key": "human name"}` | + +#### Example + +```json expandable +"basic-config-and-security/security": { + "open_policy": { + "enabled":true, + "debug": true, + "enable_api": true + }, + "additional_permissions": {} +} +``` + + +With the OPA turned on, the majority of the security rules will be dynamically evaluated based on these rules. + +Additionally, users can modify OPA rules, and define their own, through the [OPA API](/api-management/dashboard-configuration#additional-permissions-api). For Self-Managed installations you can access and modify the OPA rules from your Tyk installation directory from [schemas/dashboard.rego](/api-management/dashboard-configuration#dashboard-opa-rules). +Moreover, using these rules you can also modify request content. Our recommendation is to use those modifications in a development environment and remember to create a backup of the rego rules. + +#### Language intro +The Open Policy Agent (OPA, pronounced β€œoh-pa”) is an open source, general-purpose policy engine that unifies policy enforcement across the stack. OPA provides a high-level declarative language (Rego) that lets you specify policy as code and simple APIs to offload policy decision-making from your software. (source: https://www.openpolicyagent.org/docs/latest/) + +#### What is Rego? +OPA policies are expressed in a high-level declarative language called Rego. Rego (pronounced β€œray-go”) is purpose-built for expressing policies over complex hierarchical data structures. For detailed information on Rego see the [Policy Language](https://www.openpolicyagent.org/docs/latest/policy-language) documentation. + +Rego was inspired by Datalog, which is a well understood, decades old query language. Rego extends Datalog to support structured document models such as JSON. + +Rego queries are assertions on data stored in OPA. These queries can be used to define policies that enumerate instances of data that violate the expected state of the system. + + +#### Why use Rego? +Use Rego for defining a policy that is easy to read and write. + +Rego focuses on providing powerful support for referencing nested documents and ensuring that queries are correct and unambiguous. + +Rego is declarative so policy authors can focus on what queries should return rather than how queries should be executed. These queries are simpler and more concise than the equivalent in an imperative language. + +Like other applications which support declarative query languages, OPA is able to optimize queries to improve performance. + +Rego supports a variety of statements and functions. You can even use things like HTTP calls to build policies that depends on third-party APIs. +See more about the language itself [here](https://www.openpolicyagent.org/docs/latest/policy-language/). + + +#### Tyk policy primitives +The main building block which is required for controlling access is a "deny" rule, which should return a detailed error in case of a rejection. You can specify multiple deny rules, and they will all be evaluated. If none of the rules was matched, user will be allowed to access the resource. + +A simple deny rule with a static error message can look like: + +```javascript +deny["User is not active"] { + not input.user.active +} +``` + +You can also specify a dynamic error message: + +```javascript +# None of the permissions was matched based on path +deny[x] { + count(request_permission) == 0 + x := sprintf("Unknown action '%v'", [input.request.path]) +} +``` + +In addition, to `deny` rules, you can also modify the requests using `patch_request`. +You should respond with a JSON merge patch format https://tools.ietf.org/html/rfc7396 +For example: + +```javascript expandable +# Example: Enforce http proxy configuration for an APIs with category #external. +patch_request[x] { + request_permission[_] == "apis" + request_intent == "write" + contains(input.request.body.api_definition.name, "#external") + + x := {"api_definition": {"proxy": {"transport": {"proxy_url": "http://company-proxy:8080"}}}} +} +``` + + +#### Getting Tyk Objects +In some cases, you may want to write a rule which is based on existing Tyk Object. +For example, you can write a rule for a policy API, which depends on the metadata of the API inside it. +The policy engine has access to the `TykAPIGet` function, which essentially just does a GET call to the Tyk Dashboard API. + +Example: + +```javascript +api := TykAPIGet("/apis/api/12345") +contains(api.target_url, "external.com") +``` + +Getting changeset of current request +For requests which modify the content, you can get a changeset (e.g. difference) using the `TykDiff` function, combined with a `TykAPIGet` call to get the original object. + +Example: + +```javascript expandable +# Example of the complex rule which forbids user to change API status, if he has some custom permission +deny["You are not allowed to change API status"] { + input.user.user_permissions["test_disable_deploy"] + + # Intent is to to update API + request_permission[_] == "apis" + request_intent == "write" + + # Lets get original API object, before update + # TykAPIGet accepts API url as argument, e.g. to receive API object call: TykAPIGet("/api/apis/") + api := TykAPIGet(input.request.path) + + # TykDiff performs Object diff and returns JSON Merge Patch document https://tools.ietf.org/html/rfc7396 + # For example if only state has changed diff may look like: {"api_definition":{"state": "active"}} + diff := TykDiff(api, input.request.body) + + # API state has changed + not is_null(diff.api_definition.active) +} +``` + +#### Developer guide +Since Opa rules are declarative, so in order to test them in the majority of the cases you can test your rules without using the Tyk Dashboard, and using this pre-build Rego playground https://play.openpolicyagent.org/p/x3ila2Q8Gb +When it comes to the `TykAPIGet` and `TykDiff` functions, you can mock them in your tests. + +In order to understand how the Dashboard evaluates the rules, you can enable debugging mode by setting the `security.open_policy.debug` option, and in the Dashboard logs, you will see the detailed output with input and output of the rule engine. It can be useful to copy-paste the Dashboard log output to the Rego playground, fix the issue, and validate it on the Dashboard. + +When you modify the `dashboard.opa` file, you will need to restart your tyk Dashboard. + +#### Using the Open Policy Agent in the Dashboard + +As well as configuring OPA rules through the API, admin users can view and edit OPA rules from within the Tyk Dashboard. The advantage of configuring your OPA rules in the Dashboard is that you can use a code editor for it, emulating a proper developer experience. There are two ways you can do this: + +1. From the **OPA Rules menu**. From the Dashboard Management menu, select OPA Rules. You can view and make any changes and select whether your OPA rules should be enabled or disabled. + +OPA Rules Menu + +2. From **Developer Tools**. Using the keyboard shortcut `CMD+SHIFT+D` (or `CTRL+SHIFT+D` for PC), you can open the Developer Tools panel on any page in the Dashboard and configure the permissions. Updates are applied in real-time. + + + + + OPA rules can only be accessed by admin role users in the Dashboard. + + + +OPA Floating UI +OPA screen + +### Dashboard OPA rules + +``` expandable +# Default OPA rules +package dashboard_users +default request_intent = "write" +request_intent = "read" { input.request.method == "GET" } +request_intent = "read" { input.request.method == "HEAD" } +request_intent = "delete" { input.request.method == "DELETE" } +# Set of rules to define which permission is required for a given request intent. +# read intent requires, at a minimum, the "read" permission +intent_match("read", "read") +intent_match("read", "write") +intent_match("read", "admin") +# write intent requires either the "write" or "admin" permission +intent_match("write", "write") +intent_match("write", "admin") +# delete intent requires either the "write" or "admin permission +intent_match("delete", "write") +intent_match("delete", "admin") +# Helper to check if the user has "admin" permissions +default is_admin = false +is_admin { + input.user.user_permissions["IsAdmin"] == "admin" +} +# Check if the request path matches any of the known permissions. +# input.permissions is an object passed from the Tyk Dashboard containing mapping between user permissions (β€œread”, β€œwrite” and β€œdeny”) and the endpoint associated with the permission. +# (eg. If β€œdeny” is the permission for Analytics, it means the user would be denied the ability to make a request to β€˜/api/usage’.) +# +# Example object: +# "permissions": [ +# { +# "permission": "analytics", +# "rx": "\\/api\\/usage" +# }, +# { +# "permission": "analytics", +# "rx": "\\/api\\/uptime" +# } +# .... +# ] +# +# The input.permissions object can be extended with additional permissions (eg. you could create a permission called β€˜Monitoring’ which gives β€œread” access to the analytics API β€˜/analytics’). +# This is can be achieved inside this script using the array.concat function. +request_permission[role] { + perm := input.permissions[_] + regex.match(perm.rx, input.request.path) + role := perm.permission +} +# --------- Start "deny" rules ----------- +# A deny object contains a detailed reason behind the denial. +default allow = false +allow { count(deny) == 0 } +deny["User is not active"] { + not input.user.active +} +# If a request to an endpoint does not match any defined permissions, the request will be denied. +deny[x] { + count(request_permission) == 0 + x := sprintf("This action is unknown. You do not have permission to access '%v'.", [input.request.path]) +} +deny[x] { + perm := request_permission[_] + not is_admin + not input.user.user_permissions[perm] + x := sprintf("You do not have permission to access '%v'.", [input.request.path]) +} +# Deny requests for non-admins if the intent does not match or does not exist. +deny[x] { + perm := request_permission[_] + not is_admin + not intent_match(request_intent, input.user.user_permissions[perm]) + x := sprintf("You do not have permission to carry out '%v' operation.", [request_intent, input.request.path]) +} +# If the "deny" rule is found, deny the operation for admins +deny[x] { + perm := request_permission[_] + is_admin + input.user.user_permissions[perm] == "deny" + x := sprintf("You do not have permission to carry out '%v' operation.", [request_intent, input.request.path]) +} +# Do not allow users (excluding admin users) to reset the password of another user. +deny[x] { + request_permission[_] = "ResetPassword" + not is_admin + user_id := split(input.request.path, "/")[3] + user_id != input.user.id + x := sprintf("You do not have permission to reset the password for other users.", [user_id]) +} +# Do not allow admin users to reset passwords if it is not allowed in the global config +deny[x] { + request_permission[_] == "ResetPassword" + is_admin + not input.config.allow_admin_reset_password + not input.user.user_permissions["ResetPassword"] + x := "You do not have permission to reset the password for other users. As an admin user, this permission can be modified using OPA rules." +} +# --------- End "deny" rules ---------- +################################################################################################################## +# Demo Section: Examples of rule capabilities. # +# The rules below are not executed until additional permissions have been assigned to the user or user group. # +################################################################################################################## +# If you are testing using OPA playground, you can mock Tyk functions like this: +# +# TykAPIGet(path) = {} +# TykDiff(o1,o2) = {} +# +# You can use this pre-built playground: https://play.openpolicyagent.org/p/T1Rcz5Ugnb +# Example: Deny users the ability to change the API status with an additional permission. +# Note: This rule will not be executed unless the additional permission is set. +deny["You do not have permission to change the API status."] { + # Checks the additional user permission enabled with tyk_analytics config: `"additional_permissions":["test_disable_deploy"]` + input.user.user_permissions["test_disable_deploy"] + # Checks the request intent is to update the API + request_permission[_] == "apis" + request_intent == "write" + # Checks if the user is attempting to update the field for API status. + # TykAPIGet accepts API URL as an argument, e.g. to receive API object call: TykAPIGet("/api/apis/") + api := TykAPIGet(input.request.path) + # TykDiff performs Object diff and returns JSON Merge Patch document https://tools.ietf.org/html/rfc7396 + # eg. If only the state has changed, the diff may look like: {"active": true} + diff := TykDiff(api, input.request.body) + # Checks if API state has changed. + not is_null(diff.api_definition.active) +} +# Using the patch_request helper you can modify the content of the request +# You should respond with JSON merge patch. +# See https://tools.ietf.org/html/rfc7396 for more details +# +# Example: Modify data under a certain condition by enforcing http proxy configuration for all APIs with the #external category. +patch_request[x] { + # Enforce only for users with ["test_patch_request"] permissions. + # Remove the ["test_patch_request"] permission to enforce the proxy configuration for all users instead of those with the permission. + input.user.user_permissions["test_patch_request"] + request_permission[_] == "apis" + request_intent == "write" + contains(input.request.body.api_definition.name, "#external") + x := {"api_definition": {"proxy": {"transport": {"proxy_url": "http://company-proxy:8080"}}}} +} +# You can create additional permissions for not only individual users, but also user groups in your rules. +deny["Only '%v' group has permission to access this API"] { + # Checks for the additional user permission enabled with tyk_analytics config: '"additional_permissions":["test_admin_usergroup"] + input.user.user_permissions["test_admin_usergroup"] + # Checks that the request intent is to access the API. + request_permission[_] == "apis" + api := TykAPIGet(input.request.path) + # Checks that the API being accessed has the category #admin-teamA + contains(input.request.body.api_definition.name, "#admin-teamA") + # Checks for the user group name. + not input.user.group_name == "TeamA-Admin" +} + +``` + +### Configuring Open Policy Agent Rules + +This is an end-to-end worked example showing how to configure Open Policy Agent rules with some [additional permissions](/api-management/dashboard-configuration#additional-permissions-api). + +#### Use Case + +Tyk's [RBAC](/api-management/user-management) includes out of the box permissions to Write, Read, and Deny access to API Definitions, but what if we want to distinguish between those users who can create APIs and those users who can edit or update APIs? Essentially, we want to extend Tyk's out of the box RBAC to include more fine grained permissions that prevent an `API Editor` role from creating new APIs, but allow them to edit or update existing APIs. + +#### High Level Steps + +The high level steps to realize this use case are as follows: + +1. Create additional permissions using API +2. Create user +3. Add Open Policy Agent Rule +4. Test new rule + + +#### Create additional permissions + +To include the `API Editor` role with additional permissions, send a PUT Request to the [Dashboard Additional Permissions API endpoint](/api-management/dashboard-configuration#additional-permissions-api) `/api/org/permissions` + +**Sample Request** + +In order to add the new role/permissions use the following payload. + +```console expandable +PUT /api/org/permissions HTTP/1.1 +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "additional_permissions": { + "api_editor": "API Editor" + } +} +``` + +**Sample Response** + +```json +{ + "Status": "OK", + "Message": "Additional Permissions updated in org level", + "Meta": null +} +``` + +
+ + +Remember to set the `authorization` header to your Tyk Dashboard API Access Credentials secret, obtained from your user profile on the Dashboard UI. + +This assumes no other additional permissions already exist. If you're adding to existing permissions you'll want to send a GET to `/api/org/permissions` first, and then add the new permission to the existing list. + + + +#### Create user +In the Dashboard UI, navigate to System Management -> Users, and hit the `Add User` button. Create a user that has API `Write` access and the newly created `API Editor` permission, e.g. + +User with Additional Permission + +##### Add Open Policy Agent (OPA) Rule + +In the Dashboard UI, navigate to Dashboard Management -> OPA Rules + +Edit the rules to add the following: + +``` expandable +request_intent = "create" { input.request.method == "POST" } +request_intent = "update" { input.request.method == "PUT" } + + +# Editor and Creator intent +intent_match("create", "write") +intent_match("update", "write") + + +# API Editors not allowed to create APIs +deny[x] { + input.user.user_permissions["api_editor"] + request_permission[_] == "apis" + request_intent == "create" + x := "API Editors not allowed to create APIs." +} +``` + +Updated Default OPA Rules incorporating the above rules as follows: + +```bash expandable +# Default OPA rules +package dashboard_users +default request_intent = "write" +request_intent = "read" { input.request.method == "GET" } +request_intent = "read" { input.request.method == "HEAD" } +request_intent = "delete" { input.request.method == "DELETE" } +request_intent = "create" { input.request.method == "POST" } +request_intent = "update" { input.request.method == "PUT" } +# Set of rules to define which permission is required for a given request intent. +# read intent requires, at a minimum, the "read" permission +intent_match("read", "read") +intent_match("read", "write") +intent_match("read", "admin") +# write intent requires either the "write" or "admin" permission +intent_match("write", "write") +intent_match("write", "admin") +# delete intent requires either the "write" or "admin permission +intent_match("delete", "write") +intent_match("delete", "admin") +# Editor and Creator intent +intent_match("create", "write") +intent_match("update", "write") +# Helper to check if the user has "admin" permissions +default is_admin = false +is_admin { + input.user.user_permissions["IsAdmin"] == "admin" +} +# Check if the request path matches any of the known permissions. +# input.permissions is an object passed from the Tyk Dashboard containing mapping between user permissions (β€œread”, β€œwrite” and β€œdeny”) and the endpoint associated with the permission. +# (eg. If β€œdeny” is the permission for Analytics, it means the user would be denied the ability to make a request to β€˜/api/usage’.) +# +# Example object: +# "permissions": [ +# { +# "permission": "analytics", +# "rx": "\\/api\\/usage" +# }, +# { +# "permission": "analytics", +# "rx": "\\/api\\/uptime" +# } +# .... +# ] +# +# The input.permissions object can be extended with additional permissions (eg. you could create a permission called β€˜Monitoring’ which gives β€œread” access to the analytics API β€˜/analytics’). +# This is can be achieved inside this script using the array.concat function. +request_permission[role] { + perm := input.permissions[_] + regex.match(perm.rx, input.request.path) + role := perm.permission +} +# --------- Start "deny" rules ----------- +# A deny object contains a detailed reason behind the denial. +default allow = false +allow { count(deny) == 0 } +deny["User is not active"] { + not input.user.active +} +# If a request to an endpoint does not match any defined permissions, the request will be denied. +deny[x] { + count(request_permission) == 0 + x := sprintf("This action is unknown. You do not have permission to access '%v'.", [input.request.path]) +} +deny[x] { + perm := request_permission[_] + perm != "ResetPassword" + not is_admin + not input.user.user_permissions[perm] + x := sprintf("You do not have permission to access '%v'.", [input.request.path]) +} +# Deny requests for non-admins if the intent does not match or does not exist. +deny[x] { + perm := request_permission[_] + not is_admin + not intent_match(request_intent, input.user.user_permissions[perm]) + x := sprintf("You do not have permission to carry out '%v' operation.", [request_intent, input.request.path]) +} +# If the "deny" rule is found, deny the operation for admins +deny[x] { + perm := request_permission[_] + is_admin + input.user.user_permissions[perm] == "deny" + x := sprintf("You do not have permission to carry out '%v' operation.", [request_intent, input.request.path]) +} +# Do not allow users (excluding admin users) to reset the password of another user. +deny[x] { + request_permission[_] = "ResetPassword" + not is_admin + user_id := split(input.request.path, "/")[3] + user_id != input.user.id + x := sprintf("You do not have permission to reset the password for other users.", [user_id]) +} +# Do not allow admin users to reset passwords if it is not allowed in the global config +deny[x] { + request_permission[_] == "ResetPassword" + is_admin + not input.config.security.allow_admin_reset_password + not input.user.user_permissions["ResetPassword"] + x := "You do not have permission to reset the password for other users. As an admin user, this permission can be modified using OPA rules." +} +# API Editors not allowed to create APIs +deny[x] { + input.user.user_permissions["api_editor"] + request_permission[_] == "apis" + request_intent == "create" + x := "API Editors not allowed to create APIs." +} +# --------- End "deny" rules ---------- +################################################################################################################## +# Demo Section: Examples of rule capabilities. # +# The rules below are not executed until additional permissions have been assigned to the user or user group. # +################################################################################################################## +# If you are testing using OPA playground, you can mock Tyk functions like this: +# +# TykAPIGet(path) = {} +# TykDiff(o1,o2) = {} +# +# You can use this pre-built playground: https://play.openpolicyagent.org/p/T1Rcz5Ugnb +# Example: Deny users the ability to change the API status with an additional permission. +# Note: This rule will not be executed unless the additional permission is set. +deny["You do not have permission to change the API status."] { + # Checks the additional user permission enabled with tyk_analytics config: `"additional_permissions":["test_disable_deploy"]` + input.user.user_permissions["test_disable_deploy"] + # Checks the request intent is to update the API + request_permission[_] == "apis" + request_intent == "write" + # Checks if the user is attempting to update the field for API status. + # TykAPIGet accepts API URL as an argument, e.g. to receive API object call: TykAPIGet("/api/apis/") + api := TykAPIGet(input.request.path) + # TykDiff performs Object diff and returns JSON Merge Patch document https://tools.ietf.org/html/rfc7396 + # eg. If only the state has changed, the diff may look like: {"active": true} + diff := TykDiff(api, input.request.body) + # Checks if API state has changed. + not is_null(diff.api_definition.active) +} +# Using the patch_request helper you can modify the content of the request +# You should respond with JSON merge patch. +# See https://tools.ietf.org/html/rfc7396 for more details +# +# Example: Modify data under a certain condition by enforcing http proxy configuration for all APIs with the #external category. +patch_request[x] { + # Enforce only for users with ["test_patch_request"] permissions. + # Remove the ["test_patch_request"] permission to enforce the proxy configuration for all users instead of those with the permission. + input.user.user_permissions["test_patch_request"] + request_permission[_] == "apis" + request_intent == "write" + contains(input.request.body.api_definition.name, "#external") + x := {"api_definition": {"proxy": {"transport": {"proxy_url": "http://company-proxy:8080"}}}} +} +# You can create additional permissions for not only individual users, but also user groups in your rules. +deny["Only '%v' group has permission to access this API"] { + # Checks for the additional user permission enabled with tyk_analytics config: '"additional_permissions":["test_admin_usergroup"] + input.user.user_permissions["test_admin_usergroup"] + # Checks that the request intent is to access the API. + request_permission[_] == "apis" + api := TykAPIGet(input.request.path) + # Checks that the API being accessed has the category #admin-teamA + contains(input.request.body.api_definition.name, "#admin-teamA") + # Checks for the user group name. + not input.user.group_name == "TeamA-Admin" +} +``` + +#### Test +Login to the Dashboard UI as the new `API Editor` user and try to create a new API. You should see an `Access Denied` error message. Now try to update an existing API. This should be successful!! + + +## System Administration + +The Tyk Dashboard Admin API provides the following administrator level functions: + - Managing [organizations](#organizations). + - Creating initial [users](/api-management/dashboard-configuration#users-api-1) during boot-strapping of the system. + - Forcing a [URL reload](/api-management/dashboard-configuration#url-reload-api). + - [Exporting](#export-assets-api) and [importing](#import-assets-api) Tyk assets (orgs, APIs, policies) for backup or when migrating between environments. + - Setting up [SSO integration](#single-sign-on-api-1). + +### Organizations + +Many businesses have a complex structure, for example a lot of distinct departments where each department has its own teams. You might also need to deploy and manage multiple environments such as Production, Staging and QA for different stages in your product workflow. The Tyk Dashboard is multi-tenant capable which allows you to use a single Tyk Dashboard to host separate *organizations* for each team or environment. + +An Organization is a completely isolated unit, and has its own: + - API Definitions + - API Keys + - Users + - Developers + - Domain + - Tyk Classic Portal + +When bootstrapping your Dashboard, the first thing the bootstrap script does is to create a new default Organization. + +Additional organizations can be created and managed using the [Dashboard Admin API](#organizations-api). + +#### Tyk Gateway and organizations +The concept of an organization does not exist within the Tyk Gateway. Gateways only proxy and validate the rules imposed on them by the definitions and keys that are being processed, however at their core there are some security checks within the Gateway that ensure organizational ownership of objects. + +Tyk allows each organization to own its own set of Gateways, for example when you want to use different hosting providers you can segregate them in terms of resources, or just for security reasons. + +Self-Managed users should use [API tagging](/api-management/multiple-environments#api-tagging-with-on-premises) and enforce a tagging standard across all organizations. + +All actions in a Self-Managed installation of Tyk must use a base Organization, and all actions should stem from a User owned by that organization. + + + +A user that does not belong to an Organization is sometimes referred to as an *unbounded user*. These users have visibility across all Organizations, but should be granted read-only access. + + + +### Dashboard Audit Logs + +The audit log system captures detailed records of all requests made to endpoints under the `/api` route. These audit logs can be stored either in files (in JSON or text format) or in the database, providing flexible options for log management and retrieval. + +Subsequently, if hosting Tyk Dashboard within a Kubernetes cluster, please ensure that the configured log file path is valid and writeable. + +The Tyk Dashboard config section contains an audit section for configuring audit logging behavior. An example is listed below. + +```yaml expandable + ... + "audit": { + "enabled": true, + "format": "json", + "path": "/tmp/audit.log", + "detailed_recording": false + }, + ... +``` + +#### Configuration Parameters + +| Parameter | Description | Default | +| :---- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :--------- | +| enabled | Enable audit logging. Setting `security.audit_log_path` also enables audit logging | true | +| format | Specifies audit log file format. Valid values are `json` and `text` | `text` | +| path | Path to the audit log. Overwrites `security.audit_log_path` if it was set | | +| detailed_recording | Enable detailed records in the audit log. If set to `true` then audit log records will contain the http-request (without body) and full http-response including the body | `false` | +| store_type | Specifies the storage in which audit logs will be written, valid values are `file` and `db`. | `file` | + +Please consult [Tyk Dashboard Configuration Options](/tyk-dashboard/configuration#audit) for equivalent configuration with environment variables. + +#### JSON File Format + +Audit records the following fields for `json` format: + +| Field | Description | +| :---- | :---- | +| req_id | Unique request ID | +| org_id | Organization ID | +| date | Date in *RFC1123* format | +| timestamp | UNIX timestamp | +| ip | IP address the request originated from | +| user | Dashboard user who performed the request | +| action | Description of the action performed (e.g. Update User) | +| method | HTTP request method | +| url | URL of the request | +| status | HTTP response status of the request | +| diff | Provides a diff of changed fields (available only for PUT requests) | +| request_dump | HTTP request copy (available if `detailed_recording` is set to `true`) | +| response_dump | HTTP response copy (available if `detailed_recording` is set to `true`) | + +#### Text File Format + +The `text` format outputs all fields as plain text separated with a new line and provided in the same order as `json` format. + +#### Database Storage Support + +In addition to file storage, audit logs can be stored in the main database (MongoDB or Postgres), this feature has been available since Tyk 5.7.0. To enable database storage set `audit.store_type` to `db`: + +```yaml expandable +... + "audit": { + "enabled": true, + "store_type": "db", + "detailed_recording": false + } +... +``` + +When `store_type` is set to `db`, audit logs will be stored in the main database storage instead of a file. + +#### Retrieving Audit Logs via API + +Since Tyk 5.7.0 a new API endpoint has been added to allow authorized users to retrieve audit logs from the database storage. To know more about the API specifications, check out the swagger [documentation](/tyk-dashboard-api). +To access the audit logs through the API ensure that your user account or group has been granted the "Audit Logs" RBAC group. If you do not have the necessary permissions, please contact your system administrator. + +## Supported Database + +Tyk Dashboard requires a persistent datastore for its operations. By default MongoDB is used. From Tyk v4.0, we also support PostgreSQL. + +### MongoDB Supported Versions + + + +#### Configuring MongoDB + +Please check [here](/planning-for-production/database-settings#mongodb) for MongoDB driver and production configurations. + +### PostgreSQL Supported Versions + + + + + +SQLite support will be deprecated from Tyk 5.7.0. To avoid disrupution, please transition to PostgreSQL, MongoDB or one of the listed compatible alternatives. + + + +#### Configuring PostgreSQL + +Please check [here](#configuring-postgresql) for production configurations. + +See the following pages for configuring your SQL installation with Tyk: + +* [Configuring Tyk Dashboard](#configuring-postgresql) +* [Configuring Tyk Pumps](#configuring-postgresql) + +All data stored in SQL platforms will be identical to our existing MongoDB support. + +### Which platform should you use? + + + +Tyk no longer supports SQLite as of Tyk 5.7.0. To avoid disruption, please transition to [PostgreSQL](/planning-for-production/database-settings#postgresql), [MongoDB](/planning-for-production/database-settings#mongodb), or one of the listed compatible alternatives. + + + +We recommend the following: + +* For PoC installations, you can use PostgreSQL or MongoDB. +* For production installations, we **only** support MongoDB or PostgreSQL + +## Data Storage Solutions + +Tyk stores a variety of data in 4 separate data storage layers. You can configure each layer separately to use one of our supported database platforms. Alternatively a single platform can be used for all layers. The 4 data storage layers are as follows: +1. **Main**: Stores configurations of: APIs, Policies, Users and User Groups. +2. **Aggregate Analytics**: Data used to display Dashboard charts and [analytics](#traffic-analytics). +3. **Logs**: When [detailed logging](/api-management/troubleshooting-debugging#capturing-detailed-logs) is enabled, request and response data is logged to storage. These logs can previewed in the Dashboard [log browser](#activity-logs). +4. **Uptime**: Uptime test analytics. + +Being extensible, Tyk supports storing this data across different databases (MongoDB, MySQL and PostgreSQL etc.). For example, Tyk can be configured to store analytics in PostgreSQL, logs in MongoDB and uptime data in MySQL. + +As illustrated below it can be seen that Tyk Pump writes to one or more external data sources via a Redis store. Conversely, Tyk Dashboard reads this data from the external data sources. + +Tyk Dashboard Pump Architecture + +The following details are required to manage this configuration: +- Data storage layer type +- Database engine +- Database connection string + +The remainder of this document explains how to configure Tyk Dashboard and Tyk Pump to read and write from one or more data storage layers, respectively. + +### Configure Dashboard to Read from a Data Storage Layer + +Tyk Dashboard has configuration environment variables for each data storage layer in the following format: + +```console +TYK_DB_STORAGE__TYPE +TYK_DB_STORAGE__CONNECTIONSTRING +``` + +where *LAYER* can be *ANALYTICS*, *LOGS* or *UPTIME*. + +For example, to configure Tyk Dashboard to read logs from a mongo database, the following environment variables are required: + +```console +TYK_DB_STORAGE_LOGS_TYPE=mongo +TYK_DB_STORAGE_LOGS_CONNECTIONSTRING=mongodb://db_host_name:27017/tyk_analytics +``` + +The full set of environment variables are listed below: + +```console expandable +TYK_DB_STORAGE_MAIN_TYPE +TYK_DB_STORAGE_MAIN_CONNECTIONSTRING +TYK_DB_STORAGE_LOGS_TYPE +TYK_DB_STORAGE_LOGS_CONNECTIONSTRING +TYK_DB_STORAGE_ANALYTICS_TYPE +TYK_DB_STORAGE_ANALYTICS_CONNECTIONSTRING +TYK_DB_STORAGE_UPTIME_TYPE +TYK_DB_STORAGE_UPTIME_CONNECTIONSTRING +``` + +It should be noted that Tyk will attempt to use the configuration for the *main* data storage layer when no corresponding configuration is available for logs, uptime or analytics. + +Please refer to the [storage configuration](/tyk-dashboard/configuration#storage) section to explore the parameters for configuring Tyk Dashboard to read from different storage layers. + + +### Configure Pump to Write to Data Storage Layers? + +Please consult the Pump configuration [guide](/api-management/tyk-pump#sql-uptime-pump) for an explanation of how to configure Tyk Pump to write to different storage layers. + +The remainder of this section explains the *environment variables* that can be used to configure Tyk Pump to write to the following data storage layers: +- Uptime +- Aggregated Analytics +- Logs + +#### Write Uptime Data + +Tyk Pump can be configured to write uptime data to SQL (Postgres and SQL Lite) and Mongo. The default behavior is to write to Mongo. + +##### PostgreSQL Database + +Tyk Pump can be configured to write to a PostgreSQL database, using the following environment variables: + +- *TYK_PMP_UPTIMEPUMPCONFIG_UPTIMETYPE*: Set to *sql* to configure Pump to store logs in a SQL based database. +- *TYK_PMP_UPTIMEPUMPCONFIG_TYPE*: Set to *postgres* to configure Pump to use a PostgreSQL database for uptime data. +- *TYK_PMP_UPTIMEPUMPCONFIG_CONNECTIONSTRING*: Set the connection string for the PostgreSQL database. + +An example configuration is shown below: + +```console +TYK_PMP_UPTIMEPUMPCONFIG_UPTIMETYPE=sql +TYK_PMP_UPTIMEPUMPCONFIG_TYPE=postgres +TYK_PMP_UPTIMEPUMPCONFIG_CONNECTIONSTRING=user=postgres password=topsecretpassword host=tyk-postgres port=5432 database=tyk_analytics +``` + +Further details for configuring an uptime SQL database are available [here](/tyk-pump/tyk-pump-configuration/tyk-pump-environment-variables#uptime_pump_configuptime_type) + +##### Mongo Database + +Tyk Pump can be configured to write to a Mongo database, using the following environment variables: + +- *TYK_PMP_UPTIMEPUMPCONFIG_UPTIMETYPE*: Set to *mongo* to configure Pump to store logs in a Mongo database. +- *TYK_PMP_UPTIMEPUMPCONFIG_MONGOURL*: Set to Mongo database connection string. +- *TYK_PMP_UPTIMEPUMPCONFIG_COLLECTIONNAME*: Set to the name of the collection used to store uptime analytics. + +```console +TYK_PMP_UPTIMEPUMPCONFIG_UPTIMETYPE=mongo +TYK_PMP_UPTIMEPUMPCONFIG_MONGOURL=mongodb://db_host_name:27017/tyk_uptime_db +TYK_PMP_UPTIMEPUMPCONFIG_COLLECTIONNAME=umptime_analytics +``` + +Further details for configuring a Tyk Mongo Pump are available [here](/tyk-pump/tyk-pump-configuration/tyk-pump-environment-variables#uptime_pump_config) + +#### Write Logs Data + +Tyk Pump can be configured to write logs to Mongo or SQL based databases. + +##### Mongo Database + +Tyk Pump can be configured to write to a Mongo database by setting the following environment variables: + +- *TYK_PMP_PUMPS_LOGS_TYPE*: Set to *mongo* to configure Pump to store logs in a Mongo database. +- *TYK_PMP_PUMPS_LOGS_META_MONGOURL*: Set the connection string for the Mongo database. +- *TYK_PMP_PUMPS_LOGS_META_COLLECTION_NAME*: Set the name of the collection that will store logs in the Mongo database. + +An example is listed below: + +```console +TYK_PMP_PUMPS_LOGS_TYPE=mongo +TYK_PMP_PUMPS_LOGS_META_MONGOURL=mongodb://tyk-mongo:27017/tyk_analytics +TYK_PMP_PUMPS_LOGS_META_COLLECTIONNAME=tyk_logs +``` + +##### PostgreSQL Database + +Tyk Pump can be configured to write to a PostgreSQL database by setting the following environment variables: + +- *TYK_PMP_PUMPS_LOGS_TYPE*: Set to *SQL* to configure Pump to store logs in a SQL based database. +- *TYK_PMP_PUMPS_LOGS_META_TYPE*: Set to *postgres* to configure Pump to store logs in a PostgreSQL database. +- *TYK_PMP_PUMPS_LOGS_META_CONNECTIONSTRING*: Set the name of the connection string for the PostgreSQL database. + +```console +TYK_PMP_PUMPS_LOGS_TYPE=SQL +TYK_PMP_PUMPS_LOGS_META_TYPE=postgres +TYK_PMP_PUMPS_LOGS_META_CONNECTIONSTRING=user=postgres password=topsecretpassword host=tyk-postgres port=5432 database=tyk_analytics +``` + +##### MySQL Database + +Tyk Pump can be configured to write to a MySQL database by setting the following environment variables: + +- *TYK_PMP_PUMPS_LOGS_TYPE*: Set to *SQL* to configure Pump to store logs in a SQL based database. +- *TYK_PMP_PUMPS_LOGS_META_TYPE*: Set to *mysql* to configure Pump to store logs in a MySQL database. +- *TYK_PMP_PUMPS_LOGS_META_CONNECTIONSTRING*: Set the name of the connection string for the MySQL database. + +```console +TYK_PMP_PUMPS_LOGS_TYPE=SQL +TYK_PMP_PUMPS_LOGS_META_TYPE=mysql +TYK_PMP_PUMPS_LOGS_META_CONNECTIONSTRING=mysql://db_host_name:3306/tyk_logs_db +``` + +#### Write Aggregated Analytics Data + +Aggregated analytics corresponds to data that is used for the display of charts and graphs in [dashboard](#traffic-analytics). Tyk Pump can be configured to write aggregated analytics data to SQL based databases or MongoDB. + +##### SQL Database + + + +Tyk no longer supports SQLite as of Tyk 5.7.0. To avoid disruption, please transition to [PostgreSQL](/planning-for-production/database-settings#postgresql), [MongoDB](/planning-for-production/database-settings#mongodb), or one of the listed compatible alternatives. + + + +Storage of aggregated analytics data has been tested with PostgreSQL and SqlLite databases. The following environment variables can be used to manage this configuration: + +- *TYK_PMP_PUMPS_SQLAGGREGATE_TYPE*: Set to *sql_aggregate* to configure Pump to store aggregated analytics data for charts and graphs in dashboard to a SQL based database. +- *TYK_PMP_PUMPS_SQLAGGREGATE_META_TYPE*: The database engine used to store aggregate analytics. Tested values are *postgres* or *sqlite*. +- *TYK_PMP_PUMPS_SQLAGGREGATE_META_CONNECTIONSTRING*: The connection string for the database that will store the aggregated analytics. + +The example below demonstrates how to configure Tyk Pump to write aggregated to a PostgreSQL database: + +```console +TYK_PMP_PUMPS_SQLAGGREGATE_TYPE=SQL +TYK_PMP_PUMPS_SQLAGGREGATE_META_TYPE=postgres +TYK_PMP_PUMPS_SQLAGGREGATE_META_CONNECTIONSTRING=user=postgres password=topsecretpassword host=tyk-postgres port=5432 database=tyk_aggregated_analytics +``` + +##### Mongo Database + +Tyk Pump can be configured to write aggregated analytics data to MongoDB. Aggregated analytics are written to a collection named `z_tyk_analyticz_aggregate_{ORG ID}`, where *ORG_ID* corresponds to the ID of your organization assigned by Tyk. + +The following environment variables can be used as a minimum to manage this configuration: + +- *TYK_PMP_PUMPS_MONGOAGGREGATE_TYPE*: Set to *mongo-pump-aggregate* to configure Pump to store aggregated analytics data in a MongoDB database. +- *TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGOURL*: Mongo database connection URL. + +An example is given below: + +```console +- TYK_PMP_PUMPS_MONGOAGGREGATE_TYPE=mongo-pump-aggregate +- TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGOURL=mongodb://db_host_name:27017/tyk_aggregated_analytics_db +``` diff --git a/api-management/data-graph.mdx b/api-management/data-graph.mdx new file mode 100644 index 00000000..20b6b6f8 --- /dev/null +++ b/api-management/data-graph.mdx @@ -0,0 +1,1883 @@ +--- +title: "Universal Data Graph" +'og:description': "How to Configure data graph" +tags: ['UDG', 'Universal Data Graph', 'Datasource', 'Concepts', 'Arguments', 'Field Mapping', 'Header Management', 'Graphql', 'Kafka', 'Rest', 'Examples'] +sidebarTitle: "Universal Data Graph (UDG)" +--- + +## Overview + +The Universal Data Graph (UDG) lets you combine multiple APIs into one universal interface. +With the help of GraphQL you're able to access multiple APIs with a single query. + +It's important to note that you don't even have to build your own GraphQL server. +If you have existing REST APIs all you have to do is configure the UDG. + +With the Universal Data Graph Tyk becomes your central integration point for all your internal as well as external APIs. +In addition to this, the UDG benefits from all existing solutions that already come with your Tyk installation. +That is, your Data Graph will be secure from the start and there's a large array of middleware you can build on to power your Graph. + +Universal Datagraph Overview + +Currently supported DataSources: +- REST +- GraphQL +- SOAP (through the REST datasource) +- Kafka + + + + + To start creating your first Universal Data Graph in Tyk Dashboard, go to "Data Graphs" section of the menu. + + + +Make sure to check some of the resources to help you start: +- [How to create UDG schema](/api-management/data-graph#creating-schema) +- [How to connect data sources](/api-management/data-graph#connect-datasource) +- [How to secure the data graph](/api-management/data-graph#security) + +## Key Concepts + +### Universal Data Graph + +The Universal Data Graph (UDG) introduces a few concepts you should fully understand in order to make full use of it. + +UDG comes with a fully spec compliant GraphQL engine that you don't have to code, you just have to configure it. + +For that you have to define your "DataSources" and might want to add "Field Mappings" as well as "Arguments" to your configuration. +Read on in the sub sections to understand the full picture to use UDG to its full potential. + +To help you, we have put together the following video. + + + +### DataSources + +In most GraphQL implementations you have the concept of Resolvers. +Resolvers are functions that take optional parameters and return (resolve) some data. +Each resolver is attached to a specific type and field. + +DataSources are similar in that they are responsible for loading the data for a certain field and type. +The difference is that with DataSources you simply configure how the engine should fetch the data whereas with traditional GraphQL frameworks you have to implement the function on your own. + +DataSources can be internal as well as external. + +Internal DataSources are APIs that are already managed by Tyk, such as REST or SOAP services configured through the Dashboard. +You can take advantage of Tyk’s rich middleware ecosystem to validate and transform requests and responses for these internal DataSources. + +External DataSources are APIs that you’re not currently managing through Tyk. +For simplicity, you can add them to your data graph without first configuring them as dedicated APIs in Tyk. +If you later decide to apply middleware or other policies, you can easily transition an external DataSource into a managed internal API. + +Head over to the [connect data source](/api-management/data-graph#udg) section to learn about the supported data sources and how to connect them to Tyk. + +### Arguments + +Looking back at the example from the "Field Mappings", you might wonder how to use the "id" argument from the GraphQL query to make the correct REST API call to the user service. + +Here's the schema again: + +```graphql expandable +type Query { + user(id: Int!): User +} + +type User { + id: Int! + name: String +} +``` + +We assume you already have your DataSource attached and now want to configure it so that the path argument gets propagated accordingly. +You need to tell the GraphQL engine that when it comes to resolving the field "user", take the argument with the name "id" and use it in the URL to make the request to the REST API. +You do this by using templating syntax to inject it into the URL. +This is done from the "Configure data source" tab, which will show after clicking a schema argument or object field. +Typing an opening curly brace ( `{` ) will produce a dropdown that contains all available fields and arguments. + +```html +https://example.com/user/{{ .arguments.id }} +``` + +Create New API + +### Field Mappings + +Universal Data Graph can automatically resolve where data source information should go in the GraphQL response as long as the GraphQL schema mirrors the data source response structure. + +Let's assume you have a REST API with a user resource like this: `http://example.com/users/:id` + +The following is an example response: + +```json +{ + "id": 1, + "name": "Martin Buhr" +} +``` + +If GraphQL schema in UDG is set as the following: +```graphql expandable +type Query { + user(id: Int!): User +} + +type User { + id: Int! + name: String +} +``` +and REST data source at attached behind `user(id: Int!)` query, UDG will be able to automatically resolve where `id` and `name` values should be in UDG response. In this case no field mapping is necessary. + + + +GraphQL does not support field names with hyphens (e.g. `"user-name"`). This can be resolved by using field mappings as described below. + + + +Let's assume that the JSON response looked a little different: + +````json +{ + "id": 1, + "user_name": "Martin Buhr" +} +```` + +If this were the JSON response you received from the REST API, you must modify the path for the field "name". +This is achieved by unchecking the "Disable field mapping" checkbox and setting the Path to "user_name". + +Nested paths can be defined using a period ( . ) to separate each segment of the JSON path, *e.g.*, "name.full_name" + +In cases where the JSON response from the data source is wrapped with `[]` like this: + +```json expandable +[ + { + "id": 1, + "name": "Martin Buhr", + "phone-number": "+12 3456 7890" + } +] +``` +UDG will not be able to automatically parse `id`, `name` and `phone-number` and fields mapping needs to be used as well. To get the response from inside the brackets the following syntax has to be used in field mapping: `[0]`. + +It is also possible to use this syntax for nested paths. For example: `[0].user.phone-number` + +#### Field mapping in Tyk Dashboard + +See below how to configure the field mapping for each individual field. + +Field mapping UI + + +#### Field mapping in Tyk API definition + +If you're working with raw Tyk API definition the field mapping settings look like this: + +```json expandable +{"graphql": { + "engine": { + "field_configs": [ + { + "type_name": "User", + "field_name": "phoneNumber", + "disable_default_mapping": false, + "path": [ + "[0]", + "user", + "phone-number" + ] + } + ] + } + } + } +``` + +Notice that even though in Tyk Dashboard the nested path has a syntax with ( . ), in Tyk API definition it becomes an array of strings. + +There's more UDG concepts that would be good to understand when using it for the first time: +* [UDG Arguments](/api-management/data-graph#arguments) +* [UDG Datasources](/api-management/data-graph#udg) + +### Reusing response fields + +When using the UDG, there may be a situation where you want to access an API with data coming from another API. +Consider the following REST APIs: + + - REST API for people: `https://people-api.dev/people` + - REST API for a specific person: `https://people-api.dev/people/{person_id}` + - REST API for driver licenses: `https://driver-license-api.dev/driver-licenses/{driver_license_id}` + +The REST API for a person will give us the following response: +```json expandable +{ + "id": 1, + "name": "John Doe", + "age": 40, + "driverLicenseID": "DL1234" +} +``` + +And the REST API response for driver licenses looks like this: +```json +{ + "id": "DL1234", + "issuedBy": "United Kingdom", + "validUntil": "2040-01-01" +} +``` + +As you can see by looking at the example responses, you could use the `driverLicenseID` from the People API to obtain the driver license data from the Driver License API. + +You also want to design the schema so that it represents the relationship between a person and a driver license. +As the person object is referencing a driver license by its ID, it means that we will need to define the driver license inside the person object as a field. +Consequently, a schema representing such a relationship might look like this: + +```graphql expandable +type Query { + people: [Person] # Data source for people + person(id: Int!): Person # Data Source for a specific person +} + +type Person { + id: Int! + name: String! + age: Int! + driverLicenseID: ID + driverLicense: DriverLicense # Data Source for a driver license +} + +scalar Date + +type DriverLicense { + id: ID! + issuedBy: String! + validUntil: Date! +} +``` + +#### Defining the data source URLs + +Now it's all about defining the data source URLs. + +For the field `Query.people`, you can simply use the URL to the API: +``` +https://people-api.dev/people +``` + +The `Query.person` field needs to use its `id` argument to call the correct API endpoint. + +See [Concept: Arguments](/api-management/data-graph#arguments) to learn more about it. + ``` expandable + https://people-api.dev/people/{{.arguments.id}} + ``` + +To retrieve the driver license data you need to be able to use the `driverLicenseID` from the `Person` object. As we defined the driver license data source on the `Person` object, you can now access all properties from the `Person` object by using the `.object` placeholder. + + + +If you want to access data from the object on which the data source is defined, use the `.object` placeholder (e.g: `.object.id` to access the `id` property from an object). + + + +So the URL for the driver license data source would look like this: +``` +https://driver-license-api.dev/driver-licenses/{{.object.driverLicenseID}} +``` + Use the object placeholder + +#### Result + +A query like: +```graphql +{ + people { + id + name + age + driverLicense { + id + issuedBy + validUntil + } + } +} +``` + +... will now result in something like this: +```json +{ + "data": { + "people": [ + { + "id": 1, + "name": "John Doe", + "age": 40, + "driverLicense": { + "id": "DL1234", + "issuedBy": "United Kingdom", + "validUntil": "2040-01-01" + } + }, + { + "id": 2, + "name": "Jane Doe", + "age": 30, + "driverLicense": { + "id": "DL5555", + "issuedBy": "United Kingdom", + "validUntil": "2035-01-01" + } + } + ] + } +} +``` + + +### Header management + +With Tyk v5.2 the possibilities of managing headers for Universal Data Graph and all associated data sources have been extended. + +#### Global headers for UDG + +Global headers can be configured via Tyk API Definition. The correct place to do that is within `graphql.engine.global_headers` section. For example: + +```json +{ + "graphql": { + "engine": { + "global_headers": [ + { + "key": "global-header", + "value": "example-value" + }, + { + "key": "request-id", + "value": "$tyk_context.request_id" + } + ] + } + } +} +``` + +Global headers now have access to all [request context variables](/api-management/traffic-transformation/request-context-variables). + +By default, any header that is configured as a global header, will be forwarded to all data sources of the UDG. + +#### Data source headers + +Data source headers can be configured via Tyk API Definition and via Tyk Dashboard UI. The correct place to do that is within `graphql.engine.datasources.config.headers` section. For example: + +```json +{ + "engine": { + "data_sources": [ + { + "config": { + "headers": { + "data-source-header": "data-source-header-value", + "datasource1-jwt-claim": "$tyk_context.jwt_claims_datasource1" + } + } + } + ] + } +} +``` + +Data source headers now have access to all [request context variables](/api-management/traffic-transformation/request-context-variables). + +#### Headers priority order + +If a header has a value at the data source and global level, then the data source value takes precedence. + +For example for the below configuration: + +```json +{ + "engine": { + "data_sources": [ + { + "config": { + "headers": { + "example-header": "data-source-value", + "datasource1-jwt-claim": "$tyk_context.jwt_claims_datasource1" + } + } + } + ], + "global_headers": [ + { + "key": "example-header", + "value": "global-header-value" + }, + { + "key": "request-id", + "value": "$tyk_context.request_id" + } + ] + } +} +``` expandable + +The `example-header` header name is used globally and there is also a data source level header, with a different value. Value `data-source-value` will take priority over `global-header-value`, resulting in the following headers being sent to the data source: + +| Header name | Header value | Defined on level | +| :---------------- | :------------------------------------- | :------------------ | +| example-header | data-source-value | data source | +| datasource1 | $tyk_context.jwt_claims_datasource1 | data source | +| request-id | $tyk_context.request_id | global | + +## Connect Data Sources + +### UDG + +Datasources are the fuel to power any Unified Data Graph and the designed schema. + +Datasources can be attached to any field available in the composed UDG schema. They can also be nested within each other. + +You can add Datasources to your Universal Data Graph without adding them to Tyk as a dedicated API. This is useful for getting started but also limited in capabilities. Datasources that are managed within Tyk offer much more flexibility and allow for a much fuller API Management control. + +If you want to add quotas, rate limiting, body transformations etc. to a REST Datasource it is recommended to first import the API to Tyk. + +Supported DataSources: +- REST +- GraphQL +- SOAP (through the REST DataSource) +- Kafka + +### GraphQL + +The GraphQL Datasource is able to make GraphQL queries to your upstream GraphQL service. In terms of configuration there are no real differences between the GraphQL Datasource and the one for REST with one slight exception. + +#### GraphQL data source at operation root level + +To illustrate this we'll have a look at an example graph. + +Consider the following schema: + +```graphql +type Query { + employee(id: Int!): Employee +} +type Employee { + id: Int! + name: String! +} +``` + +Let's assume we would send the following query to a GraphQL server running this schema: + +```graphql +query TykCEO { + employee(id: 1) { + id + name + } +} +``` + +The response of this query would look like this: + +```json +{ + "data": { + "employee": { + "id": 1, + "name": "Martin Buhr" + } + } +} +``` + +Compared to a REST API one difference is obvious. The response is wrapped in the root field "data". +There's also the possibility of having a root field "errors" but that's another story. +For simplicity reasons the GraphQL Datasource will not return the "data" object but rather extract the "employee" object directly. +So if you want to get the field mappings right you don't have to think about errors or data. +You can assume that your response object looks like this: + +````json +{ + "employee": { + "id": 1, + "name": "Martin Buhr" + } +} +```` expandable + +Compared to a REST API you should be able to identify the key difference here. +The response is wrapped in the field "employee" whereas in a typical REST API you usually don't have this wrapping. + +Because of this, field mappings are by default disabled for REST APIs. +For GraphQL APIs, the mapping is enabled by default and the path is set to the root field name. + +Create New API + +Other than this slight difference what's so special about the GraphQL Datasource to give it a dedicated name? + +The GraphQL Datasource will make specification-compliant GraphQL requests to your GraphQL upstream. When you attach a GraphQL Datasource to a field the Query planner of the Tyk GraphQL engine will collect all the sub fields of a root field in order to send the correct GraphQL query to the upstream. This means you can have multiple GraphQL and REST APIs side by side in the same schema, even nested, and the query planner will always send the correct query/request to each individual upstream to fetch all the data required to return a query response. + +**How does the query planner know which Datasource is responsible for a field?** + +When the query planner enters a field it will check if there is a Datasource attached to it. +If that's the case this Datasource will be responsible for resolving this field. +If there are multiple nested fields underneath this root field they will all be collected and provided to the root field Datasource. + +If however, one of the nested fields has another Datasource attached, ownership of the Datasource will shift to this new "root" field. +After leaving this second root field ownership of the Datasource for resolving fields will again shift back to the first Datasource. + +#### GraphQL data source at type/field level + +In case you want to add GraphQL data source at a lower level of your schema - type/field - the configuration steps are as follows: + +1. Navigate to the field you want the GraphQL data source to be connected to and click on it. +2. From the right-hand side menu choose **GraphQL | Tyk** or **External GraphQL** depending on wheather your data source was previously created in Tyk or if it's an external service. +Provide a data source name and URL. + +Above steps are explained in detail in our [Getting started pages](/api-management/data-graph#connect-datasource). + + +4. Tick the box next to `Add GraphQL operation` to see additional configuration fields. This will allow you to provide a query that will execute against the data source. +5. Write the query in the `Operation` box and if you're using any variables provide those in `Variables` box. + + + + + You can use objects from your Data Graph schema as variables by referring to them using this syntax: `{{.object.code}}` + + + + +Add GQL Operation + +### Kafka + +The Kafka DataSource is able to subscribe to Kafka topics and query the events with GraphQL. + + +The Kafka DataSource utilizes consumer groups to subscribe to the given topics, and inherits all behavior of the consumer group concept. + +Consumer groups are made up of multiple cooperating consumers, and the membership of these groups can change over time. Users can easily add a new consumer to the group to scale the processing load. A consumer can also go offline either for planned maintenance or due to an unexpected failure. Kafka maintains the membership of each group and redistributes work when necessary. + +When multiple consumers are subscribed to a topic and belong to the same consumer group, each consumer in the group will receive messages from a different subset of the partitions in the topic. You should know that if you add more consumers to a single group with a single topic than you have partitions, some consumers will be idle and get no messages. + +#### Basic Configuration + +You can find the full documentation for Kafka DataSource configuration here. + +**broker_addresses** +In order to work with the Kafka DataSource, you first need a running Kafka cluster. The configuration takes a list of known broker addresses and discovers the rest of the cluster. + +``` bash +{ + "broker_addresses": ["localhost:9092"] +} +``` + +**topics** +The Kafka DataSource is able to subscribe to multiple topics at the same time but you should know that the structs of events have to match the same GraphQL schema. + +```bash +{ + "topics": ["product-updates"] +} +``` + +**group_id** +As mentioned earlier, the Kafka DataSource utilizes the consumer group concept to subscribe to topics. We use the `group_id` field to set the consumer group name. + +```bash +{ + "group_id": "product-updates-group" +} +``` + +Multiple APIs can use the same `group_id` or you can run multiple subscription queries using the same API. Please keep in mind that the Kafka DataSource inherits all behaviors of the consumer group concept. + +**client_id** +Finally, we need the `client_id` field to complete the configuration. It is a user-provided string that is sent with every request to the brokers for logging, debugging, and auditing purposes. + +```bash +{ + "client_id": "tyk-kafka-integration" +} +``` + +Here is the final configuration for the Kafka DataSource: + +```bash +{ + "broker_addresses": ["localhost:9092"], + "topics": ["product-updates"], + "group_id": "product-updates-group", + "client_id": "tyk-kafka-integration" +} +``` expandable + +The above configuration object is just a part of the API Definition Object of Tyk Gateway. + +#### Kafka Datasource configuration via Dashboard + +1. Click on the field which should have Kafka datasource attached + +2. From the right-hand side *Configure data source* panel choose KAFKA at the bottom in the *Add a new external data source* section + +Kafkaconfig + +3. Provide datasource name, broker address (at least 1), topics (at least 1), groupID, clientID. Optionally you can also choose Kafka version, balance strategy and field mapping options. + +4. Click *SAVE* button to persist the configuration. + +Once done the field you just configured will show information about data source type and name: + +KafkaList + +##### Subscribing to topics + +The `Subscription` type always defines the top-level fields that consumers can subscribe to. Let's consider the following definition: + +```bash +type Product { + name: String + price: Int + inStock: Int +} + +type Subscription { + productUpdated: Product +} +``` + +The `productUpdated` field will be updated each time a product is updated. Updating a product means a `price` or `inStock` fields of `Product` are updated and an event is published to a Kafka topic. Consumers can subscribe to the `productUpdated` field by sending the following query to the server: + +```bash +subscription Products { + productUpdated { + name + price + inStock + } +} +``` + +You can use any GraphQL client that supports subscriptions. + +##### Publishing events for testing + +In order to test the Kafka DataSource, you can publish the following event to `product-updates` topic: + +```bash +{ + "productUpdated": { + "name": "product1", + "price": 1624, + "inStock": 219 + } +} +``` expandable + +You can use any Kafka client or GUI to publish events to `product-updates`. + +When you change any of the fields, all subscribers of the `productUpdated`kafk field are going to receive the new product info. + +The result should be similar to the following: + +API Menu + + +##### API Definition for the Kafka DataSource + +The Kafka DataSource configuration: + +```bash +{ + "kind": "Kafka", + "name": "kafka-consumer-group", + "internal": false, + "root_fields": [{ + "type": "Subscription", + "fields": [ + "productUpdated" + ] + }], + "config": { + "broker_addresses": [ + "localhost:9092" + ], + "topics": [ + "product-updates" + ], + "group_id": "product-updates-group", + "client_id": "tyk-kafka-integration" + } +} +``` + +Here is a sample API definition for the Kafka DataSource. + +```bash +{ + "created_at": "2022-09-15T16:19:07+03:00", + "api_model": {}, + "api_definition": { + "api_id": "7ec1a1c117f641847c5adddfdcd4630f", + "jwt_issued_at_validation_skew": 0, + "upstream_certificates": {}, + "use_keyless": true, + "enable_coprocess_auth": false, + "base_identity_provided_by": "", + "custom_middleware": { + "pre": [], + "post": [], + "post_key_auth": [], + "auth_check": { + "name": "", + "path": "", + "require_session": false, + "raw_body_only": false + }, + "response": [], + "driver": "", + "id_extractor": { + "extract_from": "", + "extract_with": "", + "extractor_config": {} + } + }, + "disable_quota": false, + "custom_middleware_bundle": "", + "cache_options": { + "cache_timeout": 60, + "enable_cache": true, + "cache_all_safe_requests": false, + "cache_response_codes": [], + "enable_upstream_cache_control": false, + "cache_control_ttl_header": "", + "cache_by_headers": [] + }, + "enable_ip_blacklisting": false, + "tag_headers": [], + "jwt_scope_to_policy_mapping": {}, + "pinned_public_keys": {}, + "expire_analytics_after": 0, + "domain": "", + "openid_options": { + "providers": [], + "segregate_by_client": false + }, + "jwt_policy_field_name": "", + "enable_proxy_protocol": false, + "jwt_default_policies": [], + "active": true, + "jwt_expires_at_validation_skew": 0, + "config_data": {}, + "notifications": { + "shared_secret": "", + "oauth_on_keychange_url": "" + }, + "jwt_client_base_field": "", + "auth": { + "disable_header": false, + "auth_header_name": "Authorization", + "cookie_name": "", + "name": "", + "validate_signature": false, + "use_param": false, + "signature": { + "algorithm": "", + "header": "", + "use_param": false, + "param_name": "", + "secret": "", + "allowed_clock_skew": 0, + "error_code": 0, + "error_message": "" + }, + "use_cookie": false, + "param_name": "", + "use_certificate": false + }, + "check_host_against_uptime_tests": false, + "auth_provider": { + "name": "", + "storage_engine": "", + "meta": {} + }, + "blacklisted_ips": [], + "graphql": { + "schema": "type Product {\n name: String\n price: Int\n inStock: Int\n}\n\ntype Query {\n topProducts(first: Int): [Product]\n}\n\ntype Subscription {\n productUpdated: Product\n}", + "enabled": true, + "engine": { + "field_configs": [{ + "type_name": "Query", + "field_name": "topProducts", + "disable_default_mapping": false, + "path": [ + "topProducts" + ] + }, + { + "type_name": "Subscription", + "field_name": "productUpdated", + "disable_default_mapping": false, + "path": [ + "productUpdated" + ] + } + ], + "data_sources": [{ + "kind": "GraphQL", + "name": "topProducts", + "internal": false, + "root_fields": [{ + "type": "Query", + "fields": [ + "topProducts" + ] + }], + "config": { + "url": "http://localhost:4002/query", + "method": "POST", + "headers": {}, + "default_type_name": "Product" + } + }, + { + "kind": "Kafka", + "name": "kafka-consumer-group", + "internal": false, + "root_fields": [{ + "type": "Subscription", + "fields": [ + "productUpdated" + ] + }], + "config": { + "broker_addresses": [ + "localhost:9092" + ], + "topics": [ + "product-updates" + ], + "group_id": "product-updates-group", + "client_id": "tyk-kafka-integration" + } + } + ] + }, + "type_field_configurations": [], + "execution_mode": "executionEngine", + "proxy": { + "auth_headers": { + "Authorization": "Bearer eyJvcmciOiI2MWI5YmZmZTY4OGJmZWNmZjAyNGU5MzEiLCJpZCI6IjE1ZmNhOTU5YmU0YjRmMDFhYTRlODllNWE5MjczZWZkIiwiaCI6Im11cm11cjY0In0=" + } + }, + "subgraph": { + "sdl": "" + }, + "supergraph": { + "subgraphs": [], + "merged_sdl": "", + "global_headers": {}, + "disable_query_batching": false + }, + "version": "2", + "playground": { + "enabled": false, + "path": "/playground" + }, + "last_schema_update": "2022-09-15T16:45:42.062+03:00" + }, + "hmac_allowed_clock_skew": -1, + "dont_set_quota_on_create": false, + "uptime_tests": { + "check_list": [], + "config": { + "expire_utime_after": 0, + "service_discovery": { + "use_discovery_service": false, + "query_endpoint": "", + "use_nested_query": false, + "parent_data_path": "", + "data_path": "", + "cache_timeout": 60 + }, + "recheck_wait": 0 + } + }, + "enable_jwt": false, + "do_not_track": false, + "name": "Kafka DataSource", + "slug": "kafka-datasource", + "analytics_plugin": {}, + "oauth_meta": { + "allowed_access_types": [], + "allowed_authorize_types": [], + "auth_login_redirect": "" + }, + "CORS": { + "enable": false, + "max_age": 24, + "allow_credentials": false, + "exposed_headers": [], + "allowed_headers": [ + "Origin", + "Accept", + "Content-Type", + "X-Requested-With", + "Authorization" + ], + "options_passthrough": false, + "debug": false, + "allowed_origins": [ + "*" + ], + "allowed_methods": [ + "GET", + "POST", + "HEAD" + ] + }, + "event_handlers": { + "events": {} + }, + "proxy": { + "target_url": "", + "service_discovery": { + "endpoint_returns_list": false, + "cache_timeout": 0, + "parent_data_path": "", + "query_endpoint": "", + "use_discovery_service": false, + "_sd_show_port_path": false, + "target_path": "", + "use_target_list": false, + "use_nested_query": false, + "data_path": "", + "port_data_path": "" + }, + "check_host_against_uptime_tests": false, + "transport": { + "ssl_insecure_skip_verify": false, + "ssl_min_version": 0, + "proxy_url": "", + "ssl_ciphers": [] + }, + "target_list": [], + "preserve_host_header": false, + "strip_listen_path": true, + "enable_load_balancing": false, + "listen_path": "/kafka-datasource/", + "disable_strip_slash": true + }, + "client_certificates": [], + "use_basic_auth": false, + "version_data": { + "not_versioned": true, + "default_version": "", + "versions": { + "Default": { + "name": "Default", + "expires": "", + "paths": { + "ignored": [], + "white_list": [], + "black_list": [] + }, + "use_extended_paths": true, + "extended_paths": { + "ignored": [], + "white_list": [], + "black_list": [], + "transform": [], + "transform_response": [], + "transform_jq": [], + "transform_jq_response": [], + "transform_headers": [], + "transform_response_headers": [], + "hard_timeouts": [], + "circuit_breakers": [], + "url_rewrites": [], + "virtual": [], + "size_limits": [], + "method_transforms": [], + "track_endpoints": [], + "do_not_track_endpoints": [], + "validate_json": [], + "internal": [] + }, + "global_headers": {}, + "global_headers_remove": [], + "global_response_headers": {}, + "global_response_headers_remove": [], + "ignore_endpoint_case": false, + "global_size_limit": 0, + "override_target": "" + } + } + }, + "jwt_scope_claim_name": "", + "use_standard_auth": false, + "session_lifetime": 0, + "hmac_allowed_algorithms": [], + "disable_rate_limit": false, + "definition": { + "enabled": false, + "name": "", + "default": "", + "location": "header", + "key": "x-api-version", + "strip_path": false, + "strip_versioning_data": false, + "versions": {} + }, + "use_oauth2": false, + "jwt_source": "", + "jwt_signing_method": "", + "jwt_not_before_validation_skew": 0, + "use_go_plugin_auth": false, + "jwt_identity_base_field": "", + "allowed_ips": [], + "request_signing": { + "is_enabled": false, + "secret": "", + "key_id": "", + "algorithm": "", + "header_list": [], + "certificate_id": "", + "signature_header": "" + }, + "org_id": "630899e6688bfe5fd6bbe679", + "enable_ip_whitelisting": false, + "global_rate_limit": { + "rate": 0, + "per": 0 + }, + "protocol": "", + "enable_context_vars": false, + "tags": [], + "basic_auth": { + "disable_caching": false, + "cache_ttl": 0, + "extract_from_body": false, + "body_user_regexp": "", + "body_password_regexp": "" + }, + "listen_port": 0, + "session_provider": { + "name": "", + "storage_engine": "", + "meta": {} + }, + "auth_configs": { + "authToken": { + "disable_header": false, + "auth_header_name": "Authorization", + "cookie_name": "", + "name": "", + "validate_signature": false, + "use_param": false, + "signature": { + "algorithm": "", + "header": "", + "use_param": false, + "param_name": "", + "secret": "", + "allowed_clock_skew": 0, + "error_code": 0, + "error_message": "" + }, + "use_cookie": false, + "param_name": "", + "use_certificate": false + }, + "basic": { + "disable_header": false, + "auth_header_name": "Authorization", + "cookie_name": "", + "name": "", + "validate_signature": false, + "use_param": false, + "signature": { + "algorithm": "", + "header": "", + "use_param": false, + "param_name": "", + "secret": "", + "allowed_clock_skew": 0, + "error_code": 0, + "error_message": "" + }, + "use_cookie": false, + "param_name": "", + "use_certificate": false + }, + "coprocess": { + "disable_header": false, + "auth_header_name": "Authorization", + "cookie_name": "", + "name": "", + "validate_signature": false, + "use_param": false, + "signature": { + "algorithm": "", + "header": "", + "use_param": false, + "param_name": "", + "secret": "", + "allowed_clock_skew": 0, + "error_code": 0, + "error_message": "" + }, + "use_cookie": false, + "param_name": "", + "use_certificate": false + }, + "hmac": { + "disable_header": false, + "auth_header_name": "Authorization", + "cookie_name": "", + "name": "", + "validate_signature": false, + "use_param": false, + "signature": { + "algorithm": "", + "header": "", + "use_param": false, + "param_name": "", + "secret": "", + "allowed_clock_skew": 0, + "error_code": 0, + "error_message": "" + }, + "use_cookie": false, + "param_name": "", + "use_certificate": false + }, + "jwt": { + "disable_header": false, + "auth_header_name": "Authorization", + "cookie_name": "", + "name": "", + "validate_signature": false, + "use_param": false, + "signature": { + "algorithm": "", + "header": "", + "use_param": false, + "param_name": "", + "secret": "", + "allowed_clock_skew": 0, + "error_code": 0, + "error_message": "" + }, + "use_cookie": false, + "param_name": "", + "use_certificate": false + }, + "oauth": { + "disable_header": false, + "auth_header_name": "Authorization", + "cookie_name": "", + "name": "", + "validate_signature": false, + "use_param": false, + "signature": { + "algorithm": "", + "header": "", + "use_param": false, + "param_name": "", + "secret": "", + "allowed_clock_skew": 0, + "error_code": 0, + "error_message": "" + }, + "use_cookie": false, + "param_name": "", + "use_certificate": false + }, + "oidc": { + "disable_header": false, + "auth_header_name": "Authorization", + "cookie_name": "", + "name": "", + "validate_signature": false, + "use_param": false, + "signature": { + "algorithm": "", + "header": "", + "use_param": false, + "param_name": "", + "secret": "", + "allowed_clock_skew": 0, + "error_code": 0, + "error_message": "" + }, + "use_cookie": false, + "param_name": "", + "use_certificate": false + } + }, + "strip_auth_data": false, + "id": "6323264b688bfe40b7d71ab3", + "certificates": [], + "enable_signature_checking": false, + "use_openid": false, + "internal": false, + "jwt_skip_kid": false, + "enable_batch_request_support": false, + "enable_detailed_recording": false, + "scopes": { + "jwt": {}, + "oidc": {} + }, + "response_processors": [], + "use_mutual_tls_auth": false + }, + "hook_references": [], + "is_site": false, + "sort_by": 0, + "user_group_owners": [], + "user_owners": [] +} +``` expandable + +### REST + +The REST Datasource is a base component of UDG to help you add existing REST APIs to your data graph. By attaching a REST datasource to a field the engine will use the REST resource for resolving. + +We have a video which demoes this functionality for you. + + + +#### Using external REST API as a Datasource + +In order to use an external REST API as a Datasource you need to first navigate to the field which that Datasource should be attached to. + +1. Click on the field which should have a datasource attached +2. From the right-hand side *Configure data source* panel choose REST at the bottom in the *Add a new external data source* section + +ExternalREST + +3. Provide data source name, URL, method to be used. Optionally you can add headers information and configure field mapping + +ExternalRESTdetail + +4. Click the *Save & Update API* button to persist the configuration and generate a REST resolver, to resolve this field at runtime. + +#### Using Tyk REST API as a Datasource + +1. Click on the field which should have a datasource attached +2. From the right-hand side *Configure data source* panel choose *REST | Tyk* dropdown to see all available APIs + +InternalREST + +3. Choose which Tyk REST API you want to attach +4. Provide data source name, endpoint and method to be used. Optionally you can add headers information and configure field mapping + +InternalRESTdetail + +5. Click the *Save & Update API* button to persist the configuration and generate a REST resolver, to resolve this field at runtime. + +Once done the field you just configured will show information about data source type and name: + +datasourcesList + +#### Automatically creating REST UDG configuration based on OAS specification + +Tyk Dashboard users have an option to use Tyk Dashboard API and quickly transform REST API OAS specification into a UDG config and have it published in the Dasboard within seconds. + +See our [Postman collections](https://www.postman.com/tyk-technologies/workspace/tyk-public-workspace/overview) and fork `Tyk Dashboard API v5.1`. + +The endpoint you need to use is: + +```bash +POST /api/data-graphs/data-sources/import +``` + +Request body: + +```json +{ + "type": "string", + "data": "string" +} +``` + +`type` is an enum with the following possible values: + +- openapi +- asyncapi + +To import an OAS specification you need to choose `openapi`. + +If you are using Postman and your OAS document is in `yaml` format you can use a simple pre-request script to transform it into a `string`. + +```bash +pm.environment.set("oas_document", JSON.stringify(``)) +``` + +Then your request body will look like this: + +```json +{ + "type": "openapi", + "data": {{oas_document}} +} +``` expandable + +### Tyk + +Tyk DataSources are exactly the same as GraphQL or REST DataSources. + +The only difference is that you can directly choose an endpoint from your existing APIs using a drop-down. +This makes it easier to set up and prevents typos compared to typing in the URL etc. + +From a technical perspective there's another difference: + +Tyk DataSources make it possible to call into existing APIs on a Tyk Gateway, even if those are marked as internal. +They also add a lot of flexibility as you can add custom middleware, AuthZ as well as AuthN, rate limits, quotas etc. to these. + +In general, it is advised to first add all APIs you'd wish to add to a data graph as a dedicated API to Tyk. +Then in a second step you'd add these to your data graph. + +Then in a second step you'd add these to your data graph. + + + +As of `v3.2.0` internal datasorces (`TykRESTDataSource` and `TykGraphQLDataSource`) will be deprecated at the API level. Please use `HTTPJSONDataSource` or `GraphQLDataSource` respectively. + + + +## Getting Started + +### Overview + + + +In this getting started tutorial we will combine 2 different HTTP services (Users and Reviews) into one single unified UDG API. Instead of querying these two services separately (and probably merging their responses later) we'll use UDG to get result from both the API's in one single response. + +#### Prerequisites + +- Access to Tyk Dashboard +- Node.JS v.13^ (only to follow this example) + +#### Running example services locally + + + +Clone repo + +```bash +git clone https://github.com/jay-deshmukh/example-rest-api-for-udg.git +``` + +Run it locally +```bash +cd example-rest-api-for-udg +``` + +```bash +npm i +``` + +```bash +npm run build +``` + +```bash +npm start +``` + +You should see following in your terminal + +``` +Users Service Running on http://localhost:4000 +Review service running on http://localhost:4001 +``` expandable + +
+ +Now that we have Users service running on port `4000` and Reviews service running on port `4001` let's see how we can combine these two into one single UDG API in following tutorial. + + +### Creating Schema + + + +1. Create API + +To start with a Universal Data Graph from scratch head over to the dashboard and click on β€œAPIs” in the left menu. Then click the `β€œAdd New API”` and `UDG`. You might want to give your Universal Data Graph an individual name (i.e. `User-Reviews-Demo`) + + +2. Set Authentication + +To get started easily we'll set the API to `Keyless(Open)`. To do this, scroll down to the Authentication section. + + + +The API authentication is set to Keyless for demo purposes, it’s not recommended to use this setting in production, we’ll explore how to secure the UDG later in this guide. + + + +3. Configure Schema + +Switch to schema tab in your designer and you should already see a default schema. We will edit the schema as follows to connect with our datasources later. + +```gql +type Mutation { + default: String +} + +type Query { + user(id: String): User +} + +type Review { + id: String + text: String + userId: String + user: User +} + +type User { + id: String + username: String + reviews: [Review] +} + +``` expandable + +You can also import an existing schema using the import feature, file types supported : `gql` , `graphql` and `graphqls`. + +4. Save + +Click on save button and that should create our first UDG API + +
+ +Now if we try to query our UDG API it should error at this moment as we do not have any data-source attached to it, let's see how we can do that in next section. + +### Connect Datasource + + + +Upon navigating to schema tab on API details page you’ll see a split screen view with schema and user interface for available fields to configure the datasource. + +You can attach datasource to each individual field and can also re-use the datasource for multiple fields for performance benefits in case it has similar configuration (it needs to use the same upstream URL and method). + +We will start with attaching datasource to user query using following approach. + +#### 1. Select field to attach datasource. +Upon selecting the `Users` field on type `Query`, you'll see the options to configure that field for following kinds of datasources. + +* REST +* GraphQL +* Kafka + +#### 2. Select datasource type. + +Since our upstream services are REST, we'll select REST as datasource type but other kind of datasources can be used as well: + +* *Use external data source*: Will allow to configure the field to resolve with the external API (outside Tyk environment) +* *Using exiting APIs*: Which will allow to configure the field with the API that already exists in Tyk environment. +* *Re-use already configured data source*: If you already have configured a data source for the same API you can re-use the same data-source. If the data source is reused the endpoint will only be called once by Tyk. + +You can learn more about it [here](#udg) + +#### 3. Configure datasource details. + +Configure the data source with the following fields + +**Name** + + Enter a unique datasource name configuration to reuse it in the future. We will name this as `getUserById` for the given example. +When configuring a datasource name with Tyk Dashboard, a default name is created automatically by concatenating the field name and the GraphQL type name with an underscore symbol in between. For example, _getUserById_Query_. This name is editable and can be changed by the user. + +**URL** + +We will use the URL for our `Users` service which returns details of an user for given `id` i.e `http://localhost:4000/users/:id`. + +To dynamically inject the `id` for every request made, we can use templating syntax and inject `id` with user supplied argument or we can also use session object. + +To avoid typos in template you can use the UI component to automatically create a template for you. You can select from the available argument and object template options from the list generated by input component which is triggered by entering `{` in input. + +To learn more about arguments click [here](#arguments) + +To learn more about reusing response fields click [here](#reusing-response-fields) + +#### 4. Enter datasource name. + +Enter a unique datasource name your configuration to reuse it in the future. We will name this as `getUserById` for the given example + +#### 5. Select HTTP method for the URL. + +You can select the HTTP method for your upstream url. Which should be `GET` in our case. + +#### 6. Add headers (Optional) + +If you upstream expects headers, you can supply them using this. +You can also use templating syntax here to reuse request headers. + +#### 7. Select field mapping + +Keep the field mapping disabled by default. +You can use field mapping to map the API response with your schema. + +You can learn more about field mapping [here](#field-mappings) + +#### 8. Save data source + +It is important to save the datasource configuration in order to reflect the changes in your API definition. +The` β€œSave & Update API” `button will persist the full API definition. + +#### 9. Update API and Test + +Click Update the API. + +You can now query your UDG API of `user` using the Playground tab in API designer + +```gql +query getUser { + user(id:"1"){ + username + id + reviews { + id + text + user { + id + } + + } + } +} +``` + +The above query should return the response as follows + +```json +{ + "data": { + "user": { + "username": "John Doe", + "id": "1", + "reviews": null + } + } +} +``` + +#### Challenge + +1. Try to resolve `reviews` field on type `Users` +2. Try to resolve `users` field on type `Reviews` + +As you can see our query resolved for user details but returns `null` for `reviews`. + +This happens because we haven't defined datasource on field level for `reviews` on type `User`. + +``` +Notes +- For reviews field on type User +- - Description :: get reviews by userId +- - URL :: http://localhost:4001/reviews/:userId +- - Method :: GET + +- For users field on type Review +- - Description :: get user details by Id +- - URL :: http://localhost:4000/users/:userId +- - Method :: GET + +- You can reuse response filed using templating syntax example `{{.object.id}}` +``` expandable + + + +You can find the solution for the challenge in the above video. + + + +
+ +Now that we have linked datasources for our queries, let's see how we can do the same for mutations in the next section. + + +### Mutations + + + +Now that we have attached datasources to our `Query` in the schema let's try to do the same for `Mutation`. + +#### Steps for Configuration + +1. **Update Schema** + + ```gql + type Mutation { + addReview(text: String, userId: String): Review + deletReview(reviewId: String): String + } + ``` + + We’ll update the Mutatation type as above where we’ll add two operations + + * `addReview`: Which accepts two `arguments` (i.e `text` and `userId`) and adds a new review by making a `POST` request to `http://localhost:4001/reviews` endpoint, which expects something like the following in the request payload + + ``` + { + "id": "1", // UserId of the user posting review + "text": "New Review by John Doe11" // review text + } + ``` + * `deleteReview`: Which accepts one `argument` (i.e `reviewId`), that deletes a review by making a `DELETE` request to `http://localhost:4001/reviews/:reviewId` + +2. **Configure datasource.** + + Follow these steps to configure a data source for the `Mutation`. + + * Navigate to schema tab in the api where you would see the split screen view of schema editor on left and list of configurable fields on right + * Select `addReview` field from `Mutation` type + * Select `REST` option + * Set a unique datasource name + * Set the URL as `http://localhost:4001/reviews` + * Select method type as `POST` + * Set request body to relay the graphql arguments to our upstream payload as follows: + + ``` + { + "text": "{{.arguments.text}}", + "userId": "{{.arguments.userId}}" + } + ``` + * Update the API + +3. **Execute mutation operation** + + We can now test our mutation operation with the playground in API designer using the following operation + + ```gql + mutation AddReview { + addReview(text: "review using udg", userId:"1"){ + id + text + } + } + ``` + + That should return us the following response: + + ```gql + { + "data": { + "addReview": { + "id": "e201e6f3-b582-4772-b95a-d25199b4ab82", + "text": "review using udg" + } + } + } + + ``` + + +#### Challenge + +Configure a datasource to delete a review using review id. + +``` +Notes + +- For users field on type Review +- - Description :: delete review using reviewId +- - URL :: http://localhost:4001/reviews/:reviewId +- - Method :: DELETE + +- Enable field mapping to map your API response + +``` expandable + + +You can find the solution for the challenge in the above video. + + + +
+ +Now that we have a good idea how we could do CRUD operations with UDG APIs, let's see how we can secure them using policies + +### Security + + + +Due to the nature of graphql, clients can craft complex or large queries which can cause your upstream APIs to go down or have performance issues. + +Some of the common strategies to mitigate these risks include + +- Rate limiting +- Throttling +- Query depth limiting + + +For this tutorial we'll mitigate these risks using `Query Depth Limit` but you can also use common strategies like rate limiting and throttling, which you can read more about [here](/api-management/rate-limit) + +#### Steps for Configuration + +1. **Set authentication mode** + + In you Api designer core settings tab scroll down to Authentication section and set the authentication mode `Authentication Token` and update the API. + + Our API is not open and keyless anymore and would need appropriate Authentication token to execute queries. + +2. **Applying to query depth** + + Currently if users want they could run queries with unlimited depth as follows + + ```gql + query getUser { + user(id: "1") { + reviews { + user { + reviews { + user { + reviews { + user { + reviews { + user { + id + } + } + } + } + } + } + } + } + } + } + + ``` + + To avoid these kind of scenarios we will set query depth limit on the keys created to access this API. + + Although we can directly create keys by selecting this API but we'll use policy as it will make it easier to update keys for this API in future. You can read more about policies [here](/api-management/policies#what-is-a-security-policy) + + **Create Policy** + - Navigate to policies page + - Click Add Policy + - Select our API from Access Rights table + - Expand `Global Limits and Quota` section + - Unselect `Unlimited Query Depth` and set limit to `5` + - Switch to configuration tab + - Set policy name (eg. user-reviews-policy) + - Set expiration date for the keys that would be created using this policy + - Click on create policy + + **Create a key using above policy** + - Navigate to keys page + - Click Add Key + - Select our newly created policy + - Click create key + - Copy the key ID + + Now if you try to query our UDG API using the key you should see an error as follows + + ```json + { + "error": "depth limit exceeded" + } + ``` + + + +Watch the video above to see how you can use these policies to publish your UDG APIs on your portal with documentation and playground. + + + +### Field Based Permissions + + + +It is also possible to restrict user's based on fields using policies. For example you can create two policies + +1. For read-only access for users to only execute queries. + +2. For read and write access to run mutations and queries both. + +#### Creating keys with read-only access + +**Create Policy** + +- Navigate to policies page +- Click Add Policy +- Select our API from Access Rights table +- Expand Api panel under global section +- Toggle Field-Based Permissions and check Mutation +- Switch to configuration tab +- Set policy name (eg. user-reviews-policy-read-only) +- Set expiration date for the keys that would be created using this policy +- Click on create policy + +Now keys created using these policies cannot be used for mutations. + +### Header Forwarding + +**Min Version: Tyk v3.2.0** + +You’re able to configure upstream Headers dynamically, that is, you’re able to inject Headers from the client request into UDG upstream requests. For example, it can be used to access protected upstreams. + +The syntax for this is straight forward: + +``` +{{.request.headers.someheader}} +``` + + In your data sources, define your new Header name and then declare which request header's value to use: + + Forwarding Headers + + That's it! + + + +A JSON string has to be escaped before using as a header value. For example: +``` +{\"hello\":\"world\"} +``` + + + +### UDG Examples + +It is possible to import various UDG examples from the [official Tyk examples repository](https://github.com/TykTechnologies/tyk-examples). + +We offer 3 ways of importing an example into Tyk: + - Using [tyk-sync](/api-management/automations/sync#synchronize-api-configurations-with-github-actions) + - Manually import via [Dashboard API Import](/api-management/gateway-config-managing-classic#import-an-api) +- Using Tyk Dashboard to browse and import the examples directly + +#### Import via tyk-sync + +Please follow the [tyk-sync documentation](/api-management/automations/sync#examples-publish-command) to learn more about this approach. + +#### Import via Tyk Dashboard API Import + +Navigate to an example inside the [examples repository](https://github.com/TykTechnologies/tyk-examples) and grab the relevant API definition from there. +Then you can move in the Dashboard UI to `APIs -> Import API` and select `Tyk API` as source format. + +Paste the API definition inside the text box and hit `Import API`. + +You can find more detailed instructions in the [Dashboard API Import documentation section](/api-management/gateway-config-managing-classic#import-an-api). + +#### Import via Tyk Dashboard UI + +Navigate to `Data Graphs` section of the Tyk Dashboard menu. If you haven't yet created any Universal Data Graphs you will see three options in the screen - one of them `Try example data graph` - will allow you to browse all examples compatible with your Dashboard version and choose the one you want to import. + +Examples in Dashboard + +In case you have created data graphs before and your screen looks different, just use the `Add Data Graph` button and in the next step decide if you want to create one yourself, or use one of the available examples. + +Examples in Dashboard New Graph \ No newline at end of file diff --git a/api-management/event-driven-apis.mdx b/api-management/event-driven-apis.mdx new file mode 100644 index 00000000..3ff546f7 --- /dev/null +++ b/api-management/event-driven-apis.mdx @@ -0,0 +1,1027 @@ +--- +title: "Tyk Streams – Manage Event-Driven APIs" +'og:description': "Introduction to Tyk Streams" +tags: ['Tyk Streams', 'Glossary', 'Use Cases', 'Asynchronus APIs', 'Async', 'Configuration'] +sidebarTitle: "Tyk Streams" +--- + +{/* ## TODO: Add availability */} + +## Overview + +*Tyk Streams* is a feature of the Tyk API management platform that enables organizations to securely expose, +manage and monetize real-time event streams and asynchronous APIs. + +With *Tyk Streams*, you can easily connect to event brokers and streaming platforms, such as +[Apache Kafka](https://github.com/TykTechnologies/tyk-pro-docker-demo/tree/kafka), and expose them as +managed API endpoints for internal and external consumers. + +
+Tyk Streams Overview +
+ +The purpose of Tyk Streams is to provide a unified platform for managing both synchronous APIs (such as REST and +GraphQL) and asynchronous APIs, in addition to event-driven architectures. This allows organizations to leverage the +full potential of their event-driven systems while maintaining the same level of security, control and visibility they +expect from their API management solution. + +### Why use Tyk Streams + +Tyk Stream is a powerful stream processing engine integrated into the Tyk API Gateway, available as part of the Enterprise Edition. It allows you to manage asynchronous APIs and event streams as part of your API ecosystem. It provides a range of capabilities to support async API management, including: + +- **Protocol Mediation**: Tyk Streams can mediate between different asynchronous protocols and API styles, such as WebSocket, Server-Sent Events (SSE), and Webhooks. This allows you to expose your event streams in a format compatible with your consumers' requirements. +- **Security**: Apply the same security policies and controls to your async APIs as you do to your synchronous APIs. This includes features like authentication and authorization. +- **Transformations**: Transform and enrich your event data on the fly using Tyk's powerful middleware and plugin system. This allows you to adapt your event streams to meet the needs of different consumers. +- **Analytics**: Monitor the usage and performance of your async APIs with detailed analytics and reporting. Gain insights into consumer behavior and system health. +- **Developer Portal**: Publish your async APIs to the Tyk Developer Portal, which provides a centralised catalog for discovery, documentation, and subscription management. + +--- +## Getting Started + +This guide will help you implement your first event-driven API with Tyk in under 15 minutes. To illustrate the capabilities of Tyk Streams, let's consider an example: building a basic asynchronous chat application, nicknamed **Chat Jippity**. + +In this scenario, a user sends a message (e.g., asking for a joke) via a simple web interface and receives an asynchronous response generated by a backend service. + +This application flow demonstrates two key patterns enabled by Tyk Streams: acting as an **API Producer Gateway** and an **API Consumer Gateway**. + +```mermaid +sequenceDiagram + participant Browser + participant Tyk Gateway + participant Joker Service + + Note over Browser, Tyk Gateway: Consumer - SSE Connection Setup + Browser->>+Tyk Gateway: Make Server Side Events (SSE) Connection + Tyk Gateway-->>-Browser: Connection Established + + Note over Browser, discKafka: Producer - Message Flow for Request/Response + Browser->>+Tyk Gateway: (1) POST /chat (Request: "Tell me a joke") + Tyk Gateway->>+chatKafka: (2) Publish message to 'chat' topic + chatKafka-->>-Tyk Gateway: Ack (implied) + chatKafka-->>+Joker Service: (3) Consume message from 'chat' topic + Note over Joker Service: Processes request, gets joke + Joker Service->>+discKafka: (4) Publish response to 'discussion' topic + discKafka-->>-Joker Service: Ack (implied) + discKafka-->>+Tyk Gateway: (6) Consume/Receive message from 'discussion' topic + Tyk Gateway-->>-Browser: (7) Push joke response (via established WS/SSE connection) + Tyk Gateway-->>-discKafka: Ack (implied) +``` + +Let's break down how Tyk Streams facilitates this, focusing on the distinct producer and consumer roles Tyk plays: + +### Example Scenario + +#### Tyk as an API Producer Gateway (Client to Stream) + +* **Goal:** Allow a client (like a browser or a script) to easily send a message into our asynchronous system without needing direct access or knowledge of the backend message broker (Kafka in this case). +* **Scenario:** The user types "Tell me a joke" into the chat interface and hits send. +* **Flow:** + 1. The browser sends a standard HTTP `POST` request to an endpoint exposed by Tyk Gateway (e.g., `/chat`). + 2. **Tyk Streams Role (Producer):** Tyk Gateway receives this `POST` request. An API definition configured with Tyk Streams defines this endpoint as an *input*. Tyk takes the request payload and *publishes* it as a message onto a designated backend topic (e.g., the `chat` topic in Kafka). + 3. A backend service (our "Joker Service") listens to the `chat` topic for incoming requests. +* **Value Demonstrated:** + * **Protocol Bridging:** Tyk translates a synchronous HTTP POST into an asynchronous Kafka message. + * **Decoupling:** The browser only needs to know the Tyk HTTP endpoint, not the Kafka details (brokers, topic name, protocol). + * **API Management:** Tyk can enforce authentication, rate limits, etc., on the `/chat` endpoint before the message even enters the Kafka system. + +#### Tyk as an API Consumer Gateway (Stream to Client) + +* **Goal:** Deliver the asynchronous response (the joke) from the backend system to the client in real time. +* **Scenario:** The "Joker Service" has processed the request and generated a joke. It needs to send this back to the originating user's browser session. +* **Flow:** + 1. The Joker Service *publishes* the joke response as a message onto a different backend topic (e.g., the `discussion` topic in Kafka). + 2. **Tyk Streams Role (Consumer):** Tyk Gateway is configured via another (or the same) API definition to *subscribe* to the `discussion` topic. + 3. When Tyk receives a message from the `discussion` topic, it *pushes* the message content (the joke) to the appropriate client(s) (provided they have already established a connection) using a suitable real-time protocol like Server-Sent Events (SSE) or WebSockets. + **Note:** In case of multiple clients, events would round-robin amongst the consumers. + +* **Value Demonstrated:** + * **Protocol Bridging:** Tyk translates Kafka messages into SSE messages suitable for web clients. + * **Decoupling:** The browser doesn't need a Kafka client; it uses standard web protocols (SSE/WS) provided by Tyk. The Joker Service only needs to publish to Kafka, unaware of the final client protocol. + +The following sections will guide you through the prerequisites and steps to configure Tyk Gateway to implement this use case. + +### Prerequisites + +- **Docker**: We will run the entire Tyk Stack on Docker. For installation, refer to this [guide](https://docs.docker.com/desktop/setup/install/mac-install/). +- **Git**: A CLI tool to work with git repositories. For installation, refer to this [guide](https://git-scm.com/downloads) +- **Dashboard License**: We will configure Streams API using Dashboard. [Contact support](https://tyk.io/contact/) to obtain a license. +- **Curl and JQ**: These tools will be used for testing. + +### Instructions + +1. **Clone Git Repository:** + + The [tyk-demo](https://github.com/TykTechnologies/tyk-demo) repository offers a docker-compose environment you can run locally to explore Tyk streams. Open your terminal and clone the git repository using the command below. + + ```bash expandable + git clone https://github.com/TykTechnologies/tyk-demo + cd tyk-demo + ``` + +2. **Enable Tyk Streams:** + + By default, Tyk Streams is disabled. To enable Tyk Streams in the Gateway and Dashboard, you need to configure the following settings: + + Create an `.env` file and populate it with the values below: + + ```bash + DASHBOARD_LICENCE= + GATEWAY_IMAGE_REPO=tyk-gateway-ee + TYK_DB_STREAMING_ENABLED=true + TYK_GW_STREAMING_ENABLED=true + ``` + + - `DASHBOARD_LICENCE`: Add your license key. Contact [support](https://tyk.io/contact/) to obtain a license. + - `GATEWAY_IMAGE_REPO`: Tyk Streams is available as part of the Enterprise Edition of the Gateway. + - `TYK_DB_STREAMING_ENABLED` and `TYK_GW_STREAMING_ENABLED`: These must be set to `true` to enable Tyk Streams in the Dashboard and Gateway, respectively. Refer to the [configuration options](/tyk-oss-gateway/configuration#streaming) for more details. + +3. **Start Tyk Streams** + + Execute the following command: + ```bash + ./up.sh + ``` + + + + + This script also starts `Kafka` within a Docker container, which is necessary for this guide. + + + + This process will take a few minutes to complete and will display some credentials upon completion. Copy the Dashboard **username, password, and API key**, and save them for later use. + ``` + β–Ύ Tyk Demo Organisation + Username : admin-user@example.org + Password : 3LEsHO1jv1dt9Xgf + Dashboard API Key : 5ff97f66188e48646557ba7c25d8c601 + ``` + +4. **Verify Setup:** + + Open Tyk Dashboard in your browser by visiting [http://localhost:3000](http://localhost:3000) or [http://tyk-dashboard.localhost:3000](http://tyk-dashboard.localhost:3000) and login with the provided **admin** credentials. + +5. **Create Producer API:** + + Create a file `producer.json` with the below content: (**Note:** `tyk-demo-kafka-1` is the hostname used to access Kafka running in a container; alternatively, you can use the IP address assigned to your computer.) + + + + + ```json + { + "components": {}, + "info": { + "title": "jippity-chat", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": {}, + "servers": [ + { + "url": "http://tyk-gateway.localhost:8080/jippity-chat/" + } + ], + "x-tyk-api-gateway": { + "info": { + "name": "jippity-chat", + "state": { + "active": true + } + }, + "server": { + "listenPath": { + "value": "/jippity-chat/", + "strip": true + } + }, + "upstream": { + "url": "" + } + }, + "x-tyk-streaming": { + "streams": { + "default_stream": { + "input": { + "http_server": { + "address": "", + "allowed_verbs": [ + "POST" + ], + "path": "/chat", + "rate_limit": "", + "timeout": "5s" + }, + "label": "" + }, + "output": { + "kafka": { + "addresses": ["tyk-demo-kafka-1:9092"], + "max_in_flight": 10, + "topic": "chat" + }, + "label": "" + } + } + } + } + } + ``` + + + + + + Create the API by executing the following command. Be sure to replace `` with the API key you saved earlier: + + ```bash + curl -H "Authorization: " -H "Content-Type: application/vnd.tyk.streams.oas" http://localhost:3000/api/apis/streams -d @producer.json + ``` + + You should expect a response similar to the one shown below, indicating success. Note that the Meta and ID values will be different each time: + ```bash + {"Status":"OK","Message":"API created","Meta":"67e54cadbfa2f900013b501c","ID":"3ddcc8e1b1534d1d4336dc6b64a0d22f"} + ``` + +5. **Create Consumer API:** + + Create a file `consumer.json` with the below content: (**Note:** `tyk-demo-kafka-1` is the hostname used to access Kafka running in a container; alternatively, you can use the IP address assigned to your computer.) + + + + + ```json + { + "components": {}, + "info": { + "title": "jippity-discuss", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": {}, + "servers": [ + { + "url": "http://tyk-gateway.localhost:8080/jippity-discuss/" + } + ], + "x-tyk-api-gateway": { + "info": { + "name": "jippity-discuss", + "state": { + "active": true + } + }, + "server": { + "listenPath": { + "value": "/jippity-discuss/", + "strip": true + } + }, + "upstream": { + "url": "" + } + }, + "x-tyk-streaming": { + "streams": { + "default_stream": { + "input": { + "kafka": { + "addresses": ["tyk-demo-kafka-1:9092"], + "auto_replay_nacks": true, + "checkpoint_limit": 1024, + "consumer_group": "tyk-streams", + "target_version": "3.3.0", + "topics": ["discussion"] + }, + "label": "" + }, + "output": { + "http_server": { + "address": "", + "allowed_verbs": [ + "GET" + ], + "stream_path": "/sse" + }, + "label": "" + } + } + } + } + } + ``` + + + + + + Create the API by executing the following command. Be sure to replace `` with the API key you saved earlier: + + ```bash + curl -H "Authorization: " -H "Content-Type: application/vnd.tyk.streams.oas" http://localhost:3000/api/apis/streams -d @consumer.json + ``` + + You should expect a response similar to the one shown below, indicating success. Note that the Meta and ID values will be different each time: + ```bash + {"Status":"OK","Message":"API created","Meta":"67e54cadbfa2f900013b501c","ID":"3ddcc8e1b1534d1d4336dc6b64a0d22f"} + ``` + +7. **Start Joker Service:** + + Create a file `joker-service.sh` with the below content: + + + + + ```bash + #!/bin/bash + + # Container name + CONTAINER="tyk-demo-kafka-1" + + # Kafka bootstrap server + BOOTSTRAP_SERVER="localhost:9092" + + # Topics + SOURCE_TOPIC="chat" + TARGET_TOPIC="discussion" + + # Kafka consumer and producer commands + CONSUMER_CMD="/opt/kafka/bin/kafka-console-consumer.sh --bootstrap-server $BOOTSTRAP_SERVER --topic $SOURCE_TOPIC" + PRODUCER_CMD="/opt/kafka/bin/kafka-console-producer.sh --broker-list $BOOTSTRAP_SERVER --topic $TARGET_TOPIC" + + # Joke API URL + JOKE_API="https://icanhazdadjoke.com/" + + echo "Starting to listen for messages on '$SOURCE_TOPIC'..." + + # Run the consumer in the container, pipe output to a while loop + docker exec -i $CONTAINER bash -c "$CONSUMER_CMD" | while IFS= read -r message; do + # Skip empty lines + [ -z "$message" ] && continue + + echo "Received message from $SOURCE_TOPIC: $message" + + # Fetch a random joke from the API and extract it with jq + joke=$(curl -s -H "Accept: application/json" "$JOKE_API" | jq .joke) + + # Check if joke was fetched successfully + if [ -n "$joke" ]; then + response_message="In response to '$message': Here's a dad joke - $joke" + else + response_message="In response to '$message': Couldn't fetch a joke, sorry!" + fi + + # Send the response message to 'discussion' topic + echo "$response_message" | docker exec -i $CONTAINER bash -c "$PRODUCER_CMD" + + echo "Posted to $TARGET_TOPIC: $response_message" + done + + echo "Consumer stopped." + ``` + + + + + + Make the file executable and start the service. + + ```bash + chmod +x joker-service.sh + ./joker-service.sh + ``` + +8. **Test the API:** + + Open a terminal and execute the following command to start listening for messages from the Consumer API you created: + + ```bash + curl -N http://tyk-gateway.localhost:8080/jippity-discuss/sse + ``` + + In a second terminal, execute the command below to send a message to the Producer API. You can run this command multiple times and modify the message to send different messages: + + ```bash + curl -X POST http://tyk-gateway.localhost:8080/jippity-chat/chat -H "Content-Type: text/plain" -d "Tell me a joke." + ``` + + Now, you will see the message appear in the terminal window where you are listening for messages. + +**Wrapping Up:** And that’s itβ€”you’ve just created an Async API with Tyk Streams! From here, you can tweak the configuration to suit your needs, [explore glossary](#glossary), or explore more [advanced use cases](#use-cases). + +--- +## How It Works + +Tyk Streams seamlessly integrates with the Tyk API Gateway, extending its capabilities beyond traditional synchronous request/response patterns to natively support asynchronous APIs and event-driven architectures. + +This section details the architecture, components, and request processing flow of Tyk Streams. + +### High-Level Architecture + +At a high level, Tyk Streams operates within the Tyk ecosystem, interacting with several key elements: + +* **Tyk API Gateway**: The core API management platform. It is the entry point, handling initial request processing (like authentication and rate limiting) and routing requests. +* **Tyk Streams Module**: An integrated extension within the Gateway designed explicitly for asynchronous communication. It intercepts relevant requests and manages the streaming logic. +* **Event Brokers / Sources**: External systems that act as the origin or destination for data streams. Examples include Apache Kafka, NATS, MQTT brokers, or WebSocket servers. Tyk Streams connects to these systems based on API configuration. +* **Upstream Services / APIs**: The backend systems, microservices, or APIs that ultimately produce or consume the data being streamed or processed via Tyk Streams. + +Think of the Tyk Gateway as the central dispatch for all API traffic. When traffic requires asynchronous handling (like pushing data to Kafka or subscribing to an MQTT topic), the integrated Tyk Streams module manages the interaction with the specific Event Broker and Upstream Service according to the API's configuration. + +### Internal Components of Tyk Streams + +To manage these asynchronous interactions, the Tyk Streams module relies on several internal components operating within the Gateway: + +1. **Stream Middleware**: This component plugs into the Tyk Gateway's request processing chain. It runs *after* standard middleware like authentication and rate limiting but *before* the request would normally be proxied. Its job is to inspect incoming requests, identify if they match a configured stream path, and if so, divert them from the standard proxy flow into the stream handling logic. +2. **Stream Manager**: Acts as the supervisor for streaming operations defined in an API. A given stream configuration is responsible for initializing, managing the lifecycle (starting/stopping), and coordinating the necessary `Stream Instances`. It ensures the correct streaming infrastructure is ready based on the API definition. +3. **Stream Instance**: Represents a running, active instance of a specific stream processing task. Each instance executes the logic defined in its configuration – connecting to an event broker, processing messages, transforming data, handling connections, etc. There can be multiple instances depending on the configuration and workload. +4. **Stream Analytics**: This component captures connection attempts and errors related to HTTP outputs. This data can be exported to popular analytics platforms like Prometheus, OpenTelemetry, and StatsD. + +The following diagram shows the relationships and primary interactions between these internal components and how they relate to the Gateway and Upstream API: + +```mermaid +graph TD + idClient[Client] + idTykGateway[Tyk Gateway] + idStreamMiddleware[Stream Middleware] + idStreamManager[Stream Manager] + idStreamAnalytics[Stream Analytics] + idStreamInstance[Stream Instance] + idUpstreamAPI[Upstream API] + + idClient -- Request --> idTykGateway + idTykGateway -- Response --> idClient + idTykGateway -- Process Request --> idStreamMiddleware + idStreamMiddleware -- Response --> idTykGateway + idStreamMiddleware -- Configure & Manage --> idStreamManager + idStreamMiddleware -- Capture Analytics --> idStreamAnalytics + idStreamManager -- Create & Run --> idStreamInstance + idStreamInstance -- Processed Response --> idStreamMiddleware + idStreamInstance -- Process Data --> idUpstreamAPI + idUpstreamAPI -- Response --> idStreamInstance +``` + +### Request Processing Flow + +Understanding how these components work together is key. Here’s the typical flow when a request interacts with a Tyk Streams-enabled API endpoint: + +```mermaid +sequenceDiagram + participant Client + participant TykGateway as Tyk Gateway + participant StreamingMiddleware as Streaming Middleware + participant StreamManager as Stream Manager + participant StreamInstance as Stream Instance + participant UpstreamService as Upstream Service + + Client->>TykGateway: HTTP Request + TykGateway->>StreamingMiddleware: Process Request + StreamingMiddleware->>StreamingMiddleware: Strip Listen Path + StreamingMiddleware->>StreamingMiddleware: Check if path is handled by streams + + alt Path handled by streams + StreamingMiddleware->>StreamManager: Create/Get Stream Manager for request + StreamingMiddleware->>StreamManager: Match request to route + StreamManager->>StreamInstance: Handle request + StreamInstance->>Client: Stream response + else Not handled by streams + StreamingMiddleware-->>TykGateway: Continue middleware chain + TykGateway->>UpstreamService: Proxy request + UpstreamService->>TykGateway: Response + end + + TykGateway->>Client: HTTP Response +``` expandable + +1. **Request Arrival & Gateway Pre-processing**: A client sends a request to an API endpoint managed by Tyk Gateway. The request passes through the initial middleware, such as authentication, key validation, and rate limiting. +2. **Streaming Middleware Interception**: The request reaches the `Stream Middleware`. It checks the request path against the stream routes defined in the API configuration. +3. **Path Matching**: + * **If No Match**: The request is not destined for a stream. The `Stream Middleware` does nothing, and the request continues down the standard Tyk middleware chain, typically being proxied to the configured upstream service. {/* TODO: this behavior is true until 5.8.x. From 5.9.0 onwards it will return 404 Not Found. */} + * **If Match**: The request is intended for a stream. The `Stream Middleware` takes control of the request handling. +4. **Stream Manager Coordination**: The middleware interacts with the `Stream Manager` associated with the API's stream configuration. The `Stream Manager` ensures the required `Stream Instance`(s) are initialized and running based on the loaded configuration. This might involve creating a new instance or reusing a cached one. +5. **Stream Instance Execution**: The instance then executes its defined logic, interacting with the configured `Upstream Service / Event Broker` (e.g., publishing a message to Kafka, subscribing to an MQTT topic, forwarding data over a WebSocket). +6. **Analytics Capture**: The `Stream Analytics` component captures relevant metrics throughout the stream handling process. +8. **Final Gateway Response**: The response or data stream generated by the streaming components is relayed back through the Gateway to the originating client. + +### Scaling and Availability + +The beauty of Tyk Streams is that it’s baked into the Tyk Gateway, so it scales naturally as your API traffic ramps upβ€”no extra setup or separate systems required. It’s efficient too, reusing the same resources as the Gateway to keep things lean. + +--- +## Configuration Options + +Configuring Tyk Streams involves two distinct levels: + +1. **System-Level Configuration:** Enabling the Streams functionality globally within your Tyk Gateway and Tyk Dashboard instances. This activates the necessary components but doesn't define any specific streams. +2. **API-Level Configuration:** Defining the actual stream behaviors (inputs, outputs, processing logic) within a specific Tyk OAS API Definition using the `x-tyk-streaming` extension. This is where you specify *how* data flows for a particular asynchronous API. + +Let's look at each level in detail. + +### System-Level Configuration + +Before you can define streams in your APIs, you must enable the core Streams feature in both the Tyk Gateway and, if you're using it for management, the Tyk Dashboard. + +#### Tyk Gateway + +Enable the Streams processing engine within the Gateway by setting `enabled` to `true` in the `streaming` section of your `tyk.conf` file or via environment variables. + + + +```json +{ +// Partial config from tyk.conf + "streaming": { + "enabled": true // Required to activate Streams functionality + }, +// ... more config follows +} +``` + + +```bash +export TYK_GW_STREAMING_ENABLED=true +``` expandable + + + +Refer to the [Tyk Gateway Configuration Reference](/tyk-oss-gateway/configuration#streamingenabled) for more details on this setting. + +#### Tyk Dashboard + +If you manage your APIs via the Tyk Dashboard, you must also enable Streams support within the Dashboard configuration (`tyk_analytics.conf`) to expose Streams-related UI elements and functionality. + + + +```json +{ +// Partial config from tyk_analytics.conf + "streaming": { + "enabled": true // Required to activate Streams functionality + }, +// ... more config follows +} +``` + + +```bash +export TYK_DB_STREAMING_ENABLED=true +``` expandable + + + +Refer to the [Tyk Dashboard Configuration Reference](/tyk-dashboard/configuration#streamingenabled) for more details. + +### API-Level Configuration + +Once Streams is enabled at the system level, you define the specific behavior for each asynchronous API within its Tyk Open API Specification (OAS) definition. This is done using the `x-tyk-streaming` vendor extension. + +The core structure under `x-tyk-streaming` is the `streams` object, which contains one or more named stream configurations. Each named stream defines: + +* **`input`**: Specifies how data enters this stream (e.g., via an HTTP request, by consuming from Kafka, connecting via WebSocket). +* **`output`**: Specifies where the data goes after processing (e.g., published to Kafka, sent over WebSocket, delivered via webhook). + +```json +{ +// Partial config from Tyk OAS API Definition + "x-tyk-streaming": { + "streams": { + "your_stream_name": { // A unique name for this stream configuration within the API + "input": { + // Input configuration object - specifies the data source + // Example: "http_server": { ... } or "kafka": { ... } + }, + "output": { + // Output configuration object - specifies the data destination + // Example: "kafka": { ... } or "websocket_server": { ... } + } + // Optional processing/transformation steps can also be defined here + }, + "another_stream": { // You can define multiple independent streams + "input": { ... }, + "output": { ... } + } + } + }, +// ... more config follows +} +``` + +**Available Input and Output Types:** + +Tyk supports various connector types for both `input` and `output`. **The specific types available (like `http_server`, `kafka`, `http_client`, etc.) and their respective configuration parameters are detailed in the [Tyk Streams Configuration Reference](/api-management/stream-config).** Please consult this reference page for the full list of options and how to configure each one. + +**Example Configuration:** + + + + +```json +{ +// Partial config from Tyk OAS API Definition + "x-tyk-streaming": { + "streams": { + "http_to_kafka_chat": { + "input": { + "http_server": { + "path": "/chat", + "allowed_verbs": [ "POST" ], + + }, + "label": "HTTP Chat Input" + }, + "output": { + "kafka": { + "addresses": ["kafka-broker:9092"], + "topic": "chat", + + }, + "label": "Kafka Chat Output" + } + } + } + }, +// ... more config follows +} +``` expandable + +For comprehensive details on all fields within `x-tyk-streaming`, see the [Tyk OAS Extension documentation](/api-management/gateway-config-tyk-oas#xtykstreaming). + + + + + +The Tyk Dashboard provides a wizard to create Streams APIs, which generates the underlying Tyk OAS configuration shown above. + +1. Navigate to **APIs > Add New API**. +2. Select the **Streams** API type and give your API a name. Click **Configure API**. + Streams Option +3. In the **API Designer**, under the **Streams** tab, configure your desired `Input` and `Output`. Select the types (e.g., HTTP Server for input, Kafka for output) and fill in the required parameters based on the [Streams Configuration Reference](/api-management/stream-config). + Input/Output Selection +4. Configure any other API settings (e.g., Authentication, Rate Limiting) as needed in the other tabs. +5. **Save** the API. The Dashboard translates your UI configuration into the `x-tyk-streaming` JSON structure within the API definition. You can view the generated JSON in the **Advanced Options** tab under **API Definition**. + + + + + +Tyk Streams configuration (`x-tyk-streaming`) is **only supported** within **Tyk OAS API Definitions**. It is not available for legacy Tyk Classic API Definitions. + + + + + +### Supported Connectors and Protocols + +Tyk Streams provides out-of-the-box connectors for popular event brokers and async protocols, including: + +- [Apache Kafka](https://kafka.apache.org/documentation/) +- [WebSocket](https://websocket.org/guides/websocket-protocol/) +- [Server-Sent Events](https://en.wikipedia.org/wiki/Server-sent_events) (SSE) +- [Webhooks](https://en.wikipedia.org/wiki/Webhook) + + +--- +## Use Cases + +Tyk Streams brings full lifecycle API management to asynchronous APIs and event-driven architectures. It provides a +comprehensive set of capabilities to secure, transform, monitor and monetize your async APIs. + +### Security + +[Tyk Streams](/api-management/event-driven-apis#) supports all the authentication and authorization options available for traditional synchronous APIs. This +ensures that your async APIs are protected with the same level of security as your REST, GraphQL, and other API types. + +Refer this docs, to know more about [Authentication](/api-management/client-authentication) and [Authorization](/api-management/policies) in Tyk. + +### Transformations and Enrichment + +[Tyk Streams](/api-management/event-driven-apis#) allows you to transform and enrich the messages flowing through your async APIs. You can modify message payloads, filter events, combine data from multiple sources and more. + +- **[Transformation](/api-management/traffic-transformation)**: Use Tyk's powerful middleware and plugin system to transform message payloads on the fly. You can convert between different data formats (e.g., JSON to XML), filter fields, or apply custom logic. +- **[Enrichment](/api-management/plugins/overview)**: Enrich your async API messages with additional data from external sources. For example, you can lookup customer information from a database and append it to the message payload. + +### Monetization + +[Tyk Streams](/api-management/event-driven-apis#) enables you to monetize your async APIs by exposing them through the Developer Portal. Developers can discover, subscribe to and consume your async APIs using webhooks or streaming subscriptions. + +- **Developer Portal Integration**: Async APIs can be published to the Tyk Developer Portal, allowing developers to browse, subscribe, and access documentation. Developers can manage their async API subscriptions just like traditional APIs. +- **Webhooks**: Tyk supports exposing async APIs as webhooks, enabling developers to receive event notifications via HTTP callbacks. Developers can configure their webhook endpoints and subscribe to specific events or topics. + +### Complex Event Processing + +Tyk Streams allows you to perform complex event processing on streams of events in real-time. You can define custom processing logic to: + +- Filter events based on specific criteria +- Aggregate and correlate events from multiple streams +- Enrich events with additional data from other sources +- Detect patterns and sequences of events +- Trigger actions or notifications based on event conditions + +Here's an example of a Tyk Streams configuration that performs complex event processing, specifically it creates a new event stream, which filters high-value orders and enriches them with customer email addresses, by making an additional HTTP request. + +```yaml +input: + kafka: + addresses: + - "localhost:9092" # Replace with actual Kafka broker addresses + consumer_group: my-group + topics: + - orders +output: + http_server: + allowed_verbs: + - GET + path: /high-value-orders +pipeline: + processors: + - mapping: | + root = if this.order_value > 1000 { + this + } else { + deleted() + } + - branch: + processors: + - http: + headers: + Content-Type: application/json + url: http://customer-api.local/emails + verb: POST + request_map: |- + root = { + "customer_id": this.customer_id + } + result_map: root.customer_email = this.customer_email + - mapping: | + root = this.merge({ "high_value_order": true }) +``` expandable + + + +**Note:** + +The `Mapping` processor is currently available and working, but it will be officially supported starting from version 5.9.0. + +You’re welcome to explore and experiment with this feature in non-production environments today. For production use, we recommend waiting for the official release in 5.9.0 to ensure full support. + + +{/* TODO: Official bloblang support from 5.9.0 onwards */} + +In this example: + +- **Tyk Streams Setup**: Consumes events from a Kafka topic called *orders*. +- **Processor Block Configuration**: Utilizes a custom `Mapping` script that performs the following operations: + - **Filters** orders, only processing those with a value greater than 1000. + - **Enriches** the high-value orders by retrieving the customer ID and email from a separate data source. + - **Adds** a new high_value_order flag to each qualifying event. +- **Output Handling**: Processed high-value order events are exposed via a WebSocket stream at the endpoint */high-value-orders*. + +### Legacy Modernization + +Tyk Streams can help you modernise legacy applications and systems by exposing their functionality as async APIs. This allows you to: +- Decouple legacy systems from modern consumers +- Enable real-time, event-driven communication with legacy apps +- Gradually migrate away from legacy infrastructure + +Here's an example of exposing a legacy application as an async API using Tyk Streams: + +```yaml +input: + http_client: + url: "http://legacy-app/orders" + verb: GET + rate_limit: "60s" +pipeline: + processors: + - mapping: | + root.order_id = this.id + root.total = this.total + root.timestamp = this.timestamp +output: + kafka: + addresses: ["localhost:9092"] + topic: "orders" +``` expandable + +In this configuration: +- Tyk Streams periodically polls the legacy */orders* REST endpoint every 60 seconds +- The *processor* transforms the legacy response format into a simplified event structure +- The transformed events are published to a Kafka topic called *orders*, which can be consumed by modern applications + +### Async API Orchestration + +Tyk Streams enables you to orchestrate multiple async APIs and services into composite event-driven flows. You can: +- Combine events from various streams and sources +- Implement complex routing and mediation logic between async APIs +- Create reactive flows triggered by event conditions +- Fanout events to multiple downstream consumers + +Here's an example async API orchestration with Tyk Streams: + +```yaml +input: + broker: + inputs: + - kafka: + addresses: ["localhost:9092"] + topics: ["stream1"] + consumer_group: "group1" + - kafka: + addresses: ["localhost:9092"] + topics: ["stream2"] + consumer_group: "group2" +pipeline: + processors: + - switch: + cases: + - check: 'meta("kafka_topic") == "stream1"' + processors: + - mapping: | + root.type = "event_from_stream1" + root.details = this + - branch: + processors: + - http: + url: "http://api1.example.com/process" + verb: POST + body: '${! json() }' + result_map: 'root.api1_response = this' + - check: 'meta("kafka_topic") == "stream2"' + processors: + - mapping: | + root.type = "event_from_stream2" + root.details = this + - branch: + processors: + - http: + url: "http://api2.example.com/analyze" + verb: POST + body: '${! json() }' + result_map: 'root.api2_response = this' + - mapping: 'root = if this.type == "event_from_stream1" && this.api1_response.status == "ok" { this } else if this.type == "event_from_stream2" && this.api2_response.status == "ok" { this } else { deleted() }' +output: + broker: + pattern: "fan_out" + outputs: + - kafka: + addresses: ["localhost:9092"] + topic: "processed_stream1" + client_id: "tyk_fanout1" + - kafka: + addresses: ["localhost:9092"] + topic: "processed_stream2" + client_id: "tyk_fanout2" + - http_client: + url: "https://webhook.site/unique-id" + verb: POST + body: '${! json() }' +``` expandable + +1. **Input Configuration** + - Uses a broker to combine events from two different Kafka topics, stream1 and stream2, allowing for the integration of events from various streams. +2. **Complex Routing and Processing** + - A switch processor directs messages based on their origin (differentiated by Kafka topic metadata). + - Each stream’s messages are processed and conditionally sent to different APIs. + - Responses from these APIs are captured and used to decide on message processing further. +3. **Reactive Flows** + - Conditions based on API responses determine if messages are forwarded or discarded, creating a flow reactive to the content and success of API interactions. + - Fanout to Multiple Consumers: + - The broker output with a fan-out pattern sends processed messages to multiple destinations: two different Kafka topics and an HTTP endpoint, demonstrating the capability to distribute events to various downstream consumers. + +These are just a few examples of the advanced async API scenarios made possible with Tyk Streams. The platform provides a flexible and extensible framework to design, deploy and manage sophisticated event-driven architectures. + +### Monetize APIs using Developer Portal + +Tyk Streams seamlessly integrates with the Tyk Developer Portal, enabling developers to easily discover, subscribe to, and consume async APIs and event streams. This section covers how to publish async APIs to the developer portal, provide documentation, and enable developers to subscribe to events and streams. + + + +#### Publishing Async APIs to the Developer Portal + +Publishing async APIs to the Tyk Developer Portal follows a similar process to publishing traditional synchronous APIs. API publishers can create API products that include async APIs and make them available to developers through the portal. + +To publish an async API: +- In the Tyk Dashboard, create a new API and define the async API endpoints and configuration. +- Associate the async API with an API product. +- Publish the API product to the Developer Portal. +- Copy code + +{/* [Placeholder for screenshot or GIF demonstrating the process of publishing an async API to the Developer Portal] */} + + + +#### Async API Documentation + +Providing clear and comprehensive documentation is crucial for developers to understand and effectively use async APIs. While Tyk Streams does not currently support the AsyncAPI specification format, it allows API publishers to include detailed documentation for each async API. + +When publishing an async API to the Developer Portal, consider including the following information in the documentation: +- Overview and purpose of the async API +- Supported protocols and endpoints (e.g., WebSocket, Webhook) +- Event types and payloads +- Subscription and connection details +- Example code snippets for consuming the async API +- Error handling and troubleshooting guidelines + +{/* [Placeholder for screenshot showcasing async API documentation in the Developer Portal] */} + + + +#### Enabling Developers to Subscribe to Events and Streams + +Tyk Streams provides a seamless way for developers to subscribe to events and streams directly from the Developer Portal. API publishers can enable webhook subscriptions for specific API products, allowing developers to receive real-time updates and notifications. +To enable webhook subscriptions for an API product: +1. In the Tyk Developer Portal, navigate to the API product settings. +2. Enable the "Webhooks" option and specify the available events for subscription. +3. Save the API product settings. + +Enable Portal Webhooks + +{/* [Placeholder for screenshot showing the API product settings with webhook configuration] */} + +Once webhook subscriptions are enabled, developers can subscribe to events and streams by following these steps: +- In the Developer Portal, navigate to the My Apps page. +- Select the desired app. +- In the "Webhooks" section, click on "Subscribe". +- Provide the necessary details: + - *Webhook URL*: The URL where the event notifications will be sent. + - *HMAC Secret*: Provide a secret key used to sign the webhook messages for authentication. + - *Events*: Select the specific events to subscribe to. +- Save the subscription settings. +- Copy code +{/* [Placeholder for screenshot illustrating the developer's view of subscribing to webhooks] */} + +subscribe to webhooks from portal + +To configure the async API stream for webhook subscriptions, use the following output configuration in your API definition: + +```yaml +outputs: + - portal_webhook: + event_type: bar + portal_url: http://localhost:3001 + secret: +``` + +Replace `` with the secret key for signing the webhook messages. + +Enabling webhook subscriptions allows developers to easily integrate real-time updates and notifications from async APIs into their applications, enhancing the overall developer experience and facilitating seamless communication between systems. +{/* [Placeholder for a diagram illustrating the flow of webhook subscriptions and event notifications] */} + +With Tyk Streams and the Developer Portal integration, API publishers can effectively manage and expose async APIs, while developers can discover, subscribe to, and consume event streams effortlessly, enabling powerful real-time functionality in their applications. + + +--- +## Glossary + +### Event + +An event represents a significant change or occurrence within a system, such as a user action, a sensor reading, or a data update. Events are typically lightweight and contain minimal data, often just a unique identifier and a timestamp. + +### Stream + +A stream is a continuous flow of events ordered by time. Streams allow for efficient, real-time processing and distribution of events to multiple consumers. + +### Publisher (or Producer) + +A publisher is an application or system that generates events and sends them to a broker or event store for distribution to interested parties. + +### Subscriber (or Consumer) + +A subscriber is an application or system that expresses interest in receiving events from one or more streams. Subscribers can process events in real-time or store them for later consumption. + +### Broker + +A broker is an intermediary system that receives events from publishers, stores them, and forwards them to subscribers. Brokers decouple publishers from subscribers, allowing for scalable and flexible event-driven architectures. + +### Topic (or Channel) + +A topic is a named destination within a broker where events are published. Subscribers can subscribe to specific topics to receive relevant events. + +--- +## FAQ + + + +Tyk Streams is an extension to the Tyk API Gateway that supports asynchronous APIs and event-driven architectures. It solves the challenge of managing both synchronous and asynchronous APIs in a unified platform, allowing organizations to handle real-time event streams alongside traditional REST APIs. + + + +Refer this [documentation](#supported-connectors-and-protocols). + + + +Currently, Tyk Streams is only available for hybrid customers on Tyk Cloud. To enable it, [contact support](https://tyk.io/contact/). + + + +Yes, as of Tyk v5.7.0, you can publish Tyk Streams APIs to the Tyk Developer Portal. The process is similar to publishing traditional APIs: create a Tyk Streams API, create a Policy to protect it, and publish it to the Developer Portal Catalog. + + + +Tyk Streams is embedded within the Tyk Gateway and scales with your existing Tyk infrastructure. No additional infrastructure is required. + + + +Tyk Streams is available exclusively in the `enterprise` edition. Currently, it is only accessible for hybrid customers using Tyk Cloud. Please refer to the latest documentation or reach out to [Tyk support](https://tyk.io/contact/) for specific availability in your edition. + + diff --git a/api-management/external-service-integration.mdx b/api-management/external-service-integration.mdx new file mode 100644 index 00000000..a53ead50 --- /dev/null +++ b/api-management/external-service-integration.mdx @@ -0,0 +1,1430 @@ +--- +title: "Tyk Identity Broker - Integrate Social Logins, IDPs, LDAP and Custom Authentication" +'og:description': "Learn how to integrate external services with Tyk API Gateway. Discover how to use middleware plugins, webhooks, and service discovery to extend your API functionality and connect with third-party systems." +tags: ['Tyk Identity Broker', 'TIB', 'Identity Provider', 'Identity Handler', 'SSO', 'Custom Authentication', 'Custom Proxy Provder', 'SAML', 'OIDC', 'OpenID Connect', 'Profies', 'IDPs', 'Social Provider', 'LDAP'] +sidebarTitle: "Identity Management" +--- + +## Introduction + +Tyk Identity Broker (TIB) is a solution for integrating various **Identity Management Systems (such as LDAP, Social OAuth, Okta)** with your Tyk installation. + +With TIB, you gain the flexibility to connect your existing user directories to Tyk Dashboard or Developer Portal, streamlining access management and enhancing security. Whether you're looking to implement SSO, leverage social logins, or integrate with enterprise identity providers, TIB provides the tools and configurations to make it happen. + +This page introduces general features of Tyk Identity Broker (TIB) and how to configure them. If you are looking for global configurations of the TIB deployment refer this [config file](/tyk-configuration-reference/tyk-identity-broker-configuration). + +We will delve into the following key topics: + +1. **[Introduction to Tyk Identity Broker](#what-is-tyk-identity-broker-tib)**: Explore key concepts, configuration options, and implementation steps for TIB. You'll learn how to set up profiles for different identity providers, understand the flow of authentication requests, and customize the integration to fit your specific needs. + +2. **[Single Sign On with Tyk](#single-sign-on-sso)**: We will learn how to implement seamless SSO experiences for Tyk Dashboard and Developer Portal. + +3. **[SSO with different Protocols and Systems](#sso-in-tyk)**: We will explore SSO integrations with LDAP, Social OAuth, SAML, and OpenID Connect. + +4. **[Handling Custom Authentication](#custom-proxy-identify-provider)**: We will learn how to configure TIB for unique authentication scenarios using the Proxy Provider. + +## What is Tyk Identity Broker (TIB)? + +Tyk Identity Broker (TIB) is a component providing a bridge between various Identity Management Systems such as LDAP, Social OAuth (e.g. GPlus, Twitter, GitHub) or Basic Authentication providers, to your Tyk installation. + +TIB can act as a bridge between the API Gateway, Tyk Portal or even the Tyk Dashboard, and makes it easy to integrate custom IDMs to your system. + +Starting from Tyk v3.0 TIB has been added as a built-in feature of the Tyk Dashboard. You no longer have to setup a separated instance of the service to make it work with the Dashboard. You now have two options: +1. Internal TIB: Embedded in dashboard. Easy configuration and set up. Share the same port as the dashboard +2. External TIB: Installation of TIB as a different component for advanced use cases. Requires changes to the config files and separate port. + + +**What can you do with the Tyk Identity Broker (TIB)?** + +By using the identity broker in conjunction with an IDP you have the ability to perform actions such as: + +- Enabling easy access via social logins to the developer portal (e.g. GitHub login) +- Enabling internal access to the dashboard (e.g. via LDAP/ActiveDirectory) +- Enabling easy token generation from a third party for things such as mobile apps and webapps without complex configuration + +## Working of Tyk Identity Broker (TIB) + +TIB provides a simple API through which traffic can be sent. The API will match the request to a profile which then exposes two things: + +1. An **Identity Provider** that will authorize a user and validate their identity +2. An **Identity Handler** that will authenticate a user with a delegated service (in this case, Tyk) + +### Identity Providers + +Identity providers can be anything, as long as they implement the `tap.TAProvider` interface. Bundled with TIB at the moment you have four provider types: + +1. **Social** - this provides OAuth handlers for many popular social logins (such as Google, Github and Bitbucket) +2. **LDAP** - a simple LDAP protocol binder that can validate a username and password against an LDAP server (tested against OpenLDAP) +3. **Proxy** - a generic proxy handler that will forward a request to a third party and provides multiple "validators" to identify whether a response is successful or not (e.g. status code, content match and regex) +4. **SAML** - provides a way to authenticate against a SAML IDP. + +### Identity Handlers + +An identity handler will perform a predefined set of actions once a provider has validated an identity. These actions are defined as a set of action types: + +1. `GenerateOrLoginUserProfile` - this will log a user into the Tyk Dashboard (this does not create a user, it only creates a temporary session for the user to have access). This flow is defined as next: + +Generate Or Login User Profile flow + +2. `GenerateOrLoginDeveloperProfile` - this will create or login a user to the Tyk Developer Portal. The flow is similar to _GenerateOrLoginUserProfile_ but in this case if the developer doesn't exist then it will be created. + +3. `GenerateOAuthTokenForClient` - this will act as a client ID delegate and grant an Tyk provided OAuth token for a user using a fragment in the redirect URL (standard flow). The flow is defined as: + +Generate Oauth token for client + +### Exploring TIB Profiles + +TIB takes as input one or many profiles that are stored in mongo or a file (it depends on the type of installation), a profile is a configuration that outlines of how to match a identity provider with a handler and what action to perform (Example: enable Dashboard SSO using OpenID and Microsoft Azure as IDP). The Dashboard adds a user interface to manage the profiles. + +Identity Broker User Interface + +### Anatomy of a Profile +Each profile is outlined by a series of attributes that will describe: action to perform, IDP to connect, URL's to redirect on success and failure, etc. +In order to know and understand each of the attributes, implications as well as configure your own profile please consult the profile structure below: + +#### Fields that are common for all the providers + +| Field | Description | Required | +| :------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :---------- | +| ID | ID of the profile, is a string, use the name of the profile +| OrgID | Organization ID | Yes | +| ActionType | Which action is expected to be executed while using this profile, valid values are:
  • `GenerateOrLoginDeveloperProfile`: SSO portal
  • `GenerateOrLoginUserProfile`: SSO dashboard
  • `GenerateOAuthTokenForClient`: generate OAuth tokens
| Yes | +| Type | Valid values are:
  • `passthrough`: for LDAP and ProxyProvider
  • `redirect`: for SAML and Social
| Yes | +| CustomEmailField | Name of the claim associated with the email value stored in the IDP (Identity Provider). | No | +| CustomUserIDField | Name of the claim associated with the User ID value stored in the IDP (Identity Provider). | No | +| IdentityHandlerConfig.DashboardCredential | API Key that will be used to consume the dashboard API to issue nonce codes and validate user data | yes | +| ReturnURL | Where to redirect and send the claims from the IDP on login. For dashboard SSO it would be `http://dashboard-host/tap`. For classic portal SSO it would be `http://{portal-host}/sso` | yes | +| DefaultUserGroup | When mapping groups, if a group is not found, specify which group to fallback to. | No | +| CustomUserGroupField | Name of the claim associated with the Group ID values stored in the Identity Provider | No | +| UserGroupMapping | Map that contains the matching between Tyk groups and IDP group. | No | +| UserGroupSeparator | The IDP might send the groups to which the user belongs to as a single string separated by any symbol or empty spaces, with this field you can set which symbol to use to split as an array | No | +| SSOOnlyForRegisteredUsers | A boolean value to restrict the SSO only to users that already exists in the database. Users that do not exist in the database and successfully logins in the IDP will not have access to tyk | No | + + +#### LDAP profile fields + +| Field | Description | Required | +| :------------------------ | :--------------------------------------------------------------------------------------------------------------------------------- | :------------------------------- | +| LDAPUseSSL | Whether to connect with the LDAP server via TLS, e.g. *true* or *false* | No | +| LDAPServer | LDAP Server address, e.g. *ldap://hostname*. | Yes | +| LDAPPort | LDAP Port, e.g. *389* or *636*. | Yes | +| LDAPUserDN | Required to uniquely identify and locate a user's entry in the LDAP directory | Yes | +| LDAPBaseDN | Distinguished Name from where the search will start | No | +| LDAPFilter | Used for filtering in the LDAP server | No | +| LDAPEmailAttribute | The name of the field in the LDAP schema that represents the user's email. Defaults to *mail*. | No | +| LDAPFirstNameAttribute | The name of the field in the LDAP schema that represents the user's first name. Defaults to *givenName* | No | +| LDAPLastNameAttribute | The name of the field in the LDAP schema that represents the user's last name. Defaults to *sn*. | No | +| LDAPAdminUser | Admin user name | No | +| LDAPAdminPassword | Admin password | No | +| LDAPAttributes | List of attributes to return when a matching LDAP record is found, for example ['cn', 'mail', 'ou'] | Yes. It can be an empty list | +| LDAPSearchScope | The scope is an integer value that determines the depth of the search in the directory hierarchy | No | +| FailureRedirect | In the event of a login failure this is the URL that the user will be redirected to. | Yes | +| DefaultDomain | Domain in which the LDAP is running. Used to build the username but not to perform the requests. | No | +| GetAuthFromBAHeader | A boolean value that, when set to *true*, instructs TIB to gather the user and password from the Authorization header when handling the request. | No | +| SlugifyUserName | When set to *true* enhance the username so that is URL friendly. | No | + +#### ProxyProvider profile fields + +| Field | Description | Required | +| :------------------------------------ | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :---------------------------------------------- | +| TargetHost | URL of the server | Yes | +| OKCode | This is an integer represents the HTTP status code that represents a successful response from the target service. If the response code matches this value the identity broker treats it as a successful interaction. | No. But one of OKCode, OKResponse, or OKRegex should be filled | +| OKResponse | This field specifies a particular string that should match with the response body to be considered successful. | No. But one of OKCode, OKResponse, or OKRegex should be filled | +| OKRegex | Is used to validate the content of the response beyond just the HTTP status code. If the response body contains data that matches this regular expression, it is considered a successful response. | No. But one of OKCode, OKResponse, or OKRegex should be filled | +| ResponseIsJson | This parameter helps the identity broker understand how to interpret the response body from the target service. If ResponseIsJson is set to true, the broker will expect the response to be in JSON format and will process it accordingly. This includes parsing JSON data to extract relevant information. This is a boolean field. | No | +| AccessTokenField | The name of the field that contains the access token. | No | +| UsernameField | The name of the field that contains the username. | No | +| ExrtactUserNameFromBasicAuthHeader | A boolean value that, when set to true, instructs TIB to gather the user and password from the Authorization header when handling the request. | No | +#### Social profile fields + +| Field | Description | Required | +| :---------------------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------ | +| CallbackBaseURL | URL to be redirected on success login | Yes | +| FailureRedirect | URL to be redirected on failure | Yes | +| UseProviders.Name | Name of the provider to be used. Valid values: `gplus`, `github`, `twitter`, `linkedin`, `dropbox`, `digitalocean`, `bitbucket`, `salesforce`, `openid-connect` | Yes | +| UseProviders.Key | Oauth Client key | yes | +| UseProviders.Secret | Oauth Client Secret | yes | +| UseProviders.DiscoverURL | used to dynamically retrieve the OpenID Provider's configuration metadata, including endpoints and supported features, in JSON format from /.well-known/openid-configuration. | Only required when using openid-connect | +| UseProviders.Scopes | Specifies the level of access or permissions a client is requesting from the user and the authorization server, for example ["openid","email"]. | No, however when using openID the scope β€˜openid’ should be added | +| UseProviders.SkipUserInfoRequest | Determines whether to bypass the *UserInfo* endpoint request, improving performance by relying on the ID token alone for user details. | No | +| JWE.Enabled | When set to true, JWE will be enabled, allowing Tyk to decrypt the ID token received from the IdP. If set to false, the ID token will not be decrypted. | No | +| JWE.PrivateKeyLocation | Specifies the path or identifier (certid) for the certificate that contains the private key used to decrypt the ID token when JWE is enabled. This certificate must be in PEM format and include both the public certificate and the private key. | Is only required if JWE is enabled | + +#### SAML profile fields + +| Field | Description | Required | +| :---------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------------------------------------------------------------------ | +| IDPMetadataURL | This is a URL, e.g. `https://login.microsoftonline.com/your-tenant-id/federationmetadata/2007-06/federationmetadata.xml`, that links to [XML metadata](https://docs.oasis-open.org/security/saml/v2.0/saml-metadata-2.0-os.pdf) containing information necessary for interaction with SAML-enabled identity or service providers. The document contains example URLs of endpoints, information about supported bindings, identifiers and public keys. Once you create your TIB profile you can find the SP metadata file under `{Dashboard HOST}/auth/{TIB Profile Name}/saml/metadata` | Yes | +| CertLocation | An X.509 certificate and the private key for signing your requests to the IDP. The value for `CertLocation` should be the path to a single file with the cert and key concatenated, e.g. `/etc/ssl/certs/example_cert.pem`. When used in an [embedded TIB instance in the dashboard](#installing-tyk-identity-broker-tib) then the `CertLocation` value can be the *certId* from the certificate manager. For further details please refer to [SSO with SAML](#sso-with-saml) | Yes | +| SAMLBaseURL | The host of TIB, e.g. `http://tyk-dashboard:3000/`, that will be used in the metadata document for the Service Provider. This will form part of the metadata URL used as the Entity ID by the IDP. The redirects configured in the IDP must match the expected Host and URI configured in the metadata document made available by Tyk Identity Broker. | Yes | +| ForceAuthentication | Ignore any session held by the IDP and force re-login every request. Defaults to false | No | +| SAMLBinding | Key for looking up the email claim in the SAML assertion form the IDP. Defaults to: https://schemas.xmlsoap.org/ws/2005/05/identity/claims.xsd | No | +| SAMLEmailClaim | Key for looking up the email claim in the SAML assertion form the IDP. Defaults to: https://schemas.xmlsoap.org/ws/2005/05/identity/claims.xsd | No | +| SAMLForenameClaim | Key for looking up the forename claim in the SAML assertion form the IDP. Defaults to: https://schemas.xmlsoap.org/ws/2005/05/identity/claims.xsd | No | +| SAMLSurnameClaim | Key for looking up the surname claim in the SAML assertion form the IDP. Defaults to: https://schemas.xmlsoap.org/ws/2005/05/identity/claims.xsd | No | +| FailureRedirect | URL to redirect the user if the login is not successful | Yes | +| EntityId | It is used to distinguish between different entities (IDP & SP) and ensure proper routing and validation of SAML assertions and requests. Defaults to the value set in the field `IDPMetadataURL` | No | + +## Installing Tyk Identity Broker (TIB) + +There are two ways to install TIB: + +1. **Embedded TIB**: Starting from Tyk Dashboard v3.0 TIB is built-in to the dashboard, in this case TIB will store the profiles in the same mongo database configured for dashboard + +2. **Standalone TIB**: Deployed as a seperate entity. In the standalone TIB, the profiles will be stored in file indicated when the app is started + +**Pre-requisites** + +Below are the prerequisites of TIB: + +- Tyk Gateway v1.9.1+ +- Redis +- Tyk Dashboard v0.9.7.1+ (Only if you want to do SSO to Tyk Dashboard UI or Tyk Developer Portal) + +### Enable Embedded TIB + +For the embedded TIB you don't have to do anything, only ensure that in the Dashboard's config file `identity_broker` is not pointing to an external service, and `identity_broker.enabled` is set to `true`. For example: + +```json +"identity_broker": { + "enabled": true, +}, +``` + +This settings behaves as follows: + +* If `enabled` = `false` then neither the external or internal TIB will be loaded +* If `enabled` = `true` and the tib host is not present the internal TIB will be loaded +* If `enabled` = `true` and the tib host is set, then external TIB will be loaded + +### Install Standalone TIB + +Below are the three deployment options to install TIB as a standalone application: + +1. **Docker:** + + You can install via [Docker](https://hub.docker.com/r/tykio/tyk-identity-broker/). + +2. **Linux Packages:** + + You can install via [packages](https://packagecloud.io/tyk/tyk-identity-broker/install#bash-deb) (deb or rpm). + +3. **Helm Chart for Kubernetes:** + + [Tyk Helm Chart](/product-stack/tyk-charts/overview) does not support installing TIB as separate application. If you want to enable embedded TIB in Dashboard, you can do so by updating `tib.enabled` to `true` in `tyk-dashboard` chart. If you are using an umbrella chart from us (e.g. `tyk-stack` and `tyk-control-plane`), you can do so by updating `tyk-dashboard.tib.enabled` to `true`. + +### Important TIB Configurations + +#### Configure secret for hashing session cookies + +To secure session cookies within Tyk Identity Broker (TIB) when integrating with social providers, setting the `TYK_IB_SESSION_SECRET` environment variable is crucial. This variable plays a pivotal role in hashing session cookies, thereby enhancing security. By default, if this variable isn't explicitly set, TIB falls back to using the Tyk Dashboard's admin_secret when it's embedded in the dashboard. + +For a seamless and secure setup, start by generating a strong, unique secret string. It is recommended to use a string with 32 or 64 bytes to ensure optimal security, this string will be your session secret. In a Linux, Unix, or MacOS environment, you can set this variable by running the command `export TYK_IB_SESSION_SECRET='your_secret'`. + +#### Setting Absolute Paths + +No command line arguments are needed, but if you are running TIB from another directory or during startup, you will need to set the absolute paths to the profile and config files: + +```bash +Usage of ./tyk-auth-proxy: + -c=string + Path to the config file (default "tib.conf") + -p#=string + Path to the profiles file (default "profiles.json") +``` + +See [how to configure TIB](https://github.com/TykTechnologies/tyk-identity-broker#how-to-configure-tib) + + +## Exploring Tyk Identity Broker REST API + +Refer to this [document](/tyk-identity-broker/tib-rest-api) + +## Single Sign On (SSO) + +SSO gives users the ability to log in to multiple applications without the need to enter their password more than once. +Authentication protocols such as OpenID Connect and SAML enable an application to verify the identity of users from an organization without the need to self store and manage them, and without doing the identification process and exposing their passwords to that application. Their lists of users and passwords are kept safe in one single place, in the IDP that the organization has chosen to use. The Authorization server of the IdP identify the users for a pre-registered and approved application (`client` in OAuth and OIDC terminology). + +### SSO in Tyk + +SSO is sometimes complicated to understand or set up but can be easily accomplished by using the built-in [Tyk Identity Broker (TIB)](#what-is-tyk-identity-broker-tib). + +Using our Tyk-Identity-Broker (TIB), you can do both - use your existing users directory to login to the **Dashboard** or **Developer Portal** and have an SSO. TIB, among other options, supports four methods for login to Tyk's UI: + +1. [Login with 3rd party social providers](#sso-with-social-identity-providers) +2. [Login with any IdP that supports OIDC](#sso-with-openid-connect-oidc) +3. [Login with any IdP that supports SAML](#sso-with-saml) +3. [Login with LDAP](#sso-with-ldap) + +#### SSO with Open ID Connect or Social Providers + +SSO is sometimes complicated to understand or set up but once you get it and learn to use our Tyk-Identity-Broker it becomes an easy task. + +In short, all you need is as follow: + +1. Access the Identity Manager under System Management in the Tyk Dashboard +2. Create a profile for your preferred IDP +3. Get the `client_id` + `secret` that are defined on your IDP +4. Set the `Callback URL` generated by Tyk on your IDP +5. Provide your SSO profile in Tyk with the `Discover URL (well known endpoint)` +6. Visit the Login URL after saving your profile to initialize the login +7. More Docs for the flow can be found on our [GitHub TIB repo README](https://github.com/TykTechnologies/tyk-identity-broker) and our [3rd Party integration docs](/api-management/external-service-integration) + +### Tyk's REST API for SSO + +The SSO API allows you to implement custom authentication schemes for the Dashboard and Portal. You can access the API by both admin and dashboard APIs. +Our Tyk Identity Broker (TIB) internally also uses these APIs. + +#### Generate authentication token + +The Dashboard exposes two APIs: + +- `/admin/sso` - See [Dashboard Admin API SSO](/api-management/dashboard-configuration#single-sign-on-api-1) for more details. +- `/api/sso` - See [Dashboard API SSO](/api-management/dashboard-configuration#single-sign-on-api) for more details. + +which allow you to generate a temporary authentication token, valid for 60 seconds. They make same thing you can select one of them and use it. +However, the admin API requires `admin-auth` header which should be same with `admin-secret` parameter in `tyk_analytics.conf`, the regular API requires `authorization` header which should be same with the user authentication token. + +#### Using the Token + +Once you have issued a token you can login to the dashboard using the `/tap` url, or to the portal using the `/sso` URL, and provide an authentication token via the `nonce` query param. +If `nonce` is valid, Tyk will create a temporary user and log them in. + +If you want to re-use existing dashboard users, instead of creating temporary ones, you can set `"sso_enable_user_lookup": true` variable in the Dashboard config file (`tyk_analytics.conf`). This way you can set individual permissions for users logged via SSO. + +##### Set up default permissions for the dashboard + +If you use the token with `dashboard` scope, and would like to avoid login in as admin user (which is the default permissions), you can add the `sso_permission_defaults` configuration option to the Dashboard config file (`tyk_analytics.conf`) to specify SSO user permissions in the following format: + +``` expandable +"sso_permission_defaults": { + "analytics": "read", + "apis": "write", + "hooks": "write", + "idm": "write", + "keys": "write", + "policies": "write", + "portal": "write", + "system": "write", + "users": "write", + "user_groups": "write" +} +``` + +As alternative, you can set `sso_default_group_id` to specify User Group ID assigned to SSO users. + +In order to set individual user permissions, you should first create this users in the dashboard first, set needed permissions, enable `sso_enable_user_lookup` to `true` inside dashboard config. If SSO user with the same email will be found in Dashboard users, it will re-use his permissions. + +##### Sample Login Request + +```{.copyWrapper} +GET /tap?nonce=YTNiOGUzZjctYWZkYi00OTNhLTYwODItZTAzMDI3MjM0OTEw HTTP/1.1 +Host: localhost:3000 +``` + +## SSO with Social Identity Providers + +The social provider for the Tyk Identity Broker is a thin wrapper around the excellent `goth` social auth library, modified slightly to work with a multi-tenant structure. The social provider should provide seamless integration with: + +* Bitbucket +* Digital Ocean +* Dropbox +* GitHub +* Google+ +* Linkedin +* Twitter +* Salesforce + +The social provider is ideal for SSO-style logins for the Dashboard or for the Portal. For certain providers (mainly Google+), where email addresses are returned as part of the user data, a constraint can be added to validate the users domain. This is useful for Google For Business Apps users that want to grant access to their domain users for the Dashboard. + +For more social provider examples see the Tyk Identity Broker (TIB) v0.2 Repo [Readme](https://github.com/TykTechnologies/tyk-identity-broker/blob/master/README.md#social). + +### Log into an APP with Github OAuth + + + +### Log into an APP with Google (Oauth) + +A common use case for Tyk Gateway users is to enable users to log into a web app or mobile app using a social provider such as Google, but have that user use a token in the app that is time-delimited and issued by their own API (or in this case, Tyk). + +Tyk can act as an OAuth provider, but requires some glue code to work, in particular, generating a token based on the authentication of a third party, which needs to run on a server hosted by the owner of the application. This is not ideal in many scenarios where authentication has been delegated to a third-party provider (such as Google or Github). + +In this case, we can enable this flow with Tyk Gateway by Using TIB. + +What the broker will do is essentially the final leg of the authentication process without any new code, simply sending the user via TIB to the provider will suffice for them to be granted an OAuth token once they have authenticated in a standard, expected OAuth pattern. + +Assuming we have created a client ID and secret in Google Apps to grant ourselves access to the users data, we need those details, and some additional ones from Tyk itself. + +#### To Set up an OAuth client with Google Apps + +1. Go to the [Google Developer Console](https://console.developers.google.com/) and create a new app +2. Register a new OAuth client. Let's call it WebApp 1 (Select "New Credentials -> OAuth Client ID") +3. Select Web App +4. Add the following URL (modify for your domain) to the "Authorized redirect URIs" section: `http://tib-hostname:TIB-PORT/auth/{PROFILE-ID}/gplus/callback` + +#### Create an OAuth Client in Tyk Dashboard + +TIB will use the OAuth credentials for GPlus to access and authenticate the user, it will then use another set of client credentials to make the request to Tyk to generate a token response and redirect the user, this means we need to create an OAuth client in Tyk Dashboard before we can proceed. + +One quirk with the Tyk API is that requests for tokens go via the base APIs listen path (`{listen_path}/toauth/authorize`), so we will need to know the listen path and ID of this API so TIB can make the correct API calls on your behalf. + +```{.copyWrapper} expandable +{ + "ActionType": "GenerateOAuthTokenForClient", + "ID": "3", + "IdentityHandlerConfig": { + "DashboardCredential": "{DASHBOARD-API-ID}", + "DisableOneTokenPerAPI": false, + "OAuth": { + "APIListenPath": "{API-LISTEN-PATH}", + "BaseAPIID": "{BASE-API-ID}", + "ClientId": "{TYK-OAUTH-CLIENT-ID}", + "RedirectURI": "http://{APP-DOMAIN}:{PORT}/{AUTH-SUCCESS-PATH}", + "ResponseType": "token", + "Secret": "{TYK-OAUTH-CLIENT-SECRET}" + } + }, + "MatchedPolicyID": "567a86f630c55e3256000003", + "OrgID": "53ac07777cbb8c2d53000002", + "ProviderConfig": { + "CallbackBaseURL": "http://{TIB-DOMAIN}:{TIB-PORT}", + "FailureRedirect": "http://{PORTAL-DOMAIN}:{PORTAL-PORT}/portal/login/?fail=true", + "UseProviders": [{ + "Key": "GOOGLE-OAUTH-CLIENT-KEY", + "Name": "gplus", + "Secret": "GOOGLE-OAUTH-CLIENT-SECRET" + }] + }, + "ProviderConstraints": { + "Domain": "", + "Group": "" + }, + "ProviderName": "SocialProvider", + "ReturnURL": "", + "Type": "redirect" +} +``` + +There's a few new things here we need to take into account: + +* `APIListenPath`: This is the listen path of your API, TIB uses this to generate the OAuth token. +* `BaseAPIID`: The base API ID for the listen path mentioned earlier, this forms the basic access grant for the token (this will be superseded by the `MatchedPolicyID`, but is required for token generation). +* `ClientId`: The client ID for this profile within Tyk Gateway. +* `Secret`: The client secret for this profile in Tyk Gateway. +* `RedirectURI`: The Redirect URL set for this profile in the Tyk Gateway. +* `ResponseType`: This can be `token` or `authorization_code`, the first will generate a token directly, the second will generate an auth code for follow up access. For SPWA and Mobile Apps it is recommended to just use `token`. + +When TIB successfully authorizes the user, and generates the token using the relevant OAuth credentials, it will redirect the user to the relevant redirect with their token or auth code as a fragment in the URL for the app to decode and use as needed. + +There is a simplified flow, which does not require a corresponding OAuth client in Tyk Gateway, and can just generate a standard token with the same flow. + +### Log into Dashboard with Google + +Similarly to logging into an app using Tyk, OAuth and Google Plus, if we have our callback URL and client IDs set up with Google, we can use the following profile setup to access our Dashboard using a social provider: + +```{.copyWrapper} expandable +{ + "ActionType": "GenerateOrLoginUserProfile", + "ID": "2", + "IdentityHandlerConfig": null, + "MatchedPolicyID": "1C", + "OrgID": "53ac07777cbb8c2d53000002", + "ProviderConfig": { + "CallbackBaseURL": "http://:{TIB-PORT}", + "FailureRedirect": "http://{DASH-DOMAIN}:{DASH-PORT}/?fail=true", + "UseProviders": [{ + "Name": "gplus", + "Key": "GOOGLE-OAUTH-CLIENT-KEY", + "Secret": "GOOGLE-OAUTH-CLIENT-SECRET" + }] + }, + "ProviderConstraints": { + "Domain": "yourdomain.com", + "Group": "" + }, + "ProviderName": "SocialProvider", + "ReturnURL": "http://{DASH-DOMAIN}:{DASH-PORT}/tap", + "Type": "redirect" +} +``` + +The login to the Dashboard makes use of a one-time nonce to log the user in to the session. The nonce is only accessible for a few seconds. It is recommended that in production use, all of these transactions happen over SSL connections to avoid MITM snooping. + +`Domain` constraint ensures that only users from `yourdomain.com` domain-based email accounts are allowed to login. + Replace it with correct domain or remove this section if you don't want to set this constraint. + + +When TIB successfully authorizes the user, and generates the token using the relevant OAuth credentials, it will redirect the user to the relevant redirect with their token or auth code as a fragment in the URL for the app to decode and use as needed. + +There is a simplified flow, which does not require a corresponding OAuth client in Tyk Gateway, and can just generate a standard token with the same flow. + + +## SSO with OpenID Connect (OIDC) + +- Instruction on setting [SSO with Okta](#oidc-with-okta) +- Instructions on setting [SSO with Auth0](#oidc-with-auth0) +- Instructions on setting [SSO with Keycloak](#oidc-with-keycloak) +- Instructions on setting [SSO with AzureAD](#oidc-with-azure-ad) + +### OIDC with Azure AD + +This is an end-to-end worked example of how you can use [AzureAD](https://www.microsoft.com/en-gb/security/business/identity-access/microsoft-entra-id) and our [Tyk Identity Broker (TIB)](https://tyk.io/docs/concepts/tyk-components/identity-broker/ +) to log in to your Dashboard. +This guide assumes the following: + +You already have authorized access to Tyk's Dashboard. If you haven't, get the authorization key by following this [guide](/api-management/user-management#using-dashboard-api). + +#### Configuration at Azure + +1. Access your Azure Portal and navigate to the Azure Active Directory page. + +2. Go to app registrations and create or access an application you want to use for Dashboard access. + + - If you are creating an application, give it a name and register it + +3. Add a redirect URL to your application as callback to TIB in your Azure application: + + - In your app, either via the Authentication menu or the redirect URL shortcut navigate to and add the redirect to TIB in the Web category i.e. `http://localhost:3000/auth/{PROFILE-NAME-IN-TIB}/openid-connect/callback`. + + Redirect URL + +4. Go to Overview and add a secret in Client Credentials. Don't forget to copy the secret value, not the secretID. + + Overview + +Check Microsoft's [documentation](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app) for more detail. + +#### Configuration at Dashbaord + +1. Log in to your dashboard and select Identity Management, located under System Management +2. Create a profile and select OpenID Connect as the provider type +3. Under Profile Configuration, paste the secret value, clientID, and well-known endpoint URL from the Azure site. + - Profile Configuation may look something like this: + + Profile Configuration + + - The well-known endpoint URL is created by Azure and can be located by selecting Endpoints on their site + + Endpoints + +#### Test your Azure Login: + +From the browser call `http://localhost:3000/auth/{PROFILE-NAME-IN-TIB}/openid-connect` +- If it's working you'll be redirected to Azures's web page and asked for your username and password. + + Username + + Password + +- If it's working you'll be redirected to Azures's web page and asked for your username and password. + + Dashboard + +#### Enhancements + +Once it's working you can also add more enhancements such as automatic user group mapping from your AzureAD security groups or users groups to Tyk Dashboards groups. + +##### User group mapping + +Group mapping can be managed from Advanced Settings section of the Profile Configuration screen. + +Profile Configuration - Additional Options + +As illustrated in the screen below the following information must be provided: + +- Identity provider role +- Tyk User Group: This can be created from the User Groups section of the dashboard (reference a link to a page in tyk docs here to show how to create a user group). When creating your User Group, one can also select and adjust the permissions for each group. + +For more information on how to set and change user permissions, head to this [guide](/api-management/user-management#using-dashboard-ui-1) + +Profile Configuration - Raw-editor + +You can select the scopes you would like your request to include. By default, Tyk will provide the connectid scope, anything additional must be requested. + +#### OpenID Connect Example + +For debugging purposes, you can find an example we created using the OpenID Connect playground. +1. Add the redirect url found on the OpenID Connect site to the redirect urls found under the Web section + + Access redirect urls + + Additional URL Added + +2. Copy the OpenID Connect endpoint from the Azure site +3. On the OpenID Connect site select Edit. In the Server Template dropdown menu select the Custom option and paste the endpoint in the Discovery Document URL. + + Edit Button + + Custom Dropdown + +4. Press the Use Discovery Document button and this will autofill Authorization Token Endpoint, Token Endpoint, and Token Keys Endpoint + + Discovery Document + +5. Copy and paste the Client ID and Client Secret from the Azure site to your ConnectID. Scope is autofilled for you and save the configuration. + + Client ID and Secret +6. Press start at the bottom of the Request window and if done correctly, this should prompt you to sign in to your Azure account. + + OpenID Connect Step 2 +7. You should then be redirected back to OpenID Connect where you'll be shown the Exchange Code. This needs to be turned into an access token. Press the exchange button under the request and then press Next. + + OpenID Connect Step 3 + OpenID Connect Step 4 +8. We can then verify this by pressing the verify button. We can also view the information or scope of what is being returned by heading to jwt.io and viewing the payload: data there. + + OpenID Connect Step 5 +9. We are given an object with key, value pairs and we can pass in the key ie. name to our Custom User Group and the value of to our Identity Provider Role in our Tyk dashboard as shown in the example above. + + OpenID Connect Step 6 + +To try this yourself, we have included the link: https://openidconnect.net/ + +### OIDC with Okta + +This is an end-to-end worked example of how you can use [Okta](https://www.okta.com/) and the Tyk Identity Broker to log into your Dashboard. +This guide assumes the following: + +* You already have authorized access to Tyk's Dashboard. If you haven't, [get the authorization key by following this doc](/api-management/user-management#using-dashboard-api). +* For simplicity, you are running TIB locally on port 3010 +* You are able to edit TIB's configuration file. + + +#### Configuration at Okta + +1. Create a developer account on the [Okta Developer site](https://developer.okta.com/). + You'll get a domain such as `https://.okta.com/.well-known/openid-configuration` +2. Login and create a Web Application as follows: + - Under `Application`, click `Add Application` + - Choose `Web` + - Change the name of the app + - Tick `Authorization Code` + - Click `Done` + + Note: These instruction are for the new Okta's `Developer Console`, for the `Classic UI` instructions are slightly different. + + +3. Add a callback to TIB in your application: + - Under `General`, click `Edit` and update the `Login redirect URIs` field with the endpoint on TIB `http://localhost:3010/auth/{PROFILE-NAME-IN-TIB}/openid-connect/callback`. + - `{PROFILE-NAME-IN-TIB}` - this can be any string you choose, as long as you use the same one for the profile in TIB. + +4. Permissions to login via Okta: + Under the `Assignments` tab, make sure group assignments is set to *everyone* (for now, you will change this later!). + +5. This is how it should look like after step #4 +okta-create-app + +#### Configuration at TIB + +6. Set the profile in `profiles.json` as follows: + - Copy from your Okta client the `cliend ID` to `ProviderConfig.UseProviders[].key` + - Copy from your Okta client the `Client secret` to `ProviderConfig.UseProviders[].secret` + - Add Okta's discovery url `"https://.okta.com/.well-known/openid-configuration"` to `ProviderConfig.UseProviders[].DiscoverURL` + + Example of a `profiles.json` file: +```{.json} expandable +[{ + "ActionType": "GenerateOrLoginUserProfile", + "ID": "{PROFILE-NAME-IN-TIB}", + "OrgID": "5a54a74550200d0001975584", + "IdentityHandlerConfig": { + "DashboardCredential": "{DASHBOARD-SECRET}" + }, + "ProviderConfig": { + "CallbackBaseURL": "http://{TIB-DOMAIN}:{TIB-PORT}", + "FailureRedirect": "http://{DASHBOARD-DOMAIN}:{DASHBOARD-PORT}/?fail=true", + "UseProviders": [ + { + "Key": "{Okta-App-Client-ID}", + "Secret": "{Okta-App-Client-SECRET}", + "Scopes": ["openid", "email"], + "DiscoverURL": "https://.okta.com/.well-known/openid-configuration", + "Name": "openid-connect" + } + ] + }, + "ProviderName": "SocialProvider", + "ReturnURL": "http://{DASHBOARD-DOMAIN}:{DASHBOARD-PORT}/tap", + "Type": "redirect" +}] +``` + +7. Start TIB by running the binary (`profiles.json` is in the same CWD) + See [Install TIB](/api-management/external-service-integration) for detailed instructions on how to install TIB +8. Test that it works: + From the broswer call `http://localhost:3010/auth/{PROFILE-NAME-IN-TIB}/openid-connect` + - If it's working you'll be redirected to Okta's web page and will be asked to enter your Okta user name and password. + - If you were successfully authenticated by Okta then you'll be redirected to the Tyk Dashboard and login into it without going through the login page. Job's done! +9. If you need to update your profile then you can use TIB's REST API as follows: + +```{.copyWrapper} +curl http://{TIB-DOMAIN}:{TIB-PORT}/api/profiles/{PROFILE-NAME-IN-TIB} -H "Authorization: {MY-SECRET}" -H "Content-type: application/json" -X PUT --data "@./my-new-dashboard-profile.json" | prettyjson +``` + + - POST and DELETE calls apply as normal + - You can post a few profiles to TIB. + - See [TIB REST API](/tyk-identity-broker/tib-rest-api) for more details. + +#### Understanding the flow + 1. The initial call to the endpoint on TIB was redirected to Okta + 2. Okta identified the user + 3. Okta redirected the call back to TIB endpoint (according to the callback you set up on the client earlier in step 3) and from TIB + 4. TIB, via REST API call to the dashboard, created a nonce and a special session attached to it. + 5. TIB redirected the call to the dashboard to a special endpoint `/tap` ( it was defined on the profile under `ReturnURL`) with the nonce that was created. + 6. The Dashboard on the `/tap` endpoint finds the session that is attached to the `nonce`, login the user and redirect to the dashboard first page + +#### Enabling MFA and SSO + +Once it's working you can also add two more enhancements - SSO and MFA + +##### SSO login into the Dashboard via a login page + You will need to: + - set up a web server with a login page and a form for `user` and `password` + - Update `tyk_analytics.conf` to redirect logins to that url + Explicit details are in [steps 6-7](#create-login-page) + +##### Multi-Factor-Authentication (MFA) Support + MFA works out-of-the-box in Tyk since luckily Okta supports it. you would need to add it to the configuration of the account holder. Under `Security --> Multifactor --> Factor types` you can choose the types you want. For instance I chose Google Authenticator. + + 1. While trying to login to the Dashboard, Okta enforced the MFA and asked me to use the Google Authenticator: + okta-mfa-setup-1 + + 2. I had to download the Google Authenticator and identify with the generated code + okta-mfa-download-google-authenticator-2 + 3. I successfully authenticated with Google Authenticator + okta-mfa-google-auth-approved-3 + +#### Common Error +If you get a `400 Bad Request` it means the profile name in the login endpoint is not identical to the profile name in the callback that you set up on Okta's app: + +- On Okta's app - `Login redirect URIs:` `http://localhost:3010/auth/{PROFILE-NAME-IN-TIB}/openid-connect/callback`. +- The endpoint to test - `http://localhost:3010/auth/{PROFILE-NAME-IN-TIB}/openid-connect` + +okta-bad-request-wrong-callback + +### OIDC with Auth0 + +This will walk you through securing access to your Tyk Dashboard using OpenID Connect (OIDC) identity tokens with Auth0. We also have the following video that will walk you through the process. + + + +**Prerequisites** + +* A free account with [Auth0](https://auth0.com/) +* A Tyk Self-Managed or Cloud installation +* Our Tyk Identity Broker (TIB). You can use the internal version included with a Tyk Self-Managed installation and Tyk Cloud, or an external version. See [Tyk Identity Broker](#what-is-tyk-identity-broker-tib) for more details. + +#### Create a new user in Auth0 + +1. Log in to your Auth0 account. +2. Select **Users** from the **User Management** menu. + +Auth0 Create User + +3. Click Create User and complete the new user form, using the default **Username-Password-Authentication** Connection method. +4. Click Create to save your new user. +Auth0 User profile + +#### Create an Auth0 application + +You will use settings from your Auth0 application within the Tyk Dashboard Identity profile you will create. + +1. Select Applications from the Auth0 menu. +Auth0 Applications +2. Click **Create Application**. +3. Give your application a name and select **Regular Web Application** from the applications types. +Auth0 Application information +4. Click **Create**. +5. After you application has been created select the **Basic Information** tab. +Auth0 Application Basic information +6. You will use the **Domain**, **Client Id** and **Client Secret** values in the Identity profile you create next in the Tyk Dashboard. + +#### Create an Identity Management profile in your Dashboard + +1. Log in to your Tyk Dashboard as an Admin user. +2. Select **Identity Management** from the **System Management** menu. +Create Identity profile +3. Click **Create Profile**. +4. In the **Profile action** section enter a name for your profile and make sure the **Login to Tyk Dashboard** option is selected. +Identity Profile action settings +5. Click Next. In the **Provider type** section, select **OpenID Connect**. +Identity profile Provider type +6. Click Next. Copy the **Client ID** value from your **Auth0 application** > **Basic Information** and paste it in the **Client ID / Key** field. +7. Copy the **Client Secret** value from your **Auth0 application** > **Basic Information** and paste it in the **Secret** field. +8. You need to add a **Discover URL (well known endpoint)**. Use the following URL, replacing `<>` with the **Domain** value from your **Auth0 application** > **Basic Information**. + + `https://<>/.well-known/openid-configuration` + + Tyk new identity profile configuration + +9. Copy the **Callback URL** and paste it into the **Allowed Callback URLs** field in your **Auth0 application** > **Basic Information**. +Auth0 Allowed Callback URLs +10. Click **Save Changes** to update your Auth0 Application. +11. Click **Create Profile** to save your Identity profile in your Tyk Dashboard. + +#### Test your Auth0 Login + +1. From your **Identity Management Profiles** click the profile you created to open it. +Tyk Identity Profiles +2. Click the **Login URL**. +Tyk Identity Profile Config +3. You will now see the Auth0 login form in a browser tab. +Auth0 login form +4. Enter the email address and password of your Auth0 user. +5. You may be asked to authorize your Auth0 application. +Accept Auth0 application +6. Click **Accept**. +7. You will now be taken to the Tyk Dashboard. +Tyk Dashboard from Auth0 SSO login + +### OIDC with Keycloak + +This is a walk-through of how you can use [Keycloak](https://www.keycloak.org) and our (internal/embedded) Tyk Identity Broker (TIB) to log in to your Dashboard. This guide assumes you have existing Keycloak and Tyk Pro Environments. + +#### Configuration at KeyCloak + +1. In your desired Realm, create a client of OpenID Connect type, and set your desired Client ID. + + Create Client + + Set Client Type and ID + + +2. Enable client authentication, then save the client. + + Enable Client Auth + + +3. Retrieve the Secret (from the credentials tab) of the Client you just created. You will need the Client ID and Secret in later steps. + + Retrieve Client Secret + +4. Retrieve the discovery endpoint of the realm, `https:///.well-known/openid-configuration`. + + This is accessible from β€œRealm Settings” > β€œGeneral” Tab > OpenID Endpoint Configuration. You will need it in later steps. + + Keycloak discovery endpoint + + +#### Configuration at Dashboard + +1. Log in to your Dashboard and select Identity Management, located under System Management + + Select Identity Management + + +2. Create a profile, give it a name and select β€œLogin to Tyk Dashboard” + + Create a profile + + +3. Set the provider type as β€œOpenID Connect” + + OpenID Connect provider type + + +4. Fill in the Client ID, Client Secret and Discovery URL/endpoint from Keycloak (from steps 3 and 4 in Keycloak's Side) + +5. Copy the callback URL from Tyk and then you can click "Create Profile" to save the profile. + + Copy callback URL + + +6. Go to Keycloak, and paste the callback URL you just copied to β€œValid redirect URIs” in the Keycloak Client, and then save the client. + + This can be accessed by selecting the "Settings" tab when viewing a Keycloak client. + + Add Redirect URL to keycloak client + + +#### Test Keycloak Login + +1. From your **Identity Management Profiles** click the profile you created to open it. + +2. Copy the **Login URL** and paste it into a browser tab + Copy login url + +3. You will now see the Keycloak login form. + Login to keycloak + +4. Enter the email address and password of your Keycloak user. + +5. You should now be redirected to the Tyk Dashboard and logged in + Tyk Dashboard from Keycloak SSO login + + +## SSO with SAML + +SAML authentication is a way for a service provider, such as the Tyk Dashboard or Portal, to assert the Identity of a User via a third party. + +Tyk Identity Broker can act as the go-between for the Tyk Dashboard and Portal and a third party identity provider. Tyk Identity broker can also interpret and pass along information about the user who is logging in such as Name, Email and group or role metadata for enforcing role based access control in the Tyk Dashboard. + +The provider config for SAML has the following values that can be configured in a Profile: + +`SAMLBaseURL` - The host of TIB that will be used in the metadata document for the Service Provider. This will form part of the metadata URL used as the Entity ID by the IDP. The redirects configured in the IDP must match the expected Host and URI configured in the metadata document made available by Tyk Identity Broker. + +`FailureRedirect` - Where to redirect failed login requests. + +`IDPMetaDataURL` - The metadata URL of your IDP which will provide Tyk Identity Broker with information about the IDP such as EntityID, Endpoints (Single Sign On Service Endpoint, Single Logout Service Endpoint), its public X.509 cert, NameId Format, Organization info and Contact info. + +This metadata XML can be signed providing a public X.509 cert and the private key. + +`CertLocation`: An X.509 certificate and the private key for signing your requests to the IDP, this should be one single file with the cert and key concatenated. When using internal identity broker, this value should be the id of the certificate uploaded via certificate manager in dashboard, otherwise it should be a path where the certificate is placed. + +`ForceAuthentication` - Ignore any session held by the IDP and force re-login every request. + +`SAMLEmailClaim` - Key for looking up the email claim in the SAML assertion form the IDP. Defaults to: `http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress` + +`SAMLForenameClaim` - Key for looking up the forename claim in the SAML assertion form the IDP. Defaults to: `http://schemas.xmlsoap.org/ws/2005/05/identity/claims/forename` + +`SAMLSurnameClaim` - Key for looking up the surname claim in the SAML assertion form the IDP. Defaults to: `http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname` + +Example profile configuration: + +``` expandable +{ + "ActionType": "GenerateOrLoginUserProfile", + "ID": "saml-sso-login", + "OrgID": "{YOUR_ORGANIZATION_ID}", + "CustomEmailField": "", + "IdentityHandlerConfig": { + "DashboardCredential": "{DASHBOARD_USER_API_KEY}" + }, + "ProviderConfig": { + "SAMLBaseURL": "https://{HOST}", + "FailureRedirect": "http://{DASHBOARD_HOST}:{PORT}/?fail=true", + "IDPMetaDataURL": "{IDP_METADATA_URL}", + "CertLocation":"myservice.cert", + "ForceAuthentication": false, + "SAMLEmailClaim": "", + "SAMLForenameClaim": "", + "SAMLSurnameClaim": "" + }, + "ProviderName": "SAMLProvider", + "ReturnURL": "http://{DASHBOARD_URL}:{PORT}/tap", + "Type": "redirect" +} +``` +### Video Demonstration + +We have a video that walks you through getting Tyk Dashboard SSO Access via SAML using Microsoft Azure as IDP and our internal Dashboard TIB. + + + +## SSO with LDAP + +This is an end to end worked example of how you can use LDAP and our Tyk Identity Broker (TIB) to log in to your Dashboard. + +The Tyk Dashboard is the command and control center of your Tyk installation. It allows users to manage APIs, policies, keys, etc. All of this data is stored in the Dashboard's MonogDB database, including the user accounts. This works well in a lot of situations as it allows Tyk to be self-contained, but if you already have a centralised system for managing users then you may prefer to use that instead of a separate Tyk-specific database. + +The Tyk Identity Broker (TIB) is an open-source project which can be used to integrate Tyk authentication with 3rd party identity providers (IDPs). You can use this to enable your Dashboard to authenticate users with your LDAP-powered identity providers such as Active Directory. TIB has been designed as a glue-code solution, so it can integrate with almost any identity provider (IDP). See [Tyk Identity Broker Configuration](/tyk-configuration-reference/tyk-identity-broker-configuration) for details on configuring the TIB. + +### The High Level TIB Flow: + +1. User makes an authentication request against the TIB endpoint, passing their credentials +2. TIB makes request against IDP using the credentials provided +3. TIB interprets the IDP response: + + * If successful then TIB creates a user session in the Dashboard and redirects the user to the Dashboard + + * If unsuccessful, TIB redirects the user to a failure URL + +### Steps for Configuration + +This guide assumes you already have a Tyk environment set up, with a Gateway and Dashboard. If you don't, please follow the [Tyk Self-Managed getting started guide](/tyk-self-managed/install). + +The environment used for this guide is, for simplicity's sake, all contained on a single host running Ubuntu 14.04. The hostname `my-tyk-instance.com` has been set to point at `127.0.0.1`. For production environments it is recommended that each component is hosted separately and appropriate security measures are used such as HTTPS to secure connections. + +All commands shown are run from inside the Tyk host environment. + +1. **Download TIB** + + You can download TIB from the [releases page of the TIB repository on GitHub](https://github.com/TykTechnologies/tyk-identity-broker/releases). The release names contain the architecture and version i.e. `tib-linux--.tar.gz`. This example uses `amd64` and `0.2.1` for all the commands, but you should update them to use the latest version and relevant architecture for your platform. + + First step is to download TIB onto the environment: + + ```{.copyWrapper} expandable + wget https://github.com/TykTechnologies/tyk-identity-broker/releases/download/v0.2.1/tib-linux-amd64-0.2.1.tar.gz + ``` + +2. **Extract and store TIB** + + As the other Tyk components are installed in your `/opt` directory, we recommend you install TIB there too: + + ```{.copyWrapper} + tar -xvzf tib-linux-amd64-0.2.1.tar.gz + ``` + + TIB will now be extracted to the directory `tib-0.2.1`, let's move this to `/opt` and change to that directory: + + ```{.copyWrapper} + sudo mv tib-0.2.1 /opt + cd /opt/tib-0.2.1 + ``` + +3. **Configure TIB** + + There are two configuration files for TIB: + + 1. `tib.conf` for the main application configuration settings + 2. `profiles.json` to configure the profiles which TIB will attempt to authenticate against + + Out of the box you don't need to change much, but there are several attributes you should check to make sure they are correct for your environment: + + * `Secret`: The REST API secret used when configuring TIB remotely + * `TykAPISettings.GatewayConfig.Endpoint`: The URL through which TIB can communicate with your Tyk Gateway + * `TykAPISettings.GatewayConfig.Port`: The port through which TIB can communicate with your Tyk Gateway + * `TykAPISettings.GatewayConfig.AdminSecret`: The secret required for TIB to communicate with your Tyk Gateway REST API - must match the `secret` property in your Gateway's `tyk.conf` + * `TykAPISettings.DashboardConfig.Endpoint`: The URL through which TIB can communicate with your Tyk Dashboard + * `TykAPISettings.DashboardConfig.Port`: The port through which TIB can communicate with your Tyk Dashboard + * `TykAPISettings.DashboardConfig.AdminSecret`: The secret required for TIB to communicate with your Tyk Dashboard Admin REST API - must match the `admin_secret` property in your Dashboard's `tyk_analytics.conf` + + The `tib.conf` for this example is as follows (yours might require different values): + + ```{.copyWrapper} + { + "Secret": "352d20ee67be67f6340b4c0605b044b7", + "HttpServerOptions": { + "UseSSL": false, + "CertFile": "./certs/server.pem", + "KeyFile": "./certs/server.key" + }, + "BackEnd": { + "Name": "in_memory", + "ProfileBackendSettings": {}, + "IdentityBackendSettings": { + "Hosts" : { + "localhost": "6379" + }, + "Password": "", + "Database": 0, + "EnableCluster": false, + "MaxIdle": 1000, + "MaxActive": 2000 + } + }, + "TykAPISettings": { + "GatewayConfig": { + "Endpoint": "http://localhost", + "Port": "8080", + "AdminSecret": "352d20ee67be67f6340b4c0605b044b7" + }, + "DashboardConfig": { + "Endpoint": "http://localhost", + "Port": "3000", + "AdminSecret": "12345" + } + } + } + ``` + +4. **Set up the LDAP profile** + + TIB ships with a default `profiles.json` file which contains many example configuration for different scenarios. This guide is focused on LDAP authentication for the Dashboard, so we will update `profiles.json` to contain a single profile for this purpose. + + The key attributes for LDAP profile are: + + * `ID`: The ID by which we will activate the profile by calling the appropriate TIB endpoint + * `OrgId`: The organization id which the profile is connected to - make sure this is the correct id for your organization (see the [Dashboard Admin API documentation](/api-management/dashboard-configuration#organizations-api) for details on how to retrieve this) + * `IdentityHandlerConfig.DashboardCredential`: The Dashboard API Access credential which is used as authorization header + * `ProviderConfig.FailureRedirect`: The URL which TIB will redirect to if the authentication fails + * `ProviderConfig.LDAPPort`: The port through which TIB can communicate with your LDAP server + * `ProviderConfig.LDAPServer`: The URL through which TIB can communicate with your LDAP server + * `ProviderConfig.LDAPUserDN`: The distinguished name which TIB will use to identify the user - this should be updated to match your LDAP installation and must retain the `*USERNAME*` token as this is replaced by the actual username at runtime + * `ReturnURL`: The URL which TIB will redirect to if the authentication succeeds - this should be the `/tap` endpoint of your Tyk Dashboard + + The `profiles.json` for this example is as follows (again, update values for your environment): + + ```{.copyWrapper} + [ + { + "ActionType": "GenerateOrLoginUserProfile", + "ID": "1", + "OrgID": "59bfdf5b56c02c065d24638e", + "IdentityHandlerConfig": { + "DashboardCredential": "bb5735026be4400e67ed9801c2f1e2f9" + }, + "ProviderConfig": { + "FailureRedirect": "http://my-tyk-instance.com:3000/?fail=true", + "LDAPAttributes": [], + "LDAPPort": "389", + "LDAPServer": "ldap.forumsys.com", + "LDAPUserDN": "cn=*USERNAME*,dc=example,dc=com" + }, + "ProviderName": "ADProvider", + "ReturnURL": "http://my-tyk-instance.com:3000/tap", + "Type": "passthrough" + } + ] + ``` + + Notice that this is a JSON array object with a single element; an LDAP profile. The LDAP server referenced by this profile is the freely-available service provided forumsys.com. See [their documentation](https://www.forumsys.com/tutorials/integration-how-to/ldap/online-ldap-test-server/) for more information. You can use any OpenLDAP compatible server. + +5. **Start TIB** + + Start TIB by executing the TIB binary. This will produce an output log into the console which you can use to watch TIB process requests. Since TIB looks for the config file in the local directory, you should execute the application from there too. + + ```{.copyWrapper} + cd /opt/tib-0.2.1 + ./tib + ``` + + If all is well you should see TIB output a few messages when it starts: + + ``` + toth/tothic: no SESSION_SECRET environment variable is set. The default cookie store is not available and any calls will fail. Ignore this warning if you are using a different store. + INFO[0000] Tyk Identity Broker v0.2 + INFO[0000] Copyright Martin Buhr 2016 + + DEBU[0000] [MAIN] Settings Struct: {{http://localhost 8080 352d20ee67be67f6340b4c0605b044b7} {http://localhost 3000 12345}} + INFO[0000] [MAIN] Initialising Profile Configuration Store + INFO[0000] [IN-MEMORY STORE] Initialised + INFO[0000] [MAIN] Initialising Identity Cache + INFO[0000] [REDIS STORE] Initialised + INFO[0000] [FILE LOADER] Loaded: 1 profiles from profiles.json + INFO[0000] [MAIN] Broker Listening on :3010 + ``` + + Start a new shell session to carry on with the remaining process. + + + +6. **Create a login page** + + TIB works by having credentials sent to it, so a login page must be made in order to fulfill this requirement. For this example we will create a basic login form hosted by Nginx. We can't just place the login page in our Dashboard directory as the Dashboard is not a standard web server, it only serves the pages which it has been compiled to serve. Any non-compiled page will produce a 404 response. + + Install Nginx and start it: + + ```{.copyWrapper} + sudo apt-get install nginx + sudo service nginx start + ``` + + Nginx will now serve pages out of the default web root directory `/usr/share/nginx/www`. We now need to create a web page there. This command will pipe the echoed text into a file called `login.html` which is stored in the web root: + + ```{.copyWrapper} + echo \ + " \ + \ + Tyk Dashboard LDAP login \ + \ + \ +
\ + username:
\ + password:
\ + \ +
\ + \ + " \ + | sudo tee /usr/share/nginx/www/login.html > /dev/null + ``` + + The login form contains two inputs named `username` and `password`. TIB looks for these exact parameter names when processing the request, so if you are creating your own login page you must use these input names. + + Please make sure you are using `POST` method in the form, to avoid browser caching. + + The form action `http://my-tyk-instance.com:3010/auth/1/ldap` is the TIB endpoint which will start the authentication process. The URL can be broken down as follows: + + * `http://my-tyk-instance.com`: The method and hostname used to connect to TIB - you should use HTTPS to prevent confidential data from being exposed + * `3010`: The default port for TIB + * `auth`: The special TIB endpoint which accepts authentication requests + * `1`: The number of the profile which we are using - matches against the `ID` property of the profile in `profiles.json` + * `ldap`: We need to add a string to the end of the request, so we have used `ldap` here + +7. **Update the Dashboard config** + + Update the Dashboard config so that any unauthenticated requests are redirected to your custom login page. We do this by updating the `sso_custom_login_url` property of the Dashboard's `tyk_analytics.conf` file, which by default is located in the `/opt/tyk-dashboard` directory. For example (ommitting all other lines in the config file and trailing comma): + + ```{.copyWrapper} + "sso_custom_login_url": "http://my-tyk-instance.com/login.html" + ``` + + Since the Dashboard runs on port 3000 by default, this URL will use the default HTTP port of 80 which will be handled by Nginx. + +8. **Test that it works** + + Now that we have TIB installed and configured, Nginx installed and hosting our custom login page, and the Dashboard configured to redirect to that login page we can now test the solution. Remember that this example is using the LDAP provided at forumsys.com, so if you are using your own LDAP then substitute the username and password with appropriate values from your system. + + 1. Open a web browser (if you're already logged in to the Dashboard, logout now) and attempt to access the Dashboard - `http://my-tyk-instance.com:3000` + 2. This should be redirected to the custom login page - `http://my-tyk-instance.com/login.html` + 3. Enter `read-only-admin` as the username + 4. Enter `password` as the password + 5. Submit the form + 6. You should now be logged in to the Dashboard + + +## Advance LDAP Configuration + +The LDAP Identity Provider gives you functionality to bind a user to an LDAP server based on a username and password configuration. The LDAP provider currently does not extract user data from the server to populate a user object, but will provide enough defaults to work with all handlers. + +### Log into the Dashboard using LDAP + + +Below is a sample TIB profile that can be used to log a user into the Dashboard using an LDAP pass-through provider: + +```{.copyWrapper} +{ + "ActionType": "GenerateOrLoginUserProfile", + "ID": "4", + "OrgID": "{YOUR-ORG-ID}", + "IdentityHandlerConfig": { + "DashboardCredential": "ADVANCED-API-USER-API-TOKEN" + }, + "ProviderConfig": { + "FailureRedirect": "http://{DASH-DOMAIN}:{DASH-PORT}/?fail=true", + "LDAPAttributes": [], + "LDAPPort": "389", + "LDAPServer": "localhost", + "LDAPUserDN": "cn=*USERNAME*,cn=dashboard,ou=Group,dc=test-ldap,dc=tyk,dc=io" + }, + "ProviderName": "ADProvider", + "ReturnURL": "http://{DASH-DOMAIN}:{DASH-PORT}/tap", + "Type": "passthrough" +} + +``` expandable + +The only step necessary to perform this is to send a POST request to the LDAP URL. + +TIB can pull a username and password out of a request in two ways: + +1. Two form fields called "username" and "password" +2. A basic auth header using the Basic Authentication standard form + +By default, TIB will look for the two form fields. To enable Basic Auth header extraction, add `"GetAuthFromBAHeader": true` to the `ProviderConfig` section. + +The request should be a `POST`. + +If you make this request with a valid user that can bind to the LDAP server, Tyk will redirect the user to the dashboard with a valid session. There's no more to it, this mechanism is pass-through and is transparent to the user, with TIB acting as a direct client to the LDAP provider. + + + +The `LDAPUserDN` field MUST contain the special `*USERNAME*` marker in order to construct the users DN properly. + + + + +### Generate an OAuth token using LDAP + + +The configuration below will take a request that is posted to TIB, authenticate it against LDAP, if the request is valid, it will redirect to the Tyk Gateway OAuth clients' `Redirect URI` with the token as a URL fragment: + +```{.copyWrapper} +{ + "ActionType": "GenerateOAuthTokenForClient", + "ID": "6", + "IdentityHandlerConfig": { + "DashboardCredential": "{DASHBAORD-API-ID}", + "DisableOneTokenPerAPI": false, + "OAuth": { + "APIListenPath": "{API-LISTEN-PATH}", + "BaseAPIID": "{BASE-API-ID}", + "ClientId": "{TYK-OAUTH-CLIENT-ID}", + "RedirectURI": "http://{APP-DOMAIN}:{PORT}/{AUTH-SUCCESS-PATH}", + "ResponseType": "token", + "Secret": "{TYK-OAUTH-CLIENT-SECRET}" + } + }, + "MatchedPolicyID": "POLICY-ID", + "OrgID": "53ac07777cbb8c2d53000002", + "ProviderConfig": { + "FailureRedirect": "http://{APP-DOMAIN}:{PORT}/failure", + "LDAPAttributes": [], + "LDAPPort": "389", + "LDAPServer": "localhost", + "LDAPUserDN": "cn=*USERNAME*,cn=dashboard,ou=Group,dc=ldap,dc=tyk-ldap-test,dc=com" + } + "ProviderName": "ADProvider", + "ReturnURL": "", + "Type": "passthrough" +} +``` + +This configuration is useful for internal APIs that require valid OAuth tokens (e.g.a webapp or mobile app) but needs validation by an LDAP provider. + +### Log into the Developer Portal using LDAP + + +LDAP requires little configuration, we can use the same provider configuration that we used to log into the Dashboard to target the Portal instead - notice the change in the handler configuration and the return URL: + +```{.copyWrapper} +{ + "ActionType": "GenerateOrLoginDeveloperProfile", + "ID": "5", + "IdentityHandlerConfig": { + "DashboardCredential": "822f2b1c75dc4a4a522944caa757976a" + }, + "OrgID": "53ac07777cbb8c2d53000002", + "ProviderConfig": { + "FailureRedirect": "http://{PORTAL-DOMAIN}:{PORTAL-PORT}/portal/login/", + "LDAPAttributes": [], + "LDAPPort": "389", + "LDAPServer": "localhost", + "LDAPUserDN": "cn=*USERNAME*,cn=dashboard,ou=Group,dc=test-ldap,dc=tyk,dc=io" + }, + "ProviderConstraints": { + "Domain": "", + "Group": "" + }, + "ProviderName": "ADProvider", + "ReturnURL": "http://{PORTAL-DOMAIN}:{PORTAL-PORT}/portal/sso/", + "Type": "passthrough" +} +``` expandable + +Once again, a simple `POST` request is all that is needed to validate a user via an LDAP provider. + +### Using advanced LDAP search + +In some cases validation of a user CN is not enough, and it requires verifying if a user match some specific rules, like internal team ID. In this case TIB provides support for doing additional LDAP search check, and if result of this search returns only 1 record, it will pass the user. + +To make it work you need to specify 3 additional attributes in profile configuration file: + +* `LDAPBaseDN` - base DN used for doing LDAP search, for example `cn=dashboard,ou=Group` +* `LDAPFilter` - filter applied to the search, should include the `*USERNAME*`variable. For example: `((objectCategory=person)(objectClass=user)(cn=*USERNAME*))` +* `LDAPSearchScope` - This specifies the portion of the target subtree that should be considered. Supported search scope values include: 0 - baseObject (often referred to as "base"), 1 - singleLevel (often referred to as "one"), 2 - wholeSubtree (often referred to as "sub") + +For additional information about [LDAP search protocol](https://www.ldap.com/the-ldap-search-operation) + +Example profile using LDAP search filters: +```{.copyWrapper} +{ + "ActionType": "GenerateOAuthTokenForClient", + "ID": "2", + "IdentityHandlerConfig": { + "DashboardCredential": "ADVANCED-API-USER-API-TOKEN", + "DisableOneTokenPerAPI": false, + "OAuth": { + "APIListenPath": "oauth-1", + "BaseAPIID": "API-To-GRANT-ACCESS-TO", + "ClientId": "TYK-OAUTH-CLIENT-ID", + "RedirectURI": "http://your-app-domain.com/target-for-fragment", + "ResponseType": "token", + "Secret": "TYK-OAUTH-CLIENT-SECRET" + } + }, + "MatchedPolicyID": "POLICY-TO-ATTACH-TO-KEY", + "OrgID": "53ac07777cbb8c2d53000002", + "ProviderConfig": { + "FailureRedirect": "http://yourdomain.com/failure-url", + "LDAPAttributes": [], + "LDAPBaseDN": "cn=dashboard,ou=Group,dc=ldap,dc=tyk-test,dc=com", + "LDAPEmailAttribute": "mail", + "LDAPSearchScope": 2, + "LDAPFilter": "(&(objectcategory=user)(sAMAccountName=*USERNAME*)(memberOf=CN=RL - PAT - T1-00002,OU=Role,OU=Security Roles,DC=company,DC=net))", + "LDAPPort": "389", + "LDAPServer": "ldap.company.com", + "LDAPUserDN": "*USERNAME*@company.com" + }, + "ProviderName": "ADProvider", + "ReturnURL": "", + "Type": "passthrough" +} +``` expandable + + +## Custom Proxy Identify Provider + +The proxy identity provider is a generic solution to more legacy problems, as well as a way to handle flows such as basic auth access with third party providers or OAuth password grants where the request can just be passed through to the providing endpoint to return a direct response. + +The proxy provider will take a request, proxy it to an upstream host, capture the response, and analyze it for triggers of "success", if the triggers come out as true, then the provider will treat the request as authenticated and hand over to the Identity Handler to perform whatever action is required with the user data. + +Success can be triggered using three methods: + +1. Response code: e.g. if this is an API request, a simple `200` response would suffice to act as a successful authentication. +2. Response body exact match: You can have a base64 encoded body that you would expect as a successful match, if the two bodies are the same, then the request will be deemed successful. +3. Regex: Most likely, the response might be dynamic (and return a response code, timestamp or other often changing parameter), in which case you may want to just match the response to a regex. + +These can be used in conjunction as gates, e.g. a response must be `200 OK` and match the regex in order to be marked as successful. + +### JSON Data and User names + +The Proxy provider can do some clever things, such as extract JSON data from the response and decode it, as well as pull username data from the Basic Auth header (for example, if your identity provider supports dynamic basic auth). + +### Log into the Dashboard with the Proxy Provider + +The configuration below will proxy a request to `http://{TARGET-HOSTNAME}:{PORT}/` and evaluate the response status code, if the status code returned is `200` then TIB will assume the response is JSON (`"ResponseIsJson": true`) to extract an access token (e.g. if this is an OAuth pass-through request) and try and find an identity to bind the Dashboard user to in the `user_name` JSON field of the response object (`"UsernameField": "user_name"`): + +```{.copyWrapper} +{ + "ActionType": "GenerateOrLoginUserProfile", + "ID": "7", + "OrgID": "{YOUR-ORG-ID}", + "ProviderConfig": { + "AccessTokenField": "access_token", + "ExtractUserNameFromBasicAuthHeader": false, + "OKCode": 200, + "OKRegex": "", + "OKResponse": "", + "ResponseIsJson": true, + "TargetHost": "http://{TARGET-HOSTNAME}:{PORT}/", + "UsernameField": "user_name" + }, + "ProviderName": "ProxyProvider", + "ReturnURL": "http://{DASH-DOMAIN}:{DASH-PORT}/tap", + "Type": "redirect" +} +``` + + +## JSON Web Encryption with OIDC + +**Prerequisites** + +- Tyk Identity Broker v1.6.1+ or Tyk Dashboard v5.7.0+ (JWE feature is available from these versions and in all subsequent releases). +- An Identity Provider (IdP) that supports JSON Web Encryption (JWE) +- A certificate with a private key for Tyk (used to decrypt the ID token) +- A public key file for the IdP (used to encrypt the ID token) + +### Steps for Configuration + +1. **Prepare Encryption Keys** + + - Load the certificate with the private key into Tyk: + - **For embedded TIB in Dashboard:** Use Tyk Dashboard's certificate manager. In the below image you can see the module in dashboard that allows to upload certificates: + Certificate manager + - **For standalone TIB:** Store the certificate as a file accessible to Tyk + + - Load the public key into your IdP for ID token encryption (process varies by IdP) + +2. **Configure the Identity Provider** +- Create a new client in your IdP for Tyk Identity Broker + +3. **Setup OIDC Profile** + - Create a new [TIB profile](#exploring-tib-profiles): + - Select Social > OIDC as the provider + - Enter the client key and client secret from the IdP + - Copy the callback URL from TIB and add it to the IdP client's allowed redirect URLs + Profile creation + - Test the basic SSO flow to ensure it's working correctly + +4. **Enable JWE** + - [Updated the TIB profile via API](/tyk-identity-broker/tib-rest-api#update-profile) + - Add the following fields to the `ProviderConfig` section: + + ```json + ... + "ProviderConfig": { + "JWE": { + "Enabled": true, + "PrivateKeyLocation": "CERT-ID" + }, + ... + ``` + + - Set `PrivateKeyLocation` to either: + - The certificate ID from the certificate manager, or + - The file path where the certificate and private key are stored + + - Update the IdP client configuration + - Enable JWE for the client + - Provide the public key for encryption + +5. **Verification** + - Test the complete flow with JWE enabled to ensure proper functionality. + +### Troubleshooting +While setting up JWE with Tyk Identity Broker, you may encounter some challenges. This section outlines common issues and their solutions to help you navigate the implementation process smoothly. + +1. **oauth2: error decoding JWT token: jws: invalid token received, not all parts available** it means that JWE is not enabled in the profile and the IDP is already using JWE. +2. **JWE Private Key not loaded** Tyk encountered some issues while loading the certificate with the private key. Ensure that the path or certId are correct. \ No newline at end of file diff --git a/api-management/gateway-config-introduction.mdx b/api-management/gateway-config-introduction.mdx new file mode 100644 index 00000000..3e6373ce --- /dev/null +++ b/api-management/gateway-config-introduction.mdx @@ -0,0 +1,90 @@ +--- +title: "Configuring Tyk Gateway" +'og:description': "Explain the concept of Tyk API definition and the different types Tyk offers" +tags: ['API Definition', 'API Definition Object', 'API Definition Location'] +sidebarTitle: "Overview" +--- + +## Introduction + +Tyk API Gateway is a [reverse-proxy](https://en.wikipedia.org/wiki/Reverse_proxy) that serves as an intermediary managing API traffic between clients and the upstream API service. It consists of a series of middleware blocks that process API requests received from clients. These middleware perform various checks and transformations of and to the request preparing it to be routed to the upstream. The upstream API service executes core business logic and returns responses to Tyk Gateway. The response is similarly passed through a series of middleware blocks before being returned to the client. + +Each of these middleware can be configured so that it will only allow the specific requests that you want to reach your upstream, and in the correct form. The request middleware chain encompasses functionality that includes: + +- listening for requests +- authentication and authorization of the client +- rate and quota limiting +- checking that the request is valid +- applying transformations to the payload and headers +- triggering event handlers that can notify external systems of certain events +- checking availability of the upstream service +- ... and finally routing to the correct target applying load balancing between multiple upstreams if required + +You can even create custom middleware (plugins) that will perform non-standard checks and transformations. As you can imagine Tyk has a lot of configuration options to implement all of this! + + +## Configuring the Gateway + +Tyk Gateway is configurable at three levels of granularity: + +- *Gateway level* settings that apply to all API proxies hosted on Tyk +- *API level* settings that apply to a specific API proxy +- *Endpoint level* settings that apply to specific endpoints (operations consisting of HTTP method and path) within an API proxy + +Some features can be configured at multiple levels. Where this is the case, specific precedence rules apply and are described in the relevant section of the documentation. + +### Gateway level settings + +Gateway level settings are stored in a file (typically `tyk.conf`) that is applied when the Gateway starts up, affecting all API proxies deployed on Tyk. They can also be configured using the equivalent environment variables. The Gateway level settings are documented [here](/tyk-oss-gateway/configuration). + +If you are using a config file you can store settings, typically secrets, in environment variables or an external key-value store and provide references to the stored keys within the configuration file. This is explained [here](/tyk-configuration-reference/kv-store). + +### API and endpoint level settings + +API and endpoint level settings are configured using an *API definition*. + +This is a structured JSON object that encapsulates all of the details that apply specifically to that API, including the listen path, upstream target details, valid endpoints and operations, rate limits, authentication, versioning, and both built-in and custom middleware. + +You can store settings, typically secrets, in environment variables or an external key-value store and provide references to the stored keys within the API definition. This is explained [here](/tyk-configuration-reference/kv-store). + +API definition objects can be compact for a basic pass-through API, and can become very complex and large for APIs that require significant processing to be completed both before the request is proxied to the upstream service and once the response is received. + + +## API Definitions + +An *API definition* is the specification for an API proxy, providing Tyk with everything it needs to receive and process requests. Using Tyk's mock response, virtual endpoint and custom plugin functionality, you don't even need an upstream service - with a single API definition you can emulate a service entirely within Tyk, providing a [mock response](/api-management/traffic-transformation/mock-response). + +Tyk supports two types of API definition depending on the type of service that you are looking to proxy: + +- [Tyk OAS API definitions](/api-management/gateway-config-tyk-oas) are used for REST and streaming use cases +- [Tyk Classic API definitions](/api-management/gateway-config-tyk-classic) are used for GraphQL, XML/SOAP and TCP services + + + + + For versions of Tyk prior to 5.8 not all Gateway features can be configured using the Tyk OAS API definition, for edge cases you might need to use Tyk Classic for REST APIs, though we recommend updating to Tyk 5.8 and adopting Tyk OAS. + + + + +### Migrating to Tyk OAS + +In Tyk 4.1, we introduced the Tyk OAS API definition but initially it supported only a subset of the Gateway configuration options offered by Tyk Classic. Since then we have gradually added support until finally, with the launch of Tyk 5.8, we have reached effective parity with Tyk Classic and now recommend that Tyk OAS is used exclusively for REST use cases. + +**Tyk 5.8 continues to support Tyk Classic for REST, but we will not be adding support for new features to this API definition style and strongly recommend migrating to Tyk OAS.** + +For Tyk Dashboard users with an existing portfolio of Tyk Classic API definitions, we provide a [migration tool](/api-management/migrate-from-tyk-classic), available via the Dashboard API and UI. + +### Storing API definitions + +For Tyk Open Source users, API definitions should be stored in `.json` files in the following location accessible by the Tyk Gateway: +- `/var/tyk-gateway/apps` (Linux) +- `/opt/tyk-gateway/apps` (Docker) + +For Tyk Dashboard users, API definitions will be kept in your [main storage](/api-management/dashboard-configuration#data-storage-solutions). + +### A note on terminology + +It's important not to confuse the *API proxy* with the API for the upstream service. Typically we refer to *API proxy* or *API* when refering to the endpoints exposed on Tyk Gateway and *upstream* or *upstream API* for the service that you develop and deploy to perform your business logic and data handling. + + diff --git a/api-management/gateway-config-managing-classic.mdx b/api-management/gateway-config-managing-classic.mdx new file mode 100644 index 00000000..380ee1b7 --- /dev/null +++ b/api-management/gateway-config-managing-classic.mdx @@ -0,0 +1,556 @@ +--- +title: "Managing Tyk Classic API Definition" +'og:description': "How to manage Tyk Classic API definition" +tags: ['Tyk Classic API', 'Create', 'Update', 'Import', 'API Key', 'Security Policy'] +sidebarTitle: "Tyk Classic APIs" +--- + +import CreateApiInclude from '/snippets/create-api-include.mdx'; +import CreateSecurityPolicyInclude from '/snippets/create-security-policy-include.mdx'; +import CreateApiKeyInclude from '/snippets/create-api-key-include.mdx'; +import ImportApiInclude from '/snippets/import-api-include.mdx'; +import { ButtonLeft } from '/snippets/ButtonLeft.mdx'; + +## Create an API + +### What does it mean to create an API in Tyk + +You have a running service with an API that you want your users to consume; you want to protect and manage access to that API using Tyk Gateway - how do you do that? +
+For Tyk Gateway to protect and [reverse proxy](https://en.wikipedia.org/wiki/Reverse_proxy) calls to your upstream service, you need to configure an API on Tyk Gateway. The minimum information that Tyk requires is the **listen path** (which is a path on the Tyk Gateway URL that you want your consumers to call) and your **API URL** (which is the URL of your service to which Tyk should forward requests). +
+This information and other configuration values are stored in an object called a *Tyk API Definition*. Once you have created your Tyk API Definition and deployed it in the Gateway, Tyk can start serving your consumers, forwarding their requests to your upstream service's API. + +To reach a detailed guide to creating Tyk API Definitions, please choose the tab for the product you are using: + +### Tyk Cloud + +Tyk Cloud is a fully managed service that makes it easy for API teams to create, secure, publish and maintain APIs at any scale, anywhere in the world. Tyk Cloud includes everything you need to manage your global API ecosystem: [Tyk Gateways](/tyk-oss-gateway), [Tyk Dashboard](/tyk-dashboard), [Tyk Developer Portal](/tyk-developer-portal) and [Universal Data Graph](/api-management/data-graph#overview). +
+ +To embark on your API journey with Tyk Cloud, we recommend going to our [Quick Start guide](/tyk-cloud#quick-start-tyk-cloud). This guide will walk you through the process of creating your very first API in Tyk Cloud. +For an advanced step by step guide we recommend visiting our [Getting Started guide](/tyk-cloud#comprehensive-tyk-cloud-setup). This will explain advanced configuration steps relating to how to distribute your API across nodes, in addition to adding and testing your API. + +### Tyk Self-Managed + + + +If the command succeeds, you will see: +```json +{ + "action": "added", + "key": "xxxxxxxxx", + "status": "ok" +} +``` + +**What did we just do?** + +We just sent an API definition to the Tyk `/apis` endpoint. See [API definition objects](/api-management/gateway-config-tyk-classic) for details of all the available objects. These objects encapsulate all of the settings for an API within Tyk. + +Want to learn more from one of our team of engineers? + + + +### Tyk Open Source + + + +**Note: Integration with your OpenAPI documentation** + +In Tyk v4.1 we introduced support for APIs defined according to the [OpenAPI Specification v3.0.3](https://spec.openapis.org/oas/v3.0.3) (OAS). +This introduces a standard way to describe the vendor-agnostic elements of an API (the OpenAPI Definition, stored as an OpenAPI Document); we take this and add Tyk-specific configuration options to create the *Tyk OAS API Definition*. You can import your own OpenAPI document and Tyk will use this to generate the Tyk OAS API Definition. +For details on using Tyk OAS with Tyk Gateway, check out our guide to [working with Tyk OAS APIs](/api-management/gateway-config-managing-oas). + + + +**Prerequisites** + +Before you continue this tutorial, you will need a running [Tyk OSS gateway](/tyk-oss-gateway). Click the button for instructions on how to install Tyk Gateway: + + + +#### Creating an API on Tyk Gateway + +There are two ways to configure Tyk Gateway with an API definition: +1. [Create an API with the Tyk Gateway API](#using-tyk-gateway-api) - Tyk Gateway has its own APIs which provides various services including the registering of Tyk API Definitions on the Gateway. +2. [Create an API in File-based Mode](#create-an-api-in-file-based-mode) - alternatively you can create a Tyk API Definition in a file and then load it to the Gateway. + + +#### Using Tyk Gateway API + +Watch our video to learn how to add an API to Tyk's Open Source Gateway using [Postman](https://www.postman.com/downloads/). + + + +In order to use the Gateway API to create a Tyk API Definition you will need the API key for your deployment's Gateway API and then issue just one command to create the API and make it live. + +1. **Make sure you know your API secret** + + The API key to access your Tyk Gateway API is stored in your `tyk.conf` file; the property is called `secret`. You will need to provide this value in a header called `x-tyk-authorization` when making calls to the Gateway API. + +2. **Create an API** + + To create the API, let's send a Tyk API definition to the `/apis` endpoint on your Tyk Gateway. Remember to change the `x-tyk-authorization` value (API key) in the header of your API call and set the domain name and port to target your Tyk Gateway in the `curl` command. + ```curl expandable + curl -v -H "x-tyk-authorization: {your-secret}" \ + -s \ + -H "Content-Type: application/json" \ + -X POST \ + -d '{ + "name": "Hello-World", + "slug": "hello-world", + "api_id": "Hello-World", + "org_id": "1", + "use_keyless": true, + "auth": { + "auth_header_name": "Authorization" + }, + "definition": { + "location": "header", + "key": "x-api-version" + }, + "version_data": { + "not_versioned": true, + "versions": { + "Default": { + "name": "Default", + "use_extended_paths": true + } + } + }, + "proxy": { + "listen_path": "/hello-world/", + "target_url": "http://httpbin.org", + "strip_listen_path": true + }, + "active": true + }' http://{your-tyk-host}:{port}/tyk/apis | python -mjson.tool + ``` + + If the command succeeds, you will see: + ```json + { + "key": "Hello-World", + "status": "ok", + "action": "added" + } + ``` + + + +All APIs deployed on Tyk Gateway are given a unique `API ID`; if you don't provide one in the Tyk API Definition when creating the API, then an `API ID` will be generated automatically. + + + +**What did we just do?** + +We just registered a new API on your Tyk Gateway by sending a Tyk API definition to your Gateway's `/apis` endpoint. +Tyk API definitions encapsulate all of the settings for an API within Tyk Gateway and are discussed in detail in the [API section](/api-management/gateway-config-tyk-classic) of this documentation. + +**Restart or hot reload** + +Once you have created the file, you will need to either restart the Tyk Gateway, or issue a hot reload command, lets do the latter: +```curl +curl -H "x-tyk-authorization: {your-secret}" -s http://{your-tyk-host}:{port}/tyk/reload/group | python -mjson.tool +``` expandable + +This command will hot-reload your API Gateway(s) and the new API will be loaded, if you take a look at the output of the Gateway (or the logs), you will see that it should have loaded Hello-World API on `/hello-world/`. + +#### Create an API in File-based Mode + + + +APIs created without API ID in file based mode are invalid. + + + + +To create a file-based API definition is very easy. + +Create a file called `api1.json` and place it in the `/apps` folder of your Tyk Gateway installation (usually in `/var/tyk-gateway`), then add the following: +```json +{ + "name": "Test API", + "slug": "test-api", + "api_id": "1", + "org_id": "1", + "auth_configs": { + "authToken": { + "auth_header_name": "Authorization" + } + }, + "definition": { + "location": "header", + "key": "x-api-version" + }, + "version_data": { + "not_versioned": true, + "versions": { + "Default": { + "name": "Default", + "use_extended_paths": true + } + } + }, + "proxy": { + "listen_path": "/test-api/", + "target_url": "http://httpbin.org/", + "strip_listen_path": true + }, + "active": true +} +``` + +**Restart or hot reload** + +Once you have created the file, you will need to either restart the Tyk Gateway, or issue a hot reload command, lets do the latter: +```curl +curl -H "x-tyk-authorization: {your-secret}" -s https://{your-tyk-host}:{port}/tyk/reload/group | python -mjson.tool +``` expandable + +This command will hot-reload your API Gateway(s) and the new API will be loaded, if you take a look at the output of the Gateway (or the logs), you will see that it should have loaded Test API on `/test-api/`. + +Your API is now ready to use via the Gateway. + +## Secure an API + +A security policy encapsulates several options that can be applied to a key. It acts as a template that can override individual sections of an API key (or identity) in Tyk. + +See [What is a Security Policy?](/api-management/policies#what-is-a-security-policy) for more details. + +### Tyk Cloud + + + +### Tyk Self Manged + + + +### Tyk Open Source + +#### Create a Policy with the Gateway + +Adding a policy to the Tyk Gateway is very easy. Polices are loaded into memory on load and so need to be specified in advanced in a file called `policies.json`. To add a policy, simply create or edit the `/policies/policies.json` file and add the policy object to the object array: + +```json +{ + "POLICYID": { + "access_rights": { + "{API-ID}": { + "allowed_urls": [], + "api_id": "{API-ID}", + "api_name": "{API-NAME}", + "versions": [ + "Default" + ] + } + }, + "active": true, + "name": "POLICY NAME", + "rate": 1000, + "per": 1, + "quota_max": 10000, + "quota_renewal_rate": 3600, + "tags": ["Startup Users"] + } +} +``` expandable + +The above creates a new policy with a policy ID that you can define, with the rate limits, and security profile that grants access to the APIs listed in the `access_rights` section. + +- `{API-ID}`: The API ID you wish this policy to grant access to, there can be more than one of these entries. +- `{API-NAME}`: The name of the API that is being granted access to (this is not required, but helps when debugging or auditing). +- `POLICY NAME`: The name of this security policy. + +The important elements: + +- `access_rights`: A list of objects representing which APIs that you have configured to grant access to. +- `rate` and `per`: The number of requests to allow per period. +- `quota_max`: The maximum number of allowed requests over a quota period. +- `quota_renewal_rate`: how often the quota resets, in seconds. In this case we have set it to renew every hour. + +## Access an API + +### Tyk Cloud + + + +You will see a 200 response with your new key: + +```yaml +{ + "api_model": {}, + "key_id": "59bf9159adbab8abcdefghijac9299a1271641b94fbaf9913e0e048c", + "data": {...} +} +``` + +The value returned in the `key_id` parameter of the response is the access key you can now use to access the API that was specified in the `access_rights` section of the call. + +### Tyk Self Managed + + + +You will see a response with your new key: + +```json +{ + "action": "create", + "key": "c2cb92a78f944e9a46de793fe28e847e", + "status": "ok" +} +``` expandable + +The value returned in the `key` parameter of the response is the access key you can now use to access the API that was specified in the `access_rights` section of the call. + +### Tyk Open Source + +To create an API Key, you will need the API ID that we wish to grant the key access to, then creating the key is an API call to the endpoint. + +**Prerequisite** + +- You will need your API secret, this is the `secret` property of the `tyk.conf` file. + +Once you have this value, you can use them to access the Gateway API, the below `curl` command will generate a key for one of your APIs, remember to replace `{API-SECRET}`, `{API-ID}` and `{API-NAME}` with the real values as well as the `curl` domain name and port to be the correct values for your environment. + +```curl +curl -X POST -H "x-tyk-authorization: {API-SECRET}" \ + -s \ + -H "Content-Type: application/json" \ + -X POST \ + -d '{ + "allowance": 1000, + "rate": 1000, + "per": 1, + "expires": -1, + "quota_max": -1, + "org_id": "1", + "quota_renews": 1449051461, + "quota_remaining": -1, + "quota_renewal_rate": 60, + "access_rights": { + "{API-ID}": { + "api_id": "{API-ID}", + "api_name": "{API-NAME}", + "versions": ["Default"] + } + }, + "meta_data": {} + }' http://localhost:8080/tyk/keys/create | python -mjson.tool +``` expandable + +The above creates a new key with the rate limits, and security profile that grants access to the APIs listed in the `access_rights` section. + +- `{API-ID}`: The API ID you wish this policy to grant access to, there can be more than one of these entries. +- `{API-NAME}`: The name of the API being granted access to (this is not required, but helps when debugging or auditing). + +The important elements: + +- `access_rights`: A list of objects representing which APIs you have configured to grant access to. +- `rate` and `per`: The number of allowed requests per period. +- `quota_max`: The maximum number of allowed requests over a quota period. +- `quota_renewal_rate`: how often the quota resets, in seconds. In this case, we have set it to renew every hour. + +You will see a response with your new key: + +```json +{ + "action": "create", + "key": "c2cb92a78f944e9a46de793fe28e847e", + "status": "ok" +} +``` expandable + +The value returned in the `key` parameter of the response is the access key you can now use to access the API that was specified in the `access_rights` section of the call. + +## Import an API + +Tyk supports importing both API Blueprint and Swagger (OpenAPI) JSON definitions from either the Gateway or the Dashboard. Tyk will output the converted file to to `stdout`. Below are the commands you can use to get Tyk to switch to command mode and generate the respective API definitions for both API Blueprint and Swagger files. + +### API Blueprint is being deprecated + +Our support for API Blueprint is being deprecated. We have been packaging [aglio](https://github.com/danielgtaylor/aglio) in our Docker images for the Dashboard which enables rendering API Blueprint Format in the portal. This module is no longer maintained and is not compatible with newer NodeJS. If you wish to continue using this feature, you can do so by installing the module yourself in your Dockerfile. The imapct of this change is that our Docker images will no longer contain this functionality. + +As a work around, you can do the following: + +* Create API Blueprint in JSON format using the Apiary [Drafter](https://github.com/apiaryio/drafter) tool +* Convert API Blueprint to OpenAPI (Swagger) using the Apiary [API Elements CLI](https://github.com/apiaryio/api-elements.js/tree/master/packages/cli) tool. + +### Using API Blueprint + + + +See [note](#api-blueprint-is-being-deprecated) above regarding deprecation of support for API Blueprint. + + + +Tyk supports an easy way to import Apiary API Blueprints in JSON format using the command line. + +Blueprints can be imported and turned into standalone API definitions (for new APIs) and also imported as versions into existing APIs. + +It is possible to import APIs and generate mocks or to generate Allow Lists that pass-through to an upstream URL. + +All imported Blueprints must be in the JSON representation of Blueprint's markdown documents. This can be created using Apiary's [Snow Crash tool](https://github.com/apiaryio/snowcrash). + +Tyk outputs all new API definitions to `stdout`, so redirecting the output to a file is advised in order to generate new definitions to use in a real configuration. + +#### Importing a Blueprint as a new API: + +Create a new definition from the Blueprint: + +```{.copyWrapper} +./tyk --import-blueprint=blueprint.json --create-api --org-id= --upstream-target="http://widgets.com/api/" +``` + +#### Importing a definition as a version in an existing API: + +Add a version to a definition: + +```{.copyWrapper} +./tyk --import-blueprint=blueprint.json --for-api= --as-version="version_number" +``` expandable + +#### Creating your API versions as a mock + +As the API Blueprint definition allows for example responses to be embedded, these examples can be imported as forced replies, in effect mocking out the API. To enable this mode, when generating a new API or importing as a version, simply add the `--as-mock` parameter. + +### Using Swagger (OpenAPI) + +Tyk supports importing Swagger documents to create API definitions and API versions. Swagger imports do not support mocking though, so sample data and replies will need to be added manually later. + +#### Importing a Swagger document as a new API + +Create a new definition from Swagger: + +```{.copyWrapper} +./tyk --import-swagger=petstore.json --create-api --org-id= --upstream-target="http://widgets.com/api/" +``` + + +When creating a new definition from an OAS 3.0 spec, you will have to manually add the listen path after the API is created. + + + + +#### Importing a Swagger document as a version into an existing API + +Add a version to a definition: + +```{.copyWrapper} +./tyk --import-swagger=petstore.json --for-api= --as-version="version_number" +``` + +#### Mocks + +Tyk supports API mocking using our versioning `use_extended_paths` setup, adding mocked URL data to one of the three list types (white_list, black_list or ignored). In order to handle a mocked path, use an entry that has `action` set to `reply`: + +```json +"ignored": [ + { + "path": "/v1/ignored/with_id/{id}", + "method_actions": { + "GET": { + "action": "reply", + "code": 200, + "data": "Hello World", + "headers": { + "x-tyk-override": "tyk-override" + } + } + } + } +], +``` + +See [Versioning](/api-management/gateway-config-tyk-classic#tyk-classic-api-versioning) for more details. + +### Import APIs via the Dashboard API + + + +### Import APIs via the Dashboard UI + +1. **Select "APIs" from the "System Management" section** + + API listing + +2. **Click "IMPORT API"** + + Add API button location + + Tyk supports the following import options: + + 1. From an Existing Tyk API definition + 2. From a Apiary Blueprint (JSON) file + 3. From a Swagger/OpenAPI (JSON only) file + 4. From a SOAP WSDL definition file (new from v1.9) + + To import a Tyk Definition, just copy and paste the definition into the code editor. + + For Apiary Blueprint and Swagger/OpenAPI, the process is the same. For example: + + Click the "From Swagger (JSON)" option from the pop-up + + Import popup + + For WSDL: + + Import WSDL + +3. **Enter API Information** + + You need to enter the following information: + + * Your **Upstream Target** + * A **Version Name** (optional) + * An optional **Service Name** and **Port** (WSDL only) + * Copy code into the editor + +4. **Click "Generate API"** + + Your API will appear in your APIs list. If you select **EDIT** from the **ACTIONS** drop-down list, you can see the endpoints (from the [Endpoint Designer](/api-management/dashboard-configuration#exploring-api-endpoint-designer)) that have been created as part of the import process. + +### Creating a new API Version by importing an API Definition using Tyk Dashboard + +As well as importing new APIs, with Tyk, you can also use import to create a new version of an existing Tyk Classic API. + +1. Open the API Designer page and select Import Version from the **Options** drop-down. + + Import API Version Drop-Down + +2. Select either OpenAPI (v2.0 or 3.0) or WSDL/XML as your source API + +3. You need to add a new **API Version Name**. **Upstream URL** is optional. + + Import API Version Configuration + +4. Click **Import API**. + + Import API + +5. Select the **Versions** tab and your new version will be available. +6. Open the **Endpoint Designer** for your API and select your new version from **Edit Version**. +7. You will see all the endpoints are saved for your new version. + +Version Endpoints + +##### Import from an OpenAPI v2.0 Document + +1. From the Import API screen, select OpenAPI. + + Import OAS 2.0 API + +2. Paste your OAS v2.0 compliant definition into the code editor. + + OAS 2.0 definition in Editor + +3. Note that the Dashboard has detected that an OAS v2.0 definition has been imported and you need to specify an upstream URL field to proceed. + + Upstream URL + +4. Click **Import API**. + + Import API + + Your API will be added to your list of APIs. diff --git a/api-management/gateway-config-managing-oas.mdx b/api-management/gateway-config-managing-oas.mdx new file mode 100644 index 00000000..4b48cee5 --- /dev/null +++ b/api-management/gateway-config-managing-oas.mdx @@ -0,0 +1,837 @@ +--- +title: "Working with Tyk OAS APIs" +'og:description': "How to work with Tyk OAS APIs" +tags: ['Tyk OAS API', 'Create', 'Update', 'Import', 'Export', 'Versioning', 'API Key', 'Security Policy'] +sidebarTitle: "Tyk OAS APIs" +--- + +## Overview + +Tyk's support for the OpenAPI Specification is designed to fit in with your existing workflows as seamlessly as possible, whether you have one of our paid offerings, or are using our free open-source Gateway. You should be able to do a huge amount in the editor of your choice. The Tyk Dashboard's API Designer will support you whether you want to create a new API from a blank slate, or just to dip into if you want a bit of help with configuring Tyk's powerful transformation middleware. + +One of the great things about working with Tyk is that the OpenAPI document containing the OAS compliant description of your service is a single file (or group of files) that you deploy throughout your workflow. You can iterate on that document within your source control system until you are totally happy. At this point, you can publish the OpenAPI description to your Developer Portal to document what a Developer needs to use the API (and nothing they don’t need to know). As the OpenAPI description is the source of truth for the Tyk OAS API definition and can be updated without impacting the Tyk Vendor Extension, you can automate deployment of updates to your API on Tyk whenever a new version is committed into your source control. This model is very popular in GitOps and CI/CD environments. + +Tyk OAS API workflow + + + +Warning + +In Tyk Gateway release 5.3.0, Tyk OAS APIs gained feature maturity. Tyk Dashboard will automatically migrate any pre-5.3.0 Tyk OAS APIs to the feature mature standard when you upgrade to 5.3.0 or later. Tyk OAS APIs prior to v5.3.0 must be manually migrated if you are using Tyk OSS Gateway. Feature mature Tyk OAS APIs may not work with pre-5.3.0 versions of Tyk Gateway. + +It is not possible to rollback to previous versions of Tyk components with Tyk OAS APIs created in 5.3.0. + +For further details, please refer to the [release notes](/developer-support/release-notes/gateway) for Tyk Gateway v5.3.0. + + + +### API Definition Management with Tyk + +There are three methods by which API definitions can be deployed to Tyk: using the [Tyk Dashboard API Designer](/api-management/dashboard-configuration), using the [Tyk Dashboard API](/api-management/dashboard-configuration#exploring-the-dashboard-api) and using the [Tyk Gateway API](/tyk-gateway-api). + +The first two options provide access to the powerful licensed features of Tyk, whilst the third is used for open source deployments. Tyk provides additional tools to assist with automation when using the Tyk Dashboard API - namely [Tyk Operator](/api-management/automations/operator)(for Kubernetes deployments) and [Tyk Sync](/api-management/automations/sync) (for gitops). + +| Feature | API Designer | Tyk Dashboard API | Tyk Gateway API | +| :------------------- | :-------------- | :------------------- | :----------------- | +| Work with YAML format | βœ… | βœ… | ❌ | +| Work with JSON format | βœ… | βœ… | βœ… | +| Import an OpenAPI description | βœ… | βœ… | βœ… | +| Import a complete Tyk OAS API definition | βœ… | βœ… | βœ… | +| Import [multi-part OpenAPI descriptions](/api-management/gateway-config-managing-oas#multi-part-openapi-documents) | βœ… | βœ… | ❌ | +| Apply API [Templates](/api-management/dashboard-configuration#governance-using-api-templates) | βœ… | βœ… | ❌ | +| Export the OpenAPI description | βœ… | βœ… | βœ… | +| Export the Tyk OAS API definition | βœ… | βœ… | βœ… | +| Update API with new OpenAPI description | βœ… | βœ… | βœ… | +| Manage API versions | βœ… | βœ… | βœ… | +| Assign APIs to [Categories](/api-management/dashboard-configuration#governance-using-api-categories) | βœ… | βœ… | ❌ | +| Assign API [Owners](/api-management/user-management#api-ownership) | βœ… | βœ… | ❌ | + +## Creating an API + +Tyk is designed to fit into your workflow, so has full support for you to import your existing OpenAPI descriptions as the starting point for a Tyk OAS API. Tyk can automatically configure aspects of the Gateway's API security and management functions based upon the content of the OpenAPI description, for example using the security settings to configure client authentication or the endpoint examples and schemas to configure request validation and mock response middleware. + +Alternatively, if you don't have an existing OpenAPI description, you can use the API Designer to bootstrap one for you: build your API in Tyk and then export an OAS compliant description that you can use elsewhere, for example as documentation for your new API. + +### Using Tyk Dashboard API Designer to create an API + +In this tutorial we guide you through the steps to create a new Tyk OAS API using the GUI. + +{/* - hiding this video as it is out of date */} + +1. Start by selecting **APIs** from the **API Management** section + + Add new API + +2. Now select **Add New API** and then, choose **Design from scratch** + + Start designing a new API + +3. Now complete the basic configuration for your new API following the guided steps providing: + - API name + - API type (**HTTP**) + - API style (**OpenAPI**) + - [API template](/api-management/dashboard-configuration#working-with-api-templates-using-the-template-designer) (optional) + - Upstream URL + + Basic configuration of the new API + +4. Deploy the API to your Gateway + + - If you are using Tyk Cloud or a [sharded](/api-management/multiple-environments) deployment you will be prompted to select on which Gateways the API should be deployed + + Choose where to deploy the API + + - You need to set the **API status** (if you set this to **Active**, Tyk will accept requests to the API) + - You need to set the **Access** (set this to **External** to expose your API outside Tyk so that your clients can consume it) + - When creating a new API you will probably want to set API status to **Inactive** while you configure the rest of the API definition + + Set API Status + + Click **Save API** to create the API definition and, depending on the options you chose for API status and access, deploy the API to your gateway to start serving traffic. + + + + + You can see the URL given to your API, in the Info section displayed at the top of the page (**API URL**). + + + +5. Secure your API by configuring [client authentication](/api-management/client-authentication) + + From the API page: + + 1. Click **Edit** + 2. Scroll down to the **Server** section and enable **Authentication** + 3. Select **Auth Token** from the drop-down list + 4. For **Authentication token location** select **Use header value** + 5. Note that the default Auth key header name is *Authorization* + 6. Save your API + +6. Declare endpoints for your API + + 1. After selecting **Edit**, move to the **Endpoints** tab. + + Add new endpoint + + + 2. Click **Add Endpoint** then complete the requested details for the new endpoint: + + - Select a method from the drop-down list + - Add a path for your endpoint + - Add an optional summary and description + - select **Add Endpoint** + + Provide the details of the new endpoint + + 3. Your new endpoint will now be listed in the Endpoints tab + + List of all endpoints declared for the API + + 4. You can now add [middleware](/api-management/traffic-transformation) to your endpoint via the **Add Middleware** button. + + 5. Click **Save API** to apply the changes to your API. + +7. Test your API + + From the **Info** section, copy the [API base path](/api-management/gateway-config-managing-oas#api-base-path) and send a request to the API without providing an authorization token: + + ``` expandable + curl --location --request GET 'http://localhost:8181/petstore/' \ + --header 'Authorization: wrongkey' + ``` + + Note that the Gateway will respond with the following error message, confirming that authentication is required: + + ```.json + { + "error": "Access to this API has been disallowed" + } + ``` + +### Using your own code editor to create Tyk OAS API definitions + +The API definition is often generated either from the codebase or using API design tools (such as [Swagger Editor](https://editor.swagger.io/), [Postman](https://www.postman.com/) and [Stoplight](https://stoplight.io/)). + +To enjoy writing a *Tyk OAS API definition* as if it is [a native programming language](https://tyk.io/blog/get-productive-with-the-tyk-intellisense-extension/), you can add the [Tyk OAS API definition schema](https://raw.githubusercontent.com/TykTechnologies/tyk-schemas/main/JSON/draft-04/schema_TykOasApiDef_3.0.x.json) to your favorite IDE or editor. We have published a Tyk VS Code extension that provides Tyk API schema validation and auto-completion (both OAS and other schemas) in the [VS Code marketplace](https://marketplace.visualstudio.com/items?itemName=TykTechnologiesLimited.tyk-schemas). You can use it to create Tyk objects in your IDE (Tyk API definitions, Key and Tyk config file). + +#### Loading the API definition into Tyk + + + + + +Armed with a Tyk OAS API definition, in YAML or JSON format, you can use this to create an API in Tyk Dashboard with only a few clicks. + +1. Start by selecting **APIs** from the **API Management** section + + Add new API + +2. Now select **Add New API** and then, choose **Import**. + + Loading the API definition into Tyk Dashboard + + Note that you can optionally apply an [API template](/api-management/dashboard-configuration#governance-using-api-templates) by choosing **Start from template** as explained [here](/api-management/dashboard-configuration#using-a-template-when-creating-a-new-api), however in this explanation we will not be applying a template. + +3. From the Import API screen, select **Tyk API** because the object you want to import to Tyk is a complete API definition. + + Choosing what to import + + + + + On the Import API screen, there are three options for Import Type, it is important to select the correct one for the object that you want to load into Tyk: + + - openAPI is used only for [OpenAPI descriptions](/api-management/gateway-config-tyk-oas#openapi-description) (without the [Tyk Vendor Extension](/api-management/gateway-config-tyk-oas#tyk-vendor-extension)) + - TykAPI is used for a full [Tyk OAS API definition](/api-management/gateway-config-tyk-oas#what-is-a-tyk-oas-api-definition) (comprising OpenAPI description plus Tyk Vendor Extension) or Tyk Classic API definition + - WSDL/XML is used for WSDL/XML content and will result in a Tyk Classic API + + + +4. Now you can paste the entire Tyk OAS API definition into the text editor. + + Loading the API definition into Tyk Dashboard + +5. Select **Import API** to complete the import and create the API based on your API definition. + + + + + +When making calls to the Tyk Dashboard API you'll need to set the domain name and port for your environment and provide credentials in the `Authorization` field for Tyk to authorize your request, as follows: + +| Interface | Port | Authorization Header | Authorization credentials | +| :------------------- | :------ | :---------------------- | :----------------------------- | +| Tyk Dashboard API | 3000 | `Authorization` | From Dashboard User Profile | + +You can obtain your authorization credential (Dashboard API key) from the Tyk Dashboard UI: + +- Select **Edit profile** from the dropdown that appears when you click on your username in the top right corner of the screen +- Scroll to the bottom of the page were you will see your **Tyk Dashboard API Access Credentials** + +You will also need to have β€˜admin’ or β€˜api:write’ permission if [RBAC](/api-management/user-management) is enabled. + +To create the API in Tyk, you simply send your Tyk OAS API Definition in the payload to the `POST /api/apis/oas` endpoint of your Tyk Dashboard API. + +| Property | Description | +| :-------------- | :-------------------------- | +| Resource URL | `/api/apis/oas` | +| Method | `POST` | +| Type | None | +| Body | Tyk OAS API Definition | +| Parameters | Query: `templateID` | + +Using [this](https://bit.ly/39jUnuq) API definition it is possible to create a Tyk OAS API on your Tyk Gateway that forwards requests to the [Swagger Petstore](https://petstore3.swagger.io) request/response service. + +``` +curl -H "Authorization: ${DASH_KEY}" -H "Content-Type: application/json" ${DASH_URL}/apis/oas -d "$(wget -qO- https://bit.ly/39jUnuq)" +``` + +**Check request response** + +If the command succeeds, you will see the following response, where `Meta` contains the unique identifier (`id`) for the API you have just created. If you did not provide a value in the `id` field, then Tyk will automatically assign one. + +``` +{ + "Status": "OK", + "Message": "API created", + "Meta": {NEW-API-ID} +} +``` expandable + +What you have done is to send a Tyk OAS API definition to Tyk Dashboard's `/api/apis/oas` endpoint resulting in the creation of the API in your Tyk Dashboard which will automatically deploy it to your Gateway. + +You can use the optional `templateId` parameter to apply an [API Template](/api-management/dashboard-configuration#applying-a-template-when-creating-an-api-from-a-tyk-oas-api-definition) to your API definition when creating the API. + + + + + +When making calls to the Tyk Gateway API you'll need to set the domain name and port for your environment and provide credentials in the `x-tyk-authorization` field for Tyk to authorize your request, as follows: + +| Interface | Port | Authorization Header | Authorization credentials | +| :----------------- | :------ | :----------------------- | :---------------------------------- | +| Tyk Gateway API | 8080 | `x-tyk-authorization` | `secret` value set in `tyk.conf` | + +To create the API in Tyk, you simply send your Tyk OAS API Definition in the payload to the `POST /tyk/apis/oas` endpoint of your Tyk Gateway API. + +Using [this](https://bit.ly/39tnXgO) minimal API definition it is possible to create a Tyk OAS API on your Tyk Gateway using only 30 lines: + +```curl +curl --location --request POST 'http://{your-tyk-host}:{port}/tyk/apis/oas' \ +--header 'x-tyk-authorization: {your-secret}' \ +--header 'Content-Type: text/plain' \ +--data-raw +'{ + "info": { + "title": "Petstore", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "components": {}, + "paths": {}, + "x-tyk-api-gateway": { + "info": { + "name": "Petstore", + "state": { + "active": true + } + }, + "upstream": { + "url": "https://petstore.swagger.io/v2" + }, + "server": { + "listenPath": { + "value": "/petstore/", + "strip": true + } + } + } +}' +``` + +**Check request response** + +If the command succeeds, you will see the following response, where `key` contains the unique identifier (`id`) for the API you have just created. If you did not provide a value in the `id` field, then Tyk will automatically assign one. + +```.json +{ + "key": {NEW-API-ID}, + "status": "ok", + "action": "added" +} +``` + +What you have done is to send a Tyk OAS API definition to Tyk Gateway's `/tyk/apis/oas` endpoint resulting in the creation of the API in your Tyk Gateway. + +**Restart or hot reload** + +Once you have created your API you need to load it into the Gateway so that it can serve traffic. To do this you can either restart the Tyk Gateway or issue a [hot reload](/tyk-stack/tyk-gateway/important-prerequisites#hot-reload-is-critical-in-tyk-ce) command: + +```.curl +curl -H "x-tyk-authorization: {your-secret}" -s http://{your-tyk-host}:{port}/tyk/reload/group +``` expandable + +You can go to the `/apps` folder of your Tyk Gateway installation (by default in `/var/tyk-gateway`) to see where Tyk has stored your Tyk OAS API Definition. + + + + + +### Importing an OpenAPI description to create an API + +Tyk will automatically update the `servers` section in the imported OpenAPI description, adding the base path URL to which requests should be sent to access the new API. It will take the existing entry and use this to generate the upstream (target) URL if none is provided. + + + + + +If you have a valid OAS 3.0 compliant OpenAPI description, in YAML or JSON format, you can use this to create an API in Tyk Dashboard with only a few clicks. + +1. Start by selecting **APIs** from the **API Management** section + + Add new API + +2. Now select **Add New API** and then, choose **Import**. + + Loading the API definition into Tyk Dashboard + +3. From the Import API screen, select **openAPI** because the object you want to import to Tyk is an OpenAPI description. + + Choosing what to import + + + + + On the Import API screen, there are three options for Import Type, it is important to select the correct one for the object that you want to load into Tyk: + + - openAPI is used only for [OpenAPI descriptions](/api-management/gateway-config-tyk-oas#openapi-description) (without the [Tyk Vendor Extension](/api-management/gateway-config-tyk-oas#tyk-vendor-extension)) + - TykAPI is used for a full [Tyk OAS API definition](/api-management/gateway-config-tyk-oas#what-is-a-tyk-oas-api-definition) (comprising OpenAPI description plus Tyk Vendor Extension) or Tyk Classic API definition + - WSDL/XML is used for WSDL/XML content and will result in a Tyk Classic API + + + +4. Now you can choose the location of the OpenAPI description, which can be: + + - pasted into the text editor + - uploaded using a file picker + - retrieved from a file server + + Loading the API definition into Tyk Dashboard + +5. You can optionally apply an [API template](/api-management/dashboard-configuration#governance-using-api-templates) from the drop-down. + + Applying a template + +6. You can configure the *listen path* and *upstream (target) URL* in the **Manual configuration options** section. Note that if you do not provide a listen path, Tyk will default to `/` and if you do not provide an upstream URL, Tyk will use the first value provided in the [servers.url](/api-management/gateway-config-managing-oas#api-base-path) section in the OpenAPI description. + + Configuring the listen path and upstream URL + +7. Tyk can automatically configure the request processing middleware chain based upon configuration defined by the OpenAPI Specification. If your OpenAPI desription contains the relevant data then select the characteristics you would like to configure. + + Configuring the listen path and upstream URL + + | Middleware | OpenAPI data used for configuration | + |------------|-------------------------------------| + | [Request validation](/api-management/traffic-transformation/request-validation#request-schema-in-openapi-specification) | Endpoints that have `requestBody` or `schema` | + | [Mock response](/api-management/traffic-transformation/mock-response#mock-responses-using-openapi-metadata) | Endpoints with `examples` or `schema` | + | [Client authentication](/api-management/client-authentication#how-does-tyk-implement-authentication-and-authorization) | Defined in `security` and `securitySchemes` | + | [Allow list](/api-management/traffic-transformation/allow-list) | Restrict access only to declared endpoint paths | + +8. Select **Import API** to complete the import and create the API based on your API definition. + + + + + +When making calls to the Tyk Dashboard API you'll need to set the domain name and port for your environment and provide credentials in the `Authorization` field for Tyk to authorize your request, as follows: + +| Interface | Port | Authorization Header | Authorization credentials | +| :------------------- | :------ | :---------------------- | :----------------------------- | +| Tyk Dashboard API | 3000 | `Authorization` | From Dashboard User Profile | + +You can obtain your authorization credential (Dashboard API key) from the Tyk Dashboard UI: + +- Select **Edit profile** from the dropdown that appears when you click on your username in the top right corner of the screen +- Scroll to the bottom of the page were you will see your **Tyk Dashboard API Access Credentials** + +You will also need to have β€˜admin’ or β€˜api:write’ permission if [RBAC](/api-management/user-management) is enabled. + +To create the API in Tyk, you simply send your OpenAPI document in the payload to the `POST /api/apis/oas/import` endpoint of your Tyk Dashboard API. + +| Property | Description | +| :-------------- | :------------------------------------------ | +| Resource URL | `/api/apis/oas/import` | +| Method | `POST` | +| Type | None | +| Body | OpenAPI Document | +| Parameters | Query: `listenPath` `upstreamURL` `authentication` `allowList` `validateRequest` `mockResponse` `apiID` `templateId` | + +The optional parameters are: + +| Parameter | Effect | Default if omitted | +| :------------------- | :--------------------------------- | :-------------------- | +| `listenPath` | Set the listen path for the API | Defaults to `/` | +| `upstreamURL` | Set the upstream (target) URL | Defaults to the first URL in the `servers` section of the [OpenAPI description](/api-management/gateway-config-managing-oas#api-base-path) | +| `authentication` | Configure [client authentication](/api-management/client-authentication#how-does-tyk-implement-authentication-and-authorization) based on `security` and `securitySchemes` | Client authentication is not configured | +| `allowList` | Enable [allow list](/api-management/traffic-transformation/allow-list) middleware for all endpoints declared in the OpenAPI description | Allow list not configured | +| `validateRequest` | Configure [request validation](/api-management/traffic-transformation/request-validation#request-schema-in-openapi-specification) for all endpoints with `requestBody` or `schema` defined | Request validation not configured | +| `mockResponse` | Configure [mock response](/api-management/traffic-transformation/mock-response#mock-responses-using-openapi-metadata) for all endpoints with `examples` or `schema` defined | Mock response not configured | +| `apiID` | Id to be assigned to the new API | Tyk will determine and assign a unique Id | +| `templateId` | Apply the selected [API template](/api-management/dashboard-configuration#applying-a-template-when-creating-an-api-from-a-tyk-oas-api-definition) when creating the API | No template is applied | + +**Check request response** + +If the command succeeds, you will see the following response, where `Meta` contains the unique identifier (`id`) for the API you have just created. + +``` +{ + "Status": "OK", + "Message": "API created", + "Meta": {NEW-API-ID} +} +``` expandable + + + + + +When making calls to the Tyk Gateway API you'll need to set the domain name and port for your environment and provide credentials in the `x-tyk-authorization` field for Tyk to authorize your request, as follows: + +| Interface | Port | Authorization Header | Authorization credentials | +| :----------------- | :------ | :----------------------- | :---------------------------------- | +| Tyk Gateway API | 8080 | `x-tyk-authorization` | `secret` value set in `tyk.conf` | + +To create the API in Tyk, you simply send your OpenAPI document in the payload to the `POST /tyk/apis/oas/import` endpoint of your Tyk Gateway API. + +| Property | Description | +| :-------------- | :------------------------------------------ | +| Resource URL | `/tyk/apis/oas/import` | +| Method | `POST` | +| Type | None | +| Body | OpenAPI Document | +| Parameters | Query: `listenPath` `upstreamURL` `authentication` `allowList` `validateRequest` `mockResponse` `apiID` | + +The optional parameters are: + +| Parameter | Effect | Default if omitted | +| :------------------- | :--------------------------------- | :-------------------- | +| `listenPath` | Set the listen path for the API | Defaults to `/` | +| `upstreamURL` | Set the upstream (target) URL | Defaults to the first URL in the `servers` section of the [OpenAPI description](/api-management/gateway-config-managing-oas#api-base-path) | +| `authentication` | Configure [client authentication](/api-management/client-authentication#how-does-tyk-implement-authentication-and-authorization) based on `security` and `securitySchemes` | Client authentication is not configured | +| `allowList` | Enable [allow list](/api-management/traffic-transformation/allow-list) middleware for all endpoints declared in the OpenAPI description | Allow list not configured | +| `validateRequest` | Configure [request validation](/api-management/traffic-transformation/request-validation#request-schema-in-openapi-specification) for all endpoints with `requestBody` or `schema` defined | Request validation not configured | +| `mockResponse` | Configure [mock response](/api-management/traffic-transformation/mock-response#mock-responses-using-openapi-metadata) for all endpoints with `examples` or `schema` defined | Mock response not configured | +| `apiID` | Id to be assigned to the new API | Tyk will determine and assign a unique Id | + +**Check request response** + +If the command succeeds, you will see the following response, where `key` contains the unique identifier (`id`) for the API you have just created. + +```.json +{ + "key": {NEW-API-ID}, + "status": "ok", + "action": "added" +} +``` + +**Restart or hot reload** + +Once you have created your API you need to load it into the Gateway so that it can serve traffic. To do this you can either restart the Tyk Gateway or issue a [hot reload](/tyk-stack/tyk-gateway/important-prerequisites#hot-reload-is-critical-in-tyk-ce) command: + +```.curl +curl -H "x-tyk-authorization: {your-secret}" -s http://{your-tyk-host}:{port}/tyk/reload/group +``` expandable + +You can go to the `/apps` folder of your Tyk Gateway installation (by default in `/var/tyk-gateway`) to see where Tyk has stored your Tyk OAS API Definition. + + + + + +#### API base path + +The [API base path](https://swagger.io/docs/specification/v3_0/api-host-and-base-path/) is the URL that a client should use when consuming (sending requests to) the API deployed on Tyk. This will comprise the address of the Tyk Gateway plus the API's listen path. + +**Detecting an Existing API Base Path** + +When creating an API, Tyk analyzes the `servers.url` section of the OpenAPI description to determine if it already contains a valid API base path. + +- If the first entry in `servers.url` is an address on the Tyk Gateway, then this is considered a valid API base path. +- If there is not a valid API base path, then Tyk will assume that the first value in `servers.url` is the address of the upstream service - and so will use this as the *upstream (target) URL* for the API proxy. If there are multiple entries in `servers.url` Tyk will only consider the first entry and ignore all others. + +Tyk supports [OpenAPI server variables](https://learn.openapis.org/specification/servers.html#server-variables), so if the first `servers` entry contains a parameterised URL, Tyk will fill in the parameters with the values provided in the `variables` associated with that entry. + +**Setting the API Base Path** + +If the `servers.url` section did not contain a valid *API base path* then Tyk will insert a new entry in the first location in `servers.url` with a valid API base path comprising the Tyk Gateway address plus the *listen path* for the API. + +For example, given the following fragment of the OpenAPI description and importing to a Tyk Gateway at `https://my-gateway.com` specifying a listen path of `my-api`: + +```yaml + servers: + - url: https://upstream-A.com + - url: http://upstream-B.com +``` + +Tyk will configure the Tyk OAS API with the following: + +```yaml + servers: + - url: https://my-gateway.com/my-api/ + - url: https://upstream-A.com + - url: http://upstream-B.com + + x-tyk-api-gateway: + server: + listenPath: + value: /my-api/ + upstream: + url: https://upstream-A.com +``` expandable + + + +This can introduce a change to the "source of truth" (OpenAPI description) for the API (the addition of the API base path). We recommend that you export the modified OpenAPI description and apply this to your documentation, as it provides the address to which clients should direct their traffic. + + + +**Upstream URL Override** + +The servers section is not analyzed if an upstream (target) URL is specified during the import action. If an upstream URL was specified, that will be used as the upstream for the API. The API base path will still be constructed and added to the `servers` section of the OpenAPI description. + +**Tyk does not support relative URLs** + +If the first entry is a relative URL, or another format that Tyk cannot process, the import will fail with an error. + +For example attempting to import an OpenAPI description containing this configuration: + +```yaml + servers: + - url: /relative-url + - url: http://upstream-B.com +``` +will error with the following message: + +```json +{ + "status": "error", + "message": "error validating servers entry in OAS: Please update \"/relative-url\" to be a valid url or pass a valid url with upstreamURL query param" +} +``` expandable + +#### Multi-part OpenAPI documents + +OAS 3.0 allows an OpenAPI description to be [split across multiple files](https://swagger.io/docs/specification/v3_0/using-ref/) by use of the `$ref` keyword. + +This allows you to share snippets of the API definition across multiple APIs, or to have specific ownership of elements of the API configuration owned by different teams. + +From Tyk 5.8.0 there is full support for these multi-part OpenAPI documents with Tyk Dashboard. + +We consider two different types of file containing these OpenAPI descriptions: + +- the **main fragment**, which contains the `info` section +- the **secondary fragments**, which contain snippets of the OpenAPI description that are referred to using external references (`$ref`) + +Note that secondary fragments can also contain external references to other secondary fragments (but not to the main fragment). + +When creating or updating an API, you simply provide Tyk with the **main fragment** and ensure that all of the references can be resolved. + +Resolution can be: +- local, by providing a ZIP archive containing all fragments +- remote, by providing resolvable paths to the secondary fragments (this is particularly used if the main fragment is provided via URL, as all fragments can then exist on the same file server). + + +## Maintaining your APIs + +Once a Tyk OAS API has been created in Tyk the Gateway will manage traffic to the exposed endpoints and proxy requests to the upstream service. + +Your service might evolve over time, with new features and endpoints being added and others retired. Tyk's flexible API versioning and update options provide you with choice for how to reflect this evolution in the APIs you expose to your clients. + +Your OpenAPI description is a living document that describes your upstream service. When this changes (for example, due to the addition of a new endpoint) you can use Tyk's [update API](/api-management/gateway-config-managing-oas#updating-an-api) feature to seamlessly apply the updated OpenAPI description, instantly extending the API proxy to handle traffic as your upstream evolves. + +Alternatively, and especially when you need to make breaking changes as your services and APIs evolve, you can create new [versions](/api-management/api-versioning) of your API and use configurable version identifiers to route traffic to the appropriate target. + +### Updating an API + +As developers working on services it can be necessary to regularly update the API when, for example, we add endpoints or support new methods. + +One of the most powerful features of working with Tyk OAS is that you can make changes to the OpenAPI description outside Tyk and then update your API with the updated details. You can simply update the OpenAPI part of the Tyk OAS API definition without having to make any changes to the [Tyk Vendor Extension](/api-management/gateway-config-tyk-oas#tyk-vendor-extension) (`x-tyk-api-gateway`). + +You can alternatively work on the full Tyk OAS API definition outside Tyk and update your existing API proxy with the new configuration, without having to create a [new version](/api-management/api-versioning) of the API. + + + + + +If you have an updated OpenAPI description or Tyk OAS API definition, in YAML or JSON format, you can use this to modify your existing API in Tyk Dashboard with only a few clicks. + +1. Start by selecting your API from the list on the **APIs** page in the **API Management** section. + +2. Now select **Update OAS** from the **Actions** dropdown. + + Select Update OAS to import the new OpenAPI description + +3. Now you can choose the location of the file that you want to use to update your API, which can be: + + - pasted into the text editor + - uploaded using a file picker + - retrieved from a file server + + Configuring the import location and options + +4. You can re-configure the *listen path* and *upstream (target) URL* in the **Manual configuration options** section, but if you do not provide these then Tyk will leave them unchanged. + +5. Tyk must select the options to automatically configure the request processing middleware chain based upon configuration defined by the OpenAPI Specification for any new endpoints added in the update. If your OpenAPI desription contains the relevant data then select the characteristics you would like to configure. + + Configuring the listen path and upstream URL + + | Middleware | OpenAPI data used for configuration | + |------------|-------------------------------------| + | [Request validation](/api-management/traffic-transformation/request-validation#request-schema-in-openapi-specification) | Endpoints that have `requestBody` or `schema` | + | [Mock response](/api-management/traffic-transformation/mock-response#mock-responses-using-openapi-metadata) | Endpoints with `examples` or `schema` | + | [Client authentication](/api-management/client-authentication#how-does-tyk-implement-authentication-and-authorization) | Defined in `security` and `securitySchemes` | + | [Allow list](/api-management/traffic-transformation/allow-list) | Restrict access only to declared endpoint paths | + +8. Select **Import API** to complete the update. + + + + + +When making calls to the Tyk Dashboard API you'll need to set the domain name and port for your environment and provide credentials in the `Authorization` field for Tyk to authorize your request, as follows: + +| Interface | Port | Authorization Header | Authorization credentials | +| :------------------- | :------ | :---------------------- | :----------------------------- | +| Tyk Dashboard API | 3000 | `Authorization` | From Dashboard User Profile | + +You can obtain your authorization credential (Dashboard API key) from the Tyk Dashboard UI: + +- Select **Edit profile** from the dropdown that appears when you click on your username in the top right corner of the screen +- Scroll to the bottom of the page were you will see your **Tyk Dashboard API Access Credentials** + +You will also need to have β€˜admin’ or β€˜api:write’ permission if [RBAC](/api-management/user-management) is enabled. + +**Applying an Updated OpenAPI Description** + +To update just the OpenAPI description of your API in Tyk, you simply send the OpenAPI document in the payload to the `PATCH /api/apis/oas/{API-ID}` endpoint of your Tyk Gateway API. + +| Property | Description | +| :-------------- | :------------------------------------------ | +| Resource URL | `/api/apis/oas/{API-ID}` | +| Method | `PATCH` | +| Type | None | +| Body | OpenAPI document | +| Parameters | Path: `{API-ID}` | + +You need to specify which API to update - and do so using the `API-ID` value from the response you received from Tyk when creating the API. You can find this in the `x-tyk-api-gateway.info.id` field of the Tyk OAS API Definition stored in your main storage. + +**Applying an Updated Tyk OAS API Definition** + +To update the whole API in Tyk, you simply send the Tyk OAS API definition in the payload to the `PATCH /api/apis/oas/{API-ID}` endpoint of your Tyk Gateway API. + +| Property | Description | +| :-------------- | :------------------------------------------ | +| Resource URL | `/api/apis/oas/{API-ID}` | +| Method | `PATCH` | +| Type | None | +| Body | Tyk OAS API Definition | +| Parameters | Path: `{API-ID}` | + +You need to specify which API to update - and do so using the `API-ID` value from the response you received from Tyk when creating the API. You can find this in the `x-tyk-api-gateway.info.id` field of the Tyk OAS API Definition stored in your main storage. + +**Check request response** + +If the command succeeds, you will see the following response, where `Meta` contains the unique identifier (`id`) for the API you have just updated: + +```.json +{ + "Status": "OK", + "Message": "API modified", + "Meta": {API-ID} +} +``` expandable + + + + + +When making calls to the Tyk Gateway API you'll need to set the domain name and port for your environment and provide credentials in the `x-tyk-authorization` field for Tyk to authorize your request, as follows: + +| Interface | Port | Authorization Header | Authorization credentials | +| :----------------- | :------ | :----------------------- | :---------------------------------- | +| Tyk Gateway API | 8080 | `x-tyk-authorization` | `secret` value set in `tyk.conf` | + + +**Applying an Updated OpenAPI Description** + +To update just the OpenAPI description of your API in Tyk, you simply send the OpenAPI document in the payload to the `PATCH /tyk/apis/oas/{API-ID}` endpoint of your Tyk Gateway API. + +| Property | Description | +| :-------------- | :------------------------------------------ | +| Resource URL | `/tyk/apis/oas/{API-ID}` | +| Method | `PATCH` | +| Type | None | +| Body | OpenAPI document | +| Parameters | Path: `{API-ID}` Query: `templateId` | + +You need to specify which API to update - and do so using the `API-ID` value from the response you received from Tyk when creating the API. You can find this in the `x-tyk-api-gateway.info.id` field of the Tyk OAS API Definition that Tyk has stored in the `/apps` folder of your Tyk Gateway installation. + + +**Applying an Updated Tyk OAS API Definition** + +To update the whole API in Tyk, you simply send the Tyk OAS API definition in the payload to the `PATCH /tyk/apis/oas/{API-ID}` endpoint of your Tyk Gateway API. + +| Property | Description | +| :-------------- | :------------------------------------------ | +| Resource URL | `/tyk/apis/oas/{API-ID}` | +| Method | `PATCH` | +| Type | None | +| Body | Tyk OAS API Definition | +| Parameters | Path: `{API-ID}` | + +You need to specify which API to update - and do so using the `API-ID` value from the response you received from Tyk when creating the API. You can find this in the `x-tyk-api-gateway.info.id` field of the Tyk OAS API Definition that Tyk has stored in the `/apps` folder of your Tyk Gateway installation. + + +**Check request response** + +If the command succeeds, you will see the following response, where `key` contains the unique identifier (`id`) for the API you have just updated: + +```.json +{ + "key": {API-ID}, + "status": "ok", + "action": "modified" +} +``` + +**Restart or hot reload** + +Once you have updated your API you need to load it into the Gateway so that it can serve traffic. To do this you can either restart the Tyk Gateway or issue a [hot reload](/tyk-stack/tyk-gateway/important-prerequisites#hot-reload-is-critical-in-tyk-ce) command: + +```.curl +curl -H "x-tyk-authorization: {your-secret}" -s http://{your-tyk-host}:{port}/tyk/reload/group +``` + + + + + + +### Exporting an API asset + +Each API on Tyk has an API definition comprising the OpenAPI description and the Tyk Vendor Extension. We offer the facility for you to export (download) two assets for an API - just the OpenAPI description, or the full Tyk OAS API definition. + +From Tyk 5.8.0, when using Tyk Dashboard these can be exported in either JSON or YAML format; for Tyk Gateway API users the assets can only be exported in JSON format. + + + + + +1. Start by selecting your API from the list on the **APIs** page in the **API Management** section. + +2. Select **Export API** from the **Actions** dropdown. + + Select Export API to download an asset from Tyk + +3. Now you can choose what you want to export, the filename (a default is offered which is based on the API Id) and the file format (JSON or YAML). + + Choosing what to download + +4. Finally select **Export** to save the file to your local machine. + + + + + +When making calls to the Tyk Dashboard API you'll need to set the domain name and port for your environment and provide credentials in the `Authorization` field for Tyk to authorize your request, as follows: + +| Interface | Port | Authorization Header | Authorization credentials | +| :------------------- | :------ | :---------------------- | :----------------------------- | +| Tyk Dashboard API | 3000 | `Authorization` | From Dashboard User Profile | + +You can obtain your authorization credential (Dashboard API key) from the Tyk Dashboard UI: + +- Select **Edit profile** from the dropdown that appears when you click on your username in the top right corner of the screen +- Scroll to the bottom of the page were you will see your **Tyk Dashboard API Access Credentials** + +You will also need to have β€˜admin’ or β€˜api:write’ permission if [RBAC](/api-management/user-management) is enabled. + +To export an API asset, you use the `GET /api/apis/oas/{API-ID}/export` endpoint, indicating whether you require the full Tyk OAS API definition or only the OpenAPI description using the `mode` parameter. + +| Property | Description | +| :-------------- | :-------------------------------------------------- | +| Resource URL | `/api/apis/oas/{API-ID}` | +| Method | `GET` | +| Type | None | +| Parameters | Path: `{API-ID}` Query: `mode` `Content-Type` | + +Where: +- `API-ID` is the unique `id` assigned in the Tyk Vendor Extension that identifies the API +- `mode` to identify the asset to export: `public` for the OpenAPI description (default empty for full API definition) +- `Content-Type` to select the format for the exported asset: `application/x-yaml` or `application/json` + + + + +When making calls to the Tyk Gateway API you'll need to set the domain name and port for your environment and provide credentials in the `x-tyk-authorization` field for Tyk to authorize your request, as follows: + +| Interface | Port | Authorization Header | Authorization credentials | +| :----------------- | :------ | :----------------------- | :---------------------------------- | +| Tyk Gateway API | 8080 | `x-tyk-authorization` | `secret` value set in `tyk.conf` | + +To export an API asset, you use the `GET /tyk/apis/oas/{API-ID}/export` endpoint, indicating whether you require the full Tyk OAS API definition or only the OpenAPI description using the `mode` parameter. + +| Property | Description | +| :-------------- | :-------------------------------------------- | +| Resource URL | `/tyk/apis/oas/{API-ID}` | +| Method | `GET` | +| Type | None | +| Parameters | Path: `{API-ID}` Query: `mode` | + +Where: +- `API-ID` is the unique `id` assigned in the Tyk Vendor Extension that identifies the API +- `mode=public` to export the OpenAPI description (otherwise, export the full API definition) + + + diff --git a/api-management/gateway-config-tyk-classic.mdx b/api-management/gateway-config-tyk-classic.mdx new file mode 100644 index 00000000..39920f5d --- /dev/null +++ b/api-management/gateway-config-tyk-classic.mdx @@ -0,0 +1,661 @@ +--- +title: "Tyk Classic API Definition" +'og:description': "How to configure Tyk Classic API Definition" +tags: ['Gateway', 'Configuration', 'Tyk Classic', 'Tyk Classic API Definition', 'Tyk Classic API Definition Object'] +sidebarTitle: "Tyk Classic" +--- + +import ApiDefGraphql from '/snippets/api-def-graphql.mdx'; + +## Introduction to Tyk Classic + +Tyk's legacy API definition is now called Tyk Classic and is used for GraphQL, XML/SOAP and TCP services. + +From Tyk 5.8 we recommend that any REST APIs are migrated to the newer [Tyk OAS API](/api-management/gateway-config-tyk-oas) style, in order that they can benefit from simpler configuration and future enhancements. + +For Tyk Dashboard users with an existing portfolio of Tyk Classic API definitions, we provide a [migration tool](/api-management/migrate-from-tyk-classic), available via the Dashboard API and UI. + + + +For versions of Tyk prior to 5.8 not all Gateway features can be configured using the Tyk OAS API definition, for edge cases you might need to use Tyk Classic for REST APIs, though we recommend updating to Tyk 5.8 and adopting Tyk OAS. + + + +The Tyk Classic API definition has a flat structure that does not use the `omitempty` style, requiring all fields to be present even if set to null, resulting in a larger object than that for an equivalent Tyk OAS API definition. + +Note that there are some specific differences between Tyk Classic and Tyk OAS APIs, in particular with respect to [default authentication method](#configuring-authentication-for-tyk-classic-apis) and [API versioning](#tyk-classic-api-versioning). + +## Tyk Classic API versioning + +When multiple versions of a Tyk Classic API are created, the details are stored in a single API definition - unlike with Tyk OAS where a separate API definition is created for each version. The common configuration is stored in the root, whereas the details of the different versions are stored in a dedicated `version_data` object, within the API definition. + +Whilst this allows for easy management of all the API versions, it limits the number of features that can be configured differently between versions, as not all Gateway configuration options are duplicated in `version_data`. + +Tyk enforces strict access control to specific versions of APIs if these are specified in the access token (key). If, once Tyk has identified the API to load, and has allowed the access key through, it will check the access token's session data for access permissions. If it finds none, it will let the token through. However, if there are permissions and versions defined, it will be strict in **only** allowing access to that version. + +Key things to note when configuring versioning for a Tyk Classic API: + +- you must set `version_data.not_versioned` to `false` for Tyk to treat the API as versioned +- `version_data.default_version` must contain the `name` of the version that shall be treated as default (for access control and default fallback) +- you can use `version_data.paths` to configure endpoint-level ignore, allow and block lists (which can be used to configure a mock response) +- you must use `version_data.extended_paths` to configure other endpoint-level middleware +- common versioning configuration is mostly contained within the [definition](/api-management/gateway-config-tyk-classic#common-versioning-configuration) object +- configuration for the different versions is contained within the [version_data](/api-management/gateway-config-tyk-classic#version-specific-configuration) object + - this also contains some common configuration (`not_versioned` and `default_version`) + +When you first create an API, it will not be "versioned" (i.e. `not_versioned` will be set to `true`) and there will be a single version with the name `Default` created in the `version_data` section. + +### Common versioning configuration + +**Field: `definition`** +This object in the root of the Tyk Classic API definition handles information related to how Tyk should handle requests to the versioned API + +**Field: `definition.location`** +Used to configure where the versioning identifier should be provided, one of:`header`, `url`, `url-param`. + +**Field: `definition.key`** +The name of the key that contains the versioning identifier if `definition.location` is set to `header` or `url-param`. + +**Field: `definition.strip_versioning_data`** +Set this to `true` to remove the versioning identifier when creating the upstream (target) URL. + +**Field: `definition.fallback_to_default`** +Set this to `true` to invoke the default version if an invalid version is specified in the request. + +**Field: `definition.url_versioning_pattern`** +Available from Tyk 5.5.0, if you are have set both `definition.strip_versioning_data` and `definition.fallback_to_default` to `true` and are using `definition.location=url` you can configure this with a regex that matches the format that you use for the versioning identifier (`versions.{version-name}.name`) + +The following fields are either deprecated or otherwise not used for Tyk Classic API versioning and should be left with their default values: + +- `definition.default`: defaults to an empty string `""` +- `definition.enabled`: defaults to `false` +- `definition.name`: defaults to an empty string `""` +- `definition.strip_path`: deprecated field; defaults to `false` +- `definition.versions`: defaults to an empty array `{}` + +### Version specific configuration + +**Field: `version_data`** +This object contains the version status and configuration for your API + +**Field: `version_data.not_versioned`** +Set this to `false` to treat this as a versioned API. If you are not using versioning for this API you must have a single `Default` entry in the `version_data.versions` map. + +**Field: `version_data.default_version`** +Used to configure where the versioning identifier should be provided, one of:`header`, `url`, `url-param`. + +**Field: `version_data.versions`** +A list of objects that describe the versions of the API; there must be at least one (`Default`) version defined for any API (even non-versioned APIs). Each version of your API should be defined here with a unique `name`. + +**Field: `version_data.versions.{version-name}.name`** +An identifier for this version of the API, for example `Default` or `v1`. The value given here is what will Tyk will match against the value in the `definition.key`. + +**Field: `version_data.versions.{version-name}.expires`** +If a value is set then Tyk will automatically deprecate access to the API after the specified timestamp. The entry here takes the form of: `"YYYY-MM-DD HH:MM"`. If this is not set the version will never expire. + +**Field: `version_data.versions.{version-name}.paths`** +This object enables configuration of the basic allow list, block list and ignore authentication middleware for specific endpoints in the API version. You can also configure these and many other per-endpoint middleware using the `extended_paths` field. + +**Field: `version_data.versions.{version-name}.override_target`** +You can configure a different target URL here which will be used instead of the value stored in `proxy.target_url`, redirecting requests to a different hostname or domain. Note that this will also override (and so is not compatible with) upstream load balancing and Service Discovery, if configured for this API. + +**Field: `version_data.versions.{version-name}.global_headers`** +A `key:value` map of HTML headers to inject to the request. + +**Field: `version_data.versions.{version-name}.global_headers_remove`** +A list of HTML headers to remove from the request. + +**Field: `version_data.versions.{version-name}.global_size_limit`** +Apply a maximum size to the request body (payload) - in bytes. + +**Field: `version_data.versions.{version-name}.ignore_endpoint_case`** +If this boolean flag is set to `false`, Tyk will apply case sensitive matching of requests to endpoints defined in the API definition. + +**Field: `version_data.versions.{version-name}.use_extended_paths`** +Set this value to `true` if you want Tyk to apply specific middleware to endpoints in this version, configured using `version_data.versions.{version-name}.extended_paths`. + +**Field: `version_data.versions.{version-name}.extended_paths`** +This field contains a list of middleware configurations and to which paths they should be applied. The available middleware are: + +``` expandable +{ + black_list[], + white_list[], + ignore[], + track_endpoints[], + do_not_track_endpoints[], + internal[], + method_transforms[], + transform[], + transform_headers[], + transform_response[], + transform_response_headers[], + size_limits[], + validate_json[], + url_rewrites[], + virtual[], + transform_jq[], + cache[], + hard_timeouts[], + circuit_breakers[] +} +``` + +Each entry must include the `method` and `path` (identifying the endpoint) for which the middleware should be run. The other options for each middleware are documented in the [Traffic Transformation](#) section. Note that mock response functionality is provided via the `black_list[]`, `white_list[]` and `ignore[]` middleware. + + + + + +## Configuring authentication for Tyk Classic APIs + +Tyk Classic APIs *default to the auth token method* for authenticating requests. Flags in the API definition can be configured to enforce an alternative method: + +- keyless (no authentication of the client) +- basic authentication +- HMAC request signing +- Tyk as the OAuth 2.0 authorization server +- JWT authentication + +**Field: `use_keyless`** +This will switch off all key checking and open the API definition up, some analytics will still be recorded, but rate-limiting, quotas and security policies will not be possible (there is no session to attach requests to). This is a good setting for checking if Tyk works and is proxying traffic correctly. + +**Field: `auth`** +This object contains the basic configuration for the Auth (Bearer) Token method. + +**Field: `auth.auth_header_name`** +The header name (key) where Tyk should look for the token. + +**Field: `auth.use_param`** +Set this to true to instruct Tyk to expect the token in the URL parameter with key `auth.param_name`. + +**Field: `auth.param_name`** +The name of the URL parameter key containing the auth token. Note that this is case sensitive. + +**Field: `auth.use_cookie`** +Set this to true to instruct Tyk to expect the token in the URL parameter with key `auth.cookie_name`. + +**Field: `auth.cookie_name`** +The name of the cookie containing the auth token. Note that this is case sensitive. + +**Field: `auth.use_certificate`** + + +**Field: `auth.validate_signature`** +Boolean value set to `true` to enable Auth Token Signature Validation + +**Field: `auth.signature`** +Configuration for Auth Token Signature Validation + +**Field: `auth.signature.algorithm`** +The algorithm you wish to validate the signature against. Options are: +- `MasherySHA256` +- `MasheryMD5` + +**Field: `auth.signature.header`** +Header key for attempted signature + +**Field: `auth.signature.secret`** +The shared secret which was used to sign the request +- this can hold a dynamic value, by referencing `$tyk_meta` or `$tyk_context` variables. +- for example: if you have stored the shared secret in the field `individual_secret` of the session token's meta-data you would use the value `"secret": "$tyk_meta.individual_secret"`. + +**Field: `auth.signature.allowed_clock_skew`** +Maximum permitted deviation in seconds between UNIX timestamp of Tyk & UNIX timestamp used to generate the signed request + +**Field: `use_basic_auth`** +This method will enable basic auth as specified by the HTTP spec, an API with this flag set will request for a username and password and require a standard base64 Authentication header to be let through. + +**Field: `basic_auth.disable_caching`** +This disables the caching of basic authentication keys. + +**Field: `basic_auth.cache_ttl`** +This is the refresh period for the basic authentication key cache (in seconds). + +**Field: `enable_signature_checking`** +If this option is set to `true`, Tyk will implement the HMAC signing standard as proposed in the [HTTP Signatures Spec](https://web-payments.org/specs/ED/http-signatures/2014-02-01/#page-3). In particular the structure of the Authorization header and the encoding method need to be taken into account. +- this method will use a session key to identify a user and a user secret that should be used by the client to sign each request's `date` header +- it will also introduce clock skew checks, requests outside of 300ms of the system time will be rejected +- it is not recommended for Single-Page-Webapps (SPA) or Mobile apps due to the fact that secrets need to be distributed + +**Field: `hmac_allowed_algorithms`** +Tyk supports the following HMAC algorithms: β€œhmac-sha1", "hmac-sha256", "hmac-sha384", "hmac-sha512”. You can limit which ones you want to support with this option. For example, [β€œhmac-sha256”] + +**Field: `hmac_allowed_clock_skew`** +Set this value to anything larger than `0` to set the number of milliseconds that will be tolerated for clock skew. Set to `0` to prevent clock skew checks on requests (only in HMAC mode, i.e. when `enable_signature_checking` is set to `true`). + +**Field: `use_oauth2`** +This authentication method will use Tyk as the OAuth 2.0 Authorization Server. Enabling this option will cause Tyk to add OAuth2-standard endpoints to the API for `/authorize` and `/token`, these will supersede any other requests to your proxied system in order to enable the flow. + +**Field: `oauth_meta.allowed_access_types`** +This is a string array of OAuth access options depending on the OAuth grant types to be supported. Valid options are: +- `authorization_code` - client has an authorization code to request a new access token. +- `refresh_token` - client can use a refresh token to refresh expired bearer access token. + +**Field: `oauth_meta.allowed_authorize_types`** +This is a string array of OAuth authorization types. Valid options are: +- `code` - Client can request an authorization code which can be used to request an access code via a server request (traditionally reserved for server-side apps). +- `token` - Client can request an access token directly, this will not enable refresh tokens and all tokens have a 12 hour validity. Recommended for mobile apps and single-page webapps. + +**Field: `oauth_meta.auth_login_redirect`** +The Tyk OAuth flow has a dummy (intercept) `/authorize` endpoint which basically redirects the user to your login and authentication page, it will also send along all OAuth data as part of the request (so as to mimic a regular app flow). This is the URL that the user will be sent to (via `POST`). + +**Field: `notifications`** +When Tyk is used as the OAuth 2.0 Authorization Server, because it will handle access requests on your behalf once authorization codes have been issued, it will need to notify your system that these have occurred. It will `POST` key data to the URL set in these options to ensure that your system is synchronised with Tyk. + +**Field: `notifications.shared_secret`** +Posted data to your service will use this shared secret as an authorization header. This is to ensure that messages being received are from Tyk and not from another system. + +**Field: `notifications.oauth_on_keychange_url`** +The URL that will be sent the updated information - the URL will be polled up to 3 times if there is a communications failure. On a `200 OK` response it stops. + +**Field: `auth_configs`** +This section allows definition of multiple chained authentication mechanisms that will be applied to requests to the API, with distinct authentication headers identified for the different auth modes. + +For example: + +```json expandable +{ + "auth_configs": { + "authToken": { "auth_header_name": "My-Auth-Header-Key" }, + "basic": { "auth_header_name": "My-Basic-Auth-Header-Key" } + } +} +``` + +**Field: `base_identity_provided_by`** +This enables multiple authentication and indicates which authentication method provides the session object that determines access control, rate limits and usage quotas. + +It should be set to one of the following: + +- `auth_token` +- `hmac_key` +- `basic_auth_user` +- `jwt_claim` +- `oidc_user` +- `oauth_key` +- `custom_auth` + +**Field: `enable_jwt`** +Set JWT as the authentication method for this API. + +**Field: `jwt_signing_method`** +Either HMAC or RSA - HMAC requires a shared secret while RSA requires a public key to use to verify against. Please see the section on JSON web tokens for more details on how to generate these. + +**Field: `jwt_source`** +Must either be a base64 encoded valid RSA/HMAC key or a url to a resource serving JWK, this key will then be used to validate inbound JWT and throttle them according to the centralised JWT options and fields set in the configuration. See [Dynamic public key rotation using public JWKs URL](/basic-config-and-security/security/authentication-authorization/json-web-tokens#enable-dynamic-public-key-rotation-using-jwks) for more details on JWKs. + +**Field: `jwt_identity_base_field`** +Identifies the user or identity to be used in the Claims of the JWT. This will fallback to `sub` if not found. This field forms the basis of a new "virtual" token that gets used after validation. It means policy attributes are carried forward through Tyk for attribution purposes. + +Centralised JWTs add a `TykJWTSessionID` to the session metadata on create to enable upstream hosts to work with the internalised token should things need changing. + +**Field: `jwt_policy_field_name`** +The policy ID to apply to the virtual token generated for a JWT. + +**Field: `jwt_issued_at_validation_skew`** +Prevent token rejection due to clock skew between servers for Issued At claim (seconds, default: 0) + +**Field: `jwt_expires_at_validation_skew`** +Prevent token rejection due to clock skew between servers for Expires At claim (seconds, default: 0) + +**Field: `jwt_not_before_validation_skew`** +Prevent token rejection due to clock skew between servers for Not Before claim (seconds, default: 0) + +## GraphQL specific fields + + + +## General features + +### API identification + +**Field: `api_id`** +The identifier for the API This should be unique, but can actually be any kind of string. For single-instance setups this can probably be set to `1`. It is recommended to make this a UUID. The `api_id` is used to identify the API in queries to the Tyk Gateway API or Tyk Dashboard API. + +**Field: `name`** +Human readable name of the API. It is used for identification purposes but does not act as an index. + +**Field: `org_id`** +This is an identifier that can be set to indicate ownership of an API key or of an individual API. If the Org ID is set (recommended), it is prepended to any keys generated by Tyk - this enables lookups by prefixes from Redis of keys that are in the system. + +**Field: `domain`** +The domain to bind this API to. Multiple APIs can share the same domain, so long as their listen paths are unique. +This domain will affect your API only. To set up the portal domain for your organization, please register it in the main Tyk Dashboard settings file. +Your Tyk Gateway can listen on multiple domains/subdomains through the use of regular expressions, more precisely the RE2 Syntax. They are defined using the format `{name}` or `{name:pattern}`. + * `www.example.com` Matches only if domain is www.example.com + * `{subdomain:[a-z]+}.example.com` Matches dynamic subdomain + * `{subdomain:foo|bar}.example.com` will listen on foo.example.com and bar.example.com" + +**Field: `ignore_endpoint_case`** +If set to `true` when matching the URL path for requests to this API, the case of the endpoint path will be ignored. So for an API `my-api` and the endpoint `getuser`, requests to all of the following will be matched: + + * `/my-api/getuser` + * `/my-api/getUser` + * `/my-api/GetUser` + +If set to true, this will override the endpoint level settings in [Ignore](/api-management/traffic-transformation/ignore-authentication#case-sensitivity), [Allowlist](/api-management/traffic-transformation/allow-list#case-sensitivity) and [Blocklist](/api-management/traffic-transformation/block-list#case-sensitivity) middleware. This setting can be overriden at the Tyk Gateway level, and so applied to all APIs, by setting `ignore_endpoint_case` to `true` in your `tyk.conf` file. See [ignore_endpoint_case](/tyk-oss-gateway/configuration#ignore_endpoint_case) for details. + +**Field: `enable_batch_request_support`** +Set to true to enable batch support + +**Field: `id`** +This is allocated by Tyk to locate the API definition in the Dashboard main storage and bears no actual relation to the identity of the API. + +**Field: `active`** +This field is used by Tyk Dashboard to control whether the API will serve traffic. If set to `false` then on Gateway start, restart or reload, the API will be ignored and all paths and routes for that API will cease to be proxied. Any keys assigned to it will still exist, though they will not be let through for that particular API. + +### Access token management + +**Field: `session_lifetime`** +The session (API access key/token) lifetime will override the expiry date if it has been set on a key (in seconds). for example, if a key has been created that never expires, then it will remain in the session cache forever unless manually deleted. If a re-auth needs to be forced or a default expiry needs to be applied to all keys, then use this feature to set the session expiry for an entire API. + +**Field: `session_lifetime_respects_key_expiration`** +If this is set to `true` and the key expiration date is less than the `session_lifetime`, the key expiration value will be set to `session_lifetime`. Don't forget that the key expiration is set in unix timestamp but `session_lifetime` is set in seconds. Also, `session_lifetime_respects_key_expiration` exists in the global config too. When the global one is set to `true`, the one set at the API level will be ignored. + +**Field: `dont_set_quota_on_create`** +If set to true, when the keys are created, edited or added for this API, the quota cache in Redis will not be reset. + +### Traffic logs + +**Field: `enable_detailed_recording`** +If this value is set to `true`, the Gateway will record the request and response payloads in traffic logs. + +**Field: `do_not_track`** +If this value is set to `true`, the Gateway will not generate traffic logs for requests to the API. + +**Field: `tag_headers`** +This specifies a string array of HTTP headers values which turned into tags. For example, if you include the `X-Request-ID` header to `tag_headers`, for each incoming request it will include an `x-request-id-` tag to request an analytic record. This functionality can be useful if you need analytics for request headers without the body content (Enabling detailed logging is another option, but it records the full request and response objects and consumes a lot more space). + +**Field: `expire_analytics_after`** +This value (in seconds) will be used to indicate a TTL (ExpireAt) for the retention of analytics created from traffic logs generated for this API that are stored in MongoDB. If using an alternative analytics storage solution that does not respect ExpireAt then you must manage the record TTL separately. + +### OpenTelemetry + +**Field: `detailed_tracing`** +If this value is set to `true`, the Gateway will generate detailed OpenTelemetry spans for requests to the API. + +### API Level Rate Limits + +**Field: `global_rate_limit`** +The [API-level rate limit](/api-management/rate-limit#rate-limiting-layers) aggregates the traffic coming into an API from all sources and ensures that the overall rate limit is not exceeded. It is composed of a `rate` (number of requests) and `per` (interval). If either is set to `0` then no API-level limit is applied. + +**Field: `disable_rate_limit`** +If set to `true`, all rate limits are disabled for the specified API (both API-level and key-level) + +### Event handlers + +**Field: `event_handlers`** +This adds the ability to configure an API with event handlers to perform specific actions when an event occurs. + +**Field: `events`** +Each event handler that is added to the event_handlers.events section, is mapped by the event type, and then a list of each handler configuration, defined by the handler name and the handler metadata (usually some kind of configurable options for the specific handler) + +### Custom data + +**Field: `enable_context_vars`** +Context variables are extracted from the request at the start of the middleware chain, and must be explicitly enabled in order for them to be made available to your transforms. These values can be very useful for later transformation of request data, for example, in converting a Form-based POST into a JSON-based PUT or to capture an IP address as a header. + +**Field: `config_data`** +You can use this field to pass custom attributes to the virtual endpoint middleware. It is a list of key:value pairs. + +### IP Access Control + +**Field: `enable_ip_whitelisting`** +This works with the associated `allowed_ips` list and, when set to `true`, accepts only requests coming from the defined list of allowed IP addresses. + +**Field: `allowed_ips`** +A list of strings that defines the IP addresses (in [CIDR notation](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation)) that are allowed access via Tyk. This list is explicit and wildcards are not supported. + +**Field: `enable_ip_blacklisting`** +This works with the associated `blacklisted_ips` list and, when set to `true`, rejects and requests coming from the defined list of blocked IP addresses. + +**Field: `blacklisted_ips`** +A list of strings that defines the IP addresses (in [CIDR notation](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation)) that are blocked access via Tyk. This list is explicit and wildcards are not supported. + +### Cross-Origin Resource Sharing (CORS) + +**Field: `CORS.enable`** +Enable CORS for the API + +**Field: `CORS.allowed_origin`** +A list of origin domains to allow access from. Wildcards are also supported, e.g. http://*.foo.com + +**Field: `CORS.allowed_methods`** +A list of HTTP methods to allow access via. + +**Field: `CORS.allowed_headers`** +Headers that are allowed within a request. + +**Field: `CORS.exposed_headers`** +Headers that are exposed back in the response. + +**Field: `CORS.allow_credentials`** +Whether credentials (cookies) should be allowed. + +**Field: `CORS.max_age`** +Maximum age of credentials. + +**Field: `CORS.options_passthrough`** +Allow CORS OPTIONS preflight request to be proxied directly to upstream, without authentication and the rest of the checks. This means that pre-flight requests generated by web-clients such as SwaggerUI will be able to test the API using trial keys. If your service handles CORS natively, then enable this option. + +### Proxy Transport Settings + +**Field: `proxy.preserve_host_header`** +Set to `true` to preserve the host header. If `proxy.preserve_host_header` is set to `true` in an API definition then the host header in the outbound request is retained to be the inbound hostname of the proxy. + +**Field: `proxy.listen_path`** +The path to listen on, e.g. `/api` or `/`. Any requests coming into the host, on the port that Tyk is configured to run on, that go to this path will have the rules defined in the API Definition applied. Versioning assumes that different versions of an API will live on the same URL structure. If you are using URL-based versioning (e.g. `/v1/function`, `/v2/function/`) then it is recommended to set up a separate non-versioned definition for each version as they are essentially separate APIs. + +Proxied requests are literal, no re-writing takes place, for example, if a request is sent to the listen path of: `/listen-path/widgets/new` and the URL to proxy to is `http://your.api.com/api/` then the *actual* request that will land at your service will be: `http://your.api.com/api/listen-path/widgets/new`. + +This behavior can be circumvented so that the `listen_path` is stripped from the outgoing request. See the section on `strip_listen_path` below. + +**Field: `proxy.strip_listen_path`** +By setting this to `true`, Tyk will attempt to replace the `listen-path` in the outgoing request with an empty string. This means that in the above scenario where `/listen-path/widgets/new` and the URL to proxy to is `http://your.api.com/api/` becomes `http://your.api.com/api/listen-path/widgets/new`, actually changes the outgoing request to be: `http://your.api.com/api/widgets/new`. + +**Field: `proxy.target_url`** +This defines the target URL that the request should be proxied to if it passes all checks in Tyk. + +**Field: `proxy.disable_strip_slash`** +This boolean option allows you to add a way to disable the stripping of the slash suffix from a URL. + +**Field: `proxy.enable_load_balancing`** +Set this value to `true` to have a Tyk node distribute traffic across a list of servers. **Required: ** You must fill in the `target_list` section. + +**Field: `proxy.target_list`** +A list of upstream targets for load balancing (can be one or many hosts). + +**Field: `proxy.check_host_against_uptime_tests`** +If uptime tests are enabled, Tyk will check the hostname of the outbound request against the downtime list generated by the host checker. If the host is found, then it is skipped. + +**Field: `proxy.service_discovery`** +The service discovery section tells Tyk where to find information about the host to proxy to. In a clustered environment this is useful if servers are coming online and offline dynamically with new IP addresses. The service discovery module can pull out the required host data from any service discovery tool that exposes a RESTful endpoint that outputs a JSON object. + +```json expandable +{ + "enable_load_balancing": true, + "service_discovery": { + "use_discovery_service": true, + "query_endpoint": "http://127.0.0.1:4001/v2/keys/services/multiobj", + "use_nested_query": true, + "parent_data_path": "node.value", + "data_path": "array.hostname", + "port_data_path": "array.port", + "use_target_list": true, + "cache_timeout": 10 + }, +} +``` + +**Field: `proxy.service_discovery.use_discovery_service`** +Set this to `true` to enable the discovery module. + +**Field: `proxy.service_discovery.query_endpoint`** +The endpoint to call. + +**Field: `proxy.service_discovery.data_path`** +The namespace of the data path. For example, if your service responds with: + +```json expandable +{ + "action": "get", + "node": { + "key": "/services/single", + "value": "http://httpbin.org:6000", + "modifiedIndex": 6, + "createdIndex": 6 + } +} +``` + +Then your name space would be `node.value`. + +**Field: `proxy.service_discovery.use_nested_query`** +Sometimes the data you are retrieving is nested in another JSON object. For example, this is how Etcd responds with a JSON object as a value key: + +```json expandable +{ + "action": "get", + "node": { + "key": "/services/single", + "value": "{\"hostname\": \"http://httpbin.org\", \"port\": \"80\"}", + "modifiedIndex": 6, + "createdIndex": 6 + } +} +``` + +In this case, the data actually lives within this string-encoded JSON object. So in this case, you set the `use_nested_query` to `true`, and use a combination of the `data_path` and `parent_data_path` (below) + +**Field: `proxy.service_discovery.parent_data_path`** +This is the namespace of where to find the nested value. In the above example, it would be `node.value`. You would then change the `data_path` setting to be `hostname`. Tyk will decode the JSON string and then apply the `data_path` namespace to that object in order to find the value. + +**Field: `proxy.service_discovery.port_data_path`** +In the above nested example, we can see that there is a separate PORT value for the service in the nested JSON. In this case you can set the `port_data_path` value and Tyk will treat `data_path` as the hostname and zip them together (this assumes that the hostname element does not end in a slash or resource identifier such as `/widgets/`). In the example, the `port_data_path` would be `port`. + +**Field: `proxy.service_discovery.target_path`** +The target path to append to the host:port combination provided by the service discovery engine. + +**Field: `proxy.service_discovery.use_target_list`** +If you are using load_balancing, set this value to `true` and Tyk will treat the data path as a list and inject it into the target list of your API definition. + +**Field: `proxy.service_discovery.cache_timeout`** +Tyk caches target data from a discovery service. In order to make this dynamic you can set a cache value when the data expires and new data is loaded. + +**Field: `proxy.transport`** +The transport section allows you to specify a custom proxy and set the minimum TLS versions and any SSL ciphers. + +This is an example of `proxy.transport` definition followed by explanations for every field. +```json expandable +{ + "transport": { + "proxy_url": "http(s)://proxy.url:1234", + "ssl_min_version": 771, + "ssl_ciphers": [ + "TLS_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA" + ], + "ssl_insecure_skip_verify": true, + "ssl_force_common_name_check": false + } +} +``` + +**Field: `proxy.transport.proxy_url`** +Use this setting to specify your custom forward proxy and port. + +**Field: `proxy.transport.ssl_min_version`** +Use this setting to specify your minimum TLS version; note that this is limited by the version of Tyk due to underlying Golang support for legacy TLS versions. + +**Field: `proxy.transport.ssl_ciphers`** +You can add `ssl_ciphers` which takes an array of strings as its value. Each string must be one of the allowed cipher suites as defined at https://golang.org/pkg/crypto/tls/#pkg-constants. This is not applicable from TLS 1.3. + +**Field: `proxy.transport.ssl_insecure_skip_verify`** +Boolean flag to control at the API definition whether it is possible to use self-signed certs for some APIs, and actual certs for others. This also works for `TykMakeHttpRequest` & `TykMakeBatchRequest` in virtual endpoints. + +**Field: `proxy.transport.ssl_force_common_name_check`** +Use this setting to force the validation of a hostname against the certificate Common Name. + +### Upstream Authentication + +**Field: `strip_auth_data`** +When set to `true`, auth related headers will be stripped from requests proxied through the gateway. + +**Field: `request_signing`** +Configuration for Upstream Request Signing using HMAC or RSA algorithms. + +**Field: `request_signing.secret`** +The secret used for signing (not shared with the upstream). + +**Field: `request_signing.key_id`** +An identifier allocated by the upstream used to identify Tyk as the requesting client. + +**Field: `request_signing.algorithm`** +The signing algorithm to be used - one from `hmac-sha1`, `hmac-sha256`, `hmac-sha384`, `hmac-sha512`, `hmac-rsa256` + +**Field: `request_signing.header_list`** +A list of headers to be included in the signature calculation. + +**Field: `request_signing.certificate_id`** +The certificate ID used in the RSA signing operation. + +**Field: `request_signing.signature_header`** +The HTTP header to be used to pass the signature to the upstream. + +### Uptime Tests + +**Field: `uptime_tests`** +This section defines the uptime tests to run for this API. + +**Field: `uptime_tests.check_list`** +A list of tests to run, which can be either short form: + +```json expandable +{ + "uptime_tests": { + "check_list": [ + { + "url": "http://google.com/" + } + ] + } +} +``` + +or long form: + +```json expandable +{ + "uptime_tests": { + "check_list": [ + { + "url": "http://posttestserver.com/post.php?dir=uptime-checker", + "method": "POST", + "headers": { + "this": "that", + "more": "beans" + }, + "body": "VEhJUyBJUyBBIEJPRFkgT0JKRUNUIFRFWFQNCg0KTW9yZSBzdHVmZiBoZXJl", + "timeout": 1000 + } + ] + } +} +``` + +**Field: `uptime_tests.check_list.url`** +The URL to be used for the uptime test. + +**Field: `uptime_tests.check_list.method`** +The HTML method to be used for the request to the `check_list.url` (required for long form tests). + +**Field: `uptime_tests.check_list.headers`** +A list of headers to be applied to the request to the `check_list.url` as key:value pairs (only for long form tests). + +**Field: `uptime_tests.check_list.body`** +The body of the request to be sent to the `check_list.url`, this is Base64 encoded (only for long form tests). + +**Field: `uptime_tests.check_list.timeout`** +The timeout in milliseconds for the uptime check (only for long form tests). + diff --git a/api-management/gateway-config-tyk-oas.mdx b/api-management/gateway-config-tyk-oas.mdx new file mode 100644 index 00000000..fc891cd4 --- /dev/null +++ b/api-management/gateway-config-tyk-oas.mdx @@ -0,0 +1,64 @@ +--- +title: "Tyk OAS" +'og:description': "How to configure Tyk OAS API Definition" +tags: ['Gateway', 'Configuration', 'Tyk OAS', 'Tyk OAS API Definition', 'Tyk OAS API Definition Object'] +sidebarTitle: "Tyk OAS" +--- + +import XTykGateway from '/snippets/x-tyk-gateway.mdx'; + +## Introduction to Tyk OAS + +The upstream service receives requests from Tyk to the *upstream API* after processing based on the configuration applied in the Tyk API definition. Crucially the upstream service remains unaware of Tyk Gateway's processing, responding to incoming requests as it would for direct client-to-service communication. The *API proxy* deployed on Tyk is typically designed to have the same API endpoints, resources and methods that are defined for the upstream service's API. The *upstream API* will often be described according to the industry standard OpenAPI Specification - and this is where Tyk OAS comes in. + +### What is the OpenAPI Specification? + +The *OpenAPI Specification* (OAS) is a standardized framework for describing RESTful APIs in a machine-readable format (typically JSON or YAML). It defines how APIs should be documented, including details about endpoints, request/response formats, authentication, and error codes. In short, OAS is a blueprint for your APIβ€”detailing how the API behaves and how users or services can interact with it. The *OpenAPI Description* (OAD) is the actual content that adheres to this specification, essentially an object that describes the specific functionality of an API. The *OpenAPI Document* refers to a file that contains an OpenAPI description, following the OAS format. + +OpenAPI has become the de facto standard for API documentation because of its consistency, ease of use, and broad tooling support. It allows both developers and machines to interact with APIs more effectively, offering benefits like auto-generated client SDKs, server stubs, and up-to-date documentation. Tools such as Tyk also support validation, testing, and mock servers, which speeds up development and ensures consistency across API implementations. + +Tyk supports [OpenAPI Specification v3.0.x](https://spec.openapis.org/oas/v3.0.3). + +### What is a Tyk OAS API definition? + +Not every feature of an advanced API management platform such as Tyk is covered by the OpenAPI Specification. The *API definition* must provide Tyk with everything it needs to receive and process requests on behalf of the upstream service - so the OpenAPI description of the upstream API is not enough on its own to configure the Gateway. This is where the *Tyk Vendor Extension* comes in, allowing you to configure all the powerful features of Tyk Gateway that are not covered by OAS. + +The [Tyk Vendor Extension](#tyk-vendor-extension) follows the same architectural style as the OpenAPI Specification and is encapsulated in a single object that is appended to the OpenAPI description, creating a *Tyk OAS API definition*. + +#### OpenAPI description + +There are many great explanations of the features and capabilities of the OpenAPI Specification so we won't repeat it all here. A good place to start learning is from the maintainers of the specification: the [OpenAPI Initiative](https://learn.openapis.org/). The minimal set of elements that must be defined + +Tyk treats the OpenAPI description as the source of truth for the data stored within it. This means that Tyk does not duplicate those data in the Tyk Vendor Extension but rather builds upon the basic configuration defined in the OAD. + +#### Tyk Vendor Extension + +The Tyk Vendor Extension is a JSON object (`x-tyk-api-gateway`) within the Tyk OAS API definition that encapsulates all of the Gateway configuration that is not contained within the OpenAPI description. + +It is structured in four sections: + +- `info` containing metadata used by Tyk to manage the API proxy, including name, identifiers, status, and version +- `server` contains configuration for the client-gateway integration, including listen path and authentication method +- `middleware` contains configuration for the gateway's middleware chain, split into API-level and endpoint-level settings +- `upstream` contains configuration for the gateway-upstream integration, including targets, load balancing and rate limits + +The extension has been designed, as has OAS, to have minimal content so if a feature is not required for your API (for example, payload transformation) then this can be omitted from the API definition. Most features have an `enabled` flag which must be set for Tyk to apply that configuration. This can be used to include settings in the API definition and enable them only when required (useful during API development, testing and debug). + +In the OpenAPI Specification *paths* define the API endpoints, while *operations* specify the HTTP methods (GET, POST, PUT, DELETE) and actions for each endpoint. They describe how the API handles requests, including parameters, request bodies, responses, and status codes, providing a clear structure for API interactions. Tyk interprets this information directly from the OpenAPI description and uses the `operationID` field to link the endpoint level middleware configuration within the Tyk Vendor Extension to the appropriate endpoint. + +### Modifying the OpenAPI description + +Tyk will only make additions or modifications to the OAD when the user makes certain changes in the Tyk API Designer and as follows: + +- The URL on Tyk Gateway to which client requests should be sent will be added at the first location in the `servers` list +- The OpenAPI Specification declares `paths` which describe the available endpoints (paths) and the operations that can be performed on them (such as `GET`, `POST`, `PUT`, `DELETE`). Tyk will modify this list if changes are made using the Tyk API Designer, for example if an endpoint is added. + +Where Tyk might modify the OpenAPI description, this is noted in the appropriate section of the documentation. + +If changes are made via the Tyk API Designer that impact the OpenAPI description, we recommend that you export the OAD from Tyk to store in your source of truth repository. This ensures that your records outside Tyk accurately reflect the API that is consumed by your clients (for example, if you publish documentation from the OpenAPI Specification of your API). + +Equally, if you make changes to your OpenAPI description outside Tyk, we provide a simple method to update (or patch) your Tyk API definition with the updated OAD. Alternatively you might prefer to create a new version of your API for the updated OpenAPI description, depending on the current stage of the API in its lifecycle. + + + + diff --git a/api-management/gateway-events.mdx b/api-management/gateway-events.mdx new file mode 100644 index 00000000..37bfa093 --- /dev/null +++ b/api-management/gateway-events.mdx @@ -0,0 +1,930 @@ +--- +title: "Gateway Events" +'og:description': "Introduction to Gateway Events" +tags: ['Gateway', 'Events', 'Async APIs', 'Asynchronus APIs', 'Error Templates', 'Event Types', 'Event Webhooks', 'Event Metadata'] +sidebarTitle: "Gateway Events" +--- + +Tyk Gateway will generate asynchronous events when certain conditions are met, for example a rate limit being exceeded, an expired key attempting to access an API, or a circuit breaker triggering due to a slow or unresponsive upstream. + +Tyk has a flexible model for handling these API events. + +## Event categories + +There are four different categories of events that can be fired by Tyk: +- [API events](#api-events) +- [Token lifecycle events](#token-lifecycle-events) +- [Advanced quota usage events](#advanced-quota-usage-events) +- [Custom events](#custom-events) + +### API events + +Tyk can generate (or *fire*) a variety of built-in API events due to activity triggered by an API request, such as exceeded rate limits, depleted quotas or attempts to access using expired keys. The full list of standard API events is available [here](/api-management/gateway-events#api-events). + +### Token lifecycle events + +Alongside the events that are fired in response to API requests, Tyk will also mark the creation, update or deletion of access tokens (keys) with dedicated events as indicated [here](/api-management/gateway-events#token-lifecycle-events). + +### Advanced quota usage events + +Tyk will generate [standard quota events](/api-management/gateway-events#standard-quota-events) when a client quota has been consumed, but what if you want to have more granular notification of quota usage as your clients are approaching their quota limit? + +For this, Tyk provides [advanced quota monitoring](/api-management/gateway-events#monitoring-quota-consumption) that can be configured to trigger a dedicated event handler when the API usage exceeds different thresholds approaching the quota limit. + +### Custom events + +The event subsystem has been designed to be easily extensible, so the community can define additional events within the Tyk codebase which can then be handled using the exsiting event handling system. + +## Handling events with Tyk + +Tyk has a simple event handling system where *event handlers* are assigned (or registered) to the different [events](/api-management/gateway-events#event-types) that Tyk can generate. These handlers are assigned per-API so when an event is generated for an API and there is an *event handler* registered for that *event*, the handler will be triggered. + +Three different categories of *event handler* can be registered for each event: +- a [webhook](/api-management/gateway-events#event-handling-with-webhooks) that will call out to an external endpoint +- an [event log](/api-management/gateway-events#logging-api-events-1) that will write to the configured [log output](/api-management/logs-metrics#system-logs) +- your own [custom event handler](/api-management/gateway-events#custom-api-event-handlers) that will run in a JavaScript virtual machine on the Tyk server + + + + + Remember that quota usage monitoring has a [dedicated mechanism](/api-management/gateway-events#monitoring-quota-consumption) for handling these special events. + + + +### Event metadata + +When an API event is fired, if there is an *event handler* registered for that combination of API and event then the handler will be provided with a rich set of [metadata](/api-management/gateway-events#event-metadata-1) that can be used by the external system (webhook) or custom (JavaScript) code to determine the action to be taken. + +## Event Types + +The built-in events that Tyk Gateway will generate are: + +### Rate limit events + +- `RatelimitExceeded`: the rate limit has been exceeded for a specific key +- `OrgRateLimitExceeded`: the rate limit has been exceeded for a specific organization +- `RateLimitSmoothingUp`: the [intermediate rate limit allowance](/api-management/rate-limit#rate-limit-smoothing) has been increased for a specific key +- `RateLimitSmoothingDown`: the [intermediate rate limit allowance](/api-management/rate-limit#rate-limit-smoothing) has been decreased for a specific key + +### Standard quota events + +- `QuotaExceeded`: the quota for a specific key has been exceeded +- `OrgQuotaExceeded`: the quota for a specific organization has been exceeded + +### Authentication failure events + +- `AuthFailure`: a key has failed authentication or has attempted access and was denied +- `KeyExpired`: an attempt has been made to access an API using an expired key + +### API version events + +- `VersionFailure`: a key has attempted access to a version of an API that it does not have permission to access + +### Circuit breaker events + +- `BreakerTripped`: a circuit breaker on a path has tripped and been taken offline +- `BreakerReset`: a circuit breaker has reset and the path is available again +- `BreakerTriggered`: a circuit breaker has changed state, this is generated when either a `BreakerTripped`, or a `BreakerReset` event occurs; a status code in the metadata passed to the webhook will indicate which of these events was triggered + +### Uptime events + +- `HostDown`: the uptime checker has found that a host is down/not available +- `HostUp`: the uptime checker has found that a host is available again after being offline + +### Token lifecycle events + +- `TokenCreated`: a token has been created +- `TokenUpdated`: a token has been changed/updated +- `TokenDeleted`: a token has been deleted + +## Event Metadata + +When Tyk generates an [event](/api-management/gateway-events#event-types) it will compile the following metadata that is passed to the event handler: + +- `Message` (string): a human readable message from Tyk Gateway that adds detail about the event +- `Path` (string): the path of the API endpoint request that led to the event being fired +- `Origin` (string): origin data for the source of the request (if this exists) +- `Key` (string): the key that was used in the request +- `OriginatingRequest` (string): Based64-encoded [raw inbound request](#raw-request-data) + + + + + Circuit breaker events provide different metadata, see [Circuit Breakers](/planning-for-production/ensure-high-availability/circuit-breakers) to see what is provided when the `BreakerTripped`, `BreakerReset` or `BreakerTriggered` events are generated. + + + +### Using the metadata + +The metadata are exposed so that they can be used by the event handler (webhook or custom) using Go templating. For details of how each type of event handler can access these data, please see the appropriate section for [webhook](/api-management/gateway-events#webhook-payload) or [custom](/api-management/gateway-events#the-event-object) event handlers. + +### Raw Request Data + +The `OriginatingRequest` metadata is a Base64-encoded wire-protocol representation of the original request to the event handler. If you are running a service bus or queue that stores failed, throttled or other types of requests, you can decode this object and parse it in order to re-create the original intent of the request (e.g. for post-processing). + +### Logging API Events + +Tyk’s built-in logging event handler is designed primarily for debugging purposes and will store details of an API event to the configured logger output. + +The Tyk platform can be configured to log at various verbosity levels (info, debug, warn, error) and can be integrated with third-party log aggregation tools like Sentry, Logstash, Graylog, and Syslog. For full details on configuring the Tyk logger, see [this section](/api-management/logs-metrics#system-logs). + +
+ + +Logging event handlers are currently only supported by Tyk Classic APIs. + + + +### Configuring the event handler + +Registering a logging event handler to your Tyk Classic API is the same as adding any other event handler, within the `event_handlers` section of the API definition. + +The `handler_name` for the logging event handler should be set to: `eh_log_handler`. + +The `handler_meta` for the logging event handler contains a single field: +- `prefix` is a label that will be prepended to each log entry + +For example, to register event handlers to log the `AuthFailure` and `KeyExpired` events you might add the following to your API definition: + +```json expandable +{ + "event_handlers": { + "events": { + "AuthFailure": [ + { + "handler_name": "eh_log_handler", + "handler_meta": { + "prefix": "AuthFailureEvent" + } + } + ], + "KeyExpired": [ + { + "handler_name": "eh_log_handler", + "handler_meta": { + "prefix": "KeyExpiredEvent" + } + } + ] + } + } +} +``` + +In this example +- the `AuthFailure` event will trigger the event handler to generate a log with the prefix `AuthFailureEvent` +- the `KeyExpired` event will trigger the event handler to generate a log with the prefix `KeyExpiredEvent` + +When the event handler is triggered an entry will be made in the log containing the corresponding prefix, which can be useful for monitoring and debugging purposes. + +## Event handling with webhooks + +### Overview + +A webhook is a mechanism for real-time, event-driven communication between different systems or applications over the internet. It is an HTTP callback, typically an HTTP POST request that occurs when something happens. Webhooks are real-time, automated and lightweight. Notifications are sent immediately when events occur without the need for the receiving service to poll. + +In the context of Tyk Gateway, webhooks are event handlers that can be registered against API Events. The webhook will be triggered when the corresponding event is fired and will send a customizable fixed payload to any open endpoint. + +#### When to use webhook event handlers + +There are many occasions when you might use webhooks for event handling, here are just a few examples. + +##### Rate limit violations + +When an API consumer exceeds their allocated rate limit, the `RatelimitExceeded` event will be fired. A webhook event handler can be employed to notify an upstream system to take actions such as updating a dashboard, notifying the account manager, or adjusting the client's service tier. + +##### API key lifecycle events + +When an expired API key is used to access an API, the client will receive an error and the `KeyExpired` event will be fired. A webhook event handler can be employed to notify an upstream system to take actions such as renewing the key, logging the failure in a CRM or notifying the account manager to initiate customer communication. + +##### Upstream service problems + +When an API circuit breaker triggers due to an unresponsive upstream service, the `BreakerTripped` event will be fired. A webhook event handler can be employed to update monitoring dashboards or to trigger automated recovery scripts or processes. + +#### How webhook event handlers work + +With Tyk Gateway, the webhook event handler is a process that runs asynchronously in response to an API event being fired. It will issue an HTTP request to any open endpoint and is fully configurable within the API definition. + +The HTTP method, body, header values, and target URL can all be configured in the API definition. The [request body](#webhook-payload) is generated using a Tyk template file that has access to the [event metadata](/api-management/gateway-events#event-metadata-1). + +The webhook event handler runs in its own process and so does not block the operation of the Gateway. + +##### Webhook cooldown + +It is very likely that an `AuthFailure` event will fire on the same endpoint more than once if the requesting client is automated. If this event triggered a webhook that caused an email to be sent, then if this event occurred 10 times a second, the email recipient would be flooded with emails. In an attempt to mitigate against events such as this, you can set a cooldown timer, in the webhook handler. This prevents the webhook from being triggered again if the event is fired again within the time period specified. + +##### Webhook payload + +When your webhook event handler is triggered, it will send an HTTP request to the configured target. For HTTP methods that support a request body, for example `POST`, the event handler will process a [Go template](/api-management/traffic-transformation/go-templates) to produce the payload. + +If no template is provided in the webhook event handler configuration in the API definition, Tyk Gateway will look for the default file `templates/default_webhook.json`. Any text file accessible to the Gateway can be used to store the Go template to be used by the event handler when constructing the payload. + +The event handler has access to the [event metadata](/api-management/gateway-events#event-metadata-1) and this can be accessed by the template using the `{{.Meta.XXX}}` namespace. + +The [event type](/api-management/gateway-events#event-types) that triggered the event handler can be accessed as `{{.Type}}`. + +For most event types, the default webhook template has this form: + +```json expandable +{ + "event": "{{.Type}}", + "message": "{{.Meta.Message}}", + "path": "{{.Meta.Path}}", + "origin": "{{.Meta.Origin}}", + "key": "{{.Meta.Key}}" +} +``` + +This would generate a request body (payload) such as: +```json expandable +{ + "event": "RatelimitExceeded", + "message": "API Rate Limit Exceeded", + "path": "/example-global-webhook/", + "origin": "99.242.139.220", + "key": "apilimiter-66336c67cb7191f791f167134b20d1f4c14b4bb5672b57f4b2813c86" +} +``` + +#### Using webhooks with Tyk Dashboard + +Webhook event handlers are configured within the API definition, which is used by Tyk Gateway to determine the appropriate action to be performed in response to a Gateway event. + +When using Tyk Dashboard, you are able to create *global webhooks* that can be re-used across multiple events and APIs, allowing you to modify the webhook configuration for a batch of APIs and/or events from one location. + +##### Local and global webhooks + +Tyk Dashboard supports the declaration of webhooks *globally* and *locally*: +- **Global webhooks** are declared outside the API definition and linked via a *webhook id*; changes to the global webhook definition will be reflected in all APIs that reference that *webhook id* +- **Local webhooks** are fully defined within the API definition; changes to the local webhook configuration will affect only the API within which it is defined + +*Global webhook definitions* are registered with the Dashboard using the [UI](#creating-a-global-webhook-definition-using-tyk-dashboard) or [Dashboard API](/api-management/dashboard-configuration#web-hooks-api) and assigned a unique *webhook id* that can be obtained via the [Dashboard API](/api-management/dashboard-configuration#list-web-hooks) or via drop-down selection within the UI. + +If you assign a global webhook definition to an API to handle an event, then Tyk Dashboard will retrieve the definition and update it in the API definition when the API is loaded (or re-loaded) to the Gateway. + +##### Creating a global webhook definition using Tyk Dashboard + +To create a global webhook definition from the Dashboard UI you should follow these steps: + +**Steps for Configuration** + +1. **Create the webhook definition** + + Select **Webhooks** from the **API Management** Menu: + + Webhooks menu item + + Click **Add Webhook**. + + Add webhook button + +2. **Configure the webhook** + + Now you need to tell Tyk how and where to send the request. You can include custom headers, for example to inform the target service that the request has come from Tyk - remember to click **ADD** to add the custom header to the configuration. + + Add webhook detail + + Click **Save** to save it. + +
+ +If you're using Tyk OAS APIs, then you can find details and examples of how to configure webhook event handlers [here](/api-management/gateway-events#webhook-event-handlers-with-tyk-oas-apis). + +If you're using Tyk Classic APIs, then you can find details and examples of how to configure webhook event handlers [here](/api-management/gateway-events#webhook-event-handlers-with-tyk-classic-apis). + +### Webhook event handlers with Tyk OAS APIs + +[Webhooks](/api-management/gateway-events#event-handling-with-webhooks) are event handlers that can be registered against API Events. The webhook will be triggered when the corresponding event is fired and will send a customizable fixed payload to any open endpoint. + +Webhooks are configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation). You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the legacy Tyk Classic APIs, then check out the [Tyk Classic](/api-management/gateway-events#webhook-event-handlers-with-tyk-classic-apis) page. + +#### Set up a webhook event handler in the Tyk OAS API Definition + +Event handling is configured by adding the `eventHandlers` object to the `server` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition. + +The `eventHandlers` object is an array containing configurations for all event handlers registered with the API. + +##### Local webhook configuration + +When using a local webhook, the event handler element in the `eventHandlers` object has the following configuration which fully declares the webhook behaviour: +- `enabled`: enable the event handler +- `trigger`: the API event that will trigger the webhook +- `type`: the type of event handler, in this case should be set to `webhook` +- `cooldownPeriod`: the [webhook cooldown](/api-management/gateway-events#webhook-cooldown) for duplicate events (in duration format, e.g. 10s, 1m30s); use this to prevent flooding of the target endpoint when multiple events are fired in quick succession +- `name`: a human readable name for the webhook, which will be displayed in Tyk Dashboard +- `url`: this is an **absolute URL** to which the request will be sent +- `method`: this can be any of `GET`, `PUT`, `POST`, `PATCH` or `DELETE` and will be the HTTP method used to send the request; methods that do not support an encoded request body will not have the event metadata provided with the request; we advise using `POST` where possible +- `bodyTemplate`: this is the path to the [webhook template](/api-management/gateway-events#webhook-payload) that will be used to construct the request body +- `headers`: a map of custom headers to be provided with the request + +For example: +```json {hl_lines=["18-33"],linenos=true, linenostart=1} expandable +{ + "info": { + "title": "example-local-webhook", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": {}, + "components": {}, + "x-tyk-api-gateway": { + "info": { + "name": "example-local-webhook", + "state": { + "active": true + } + }, + "server": { + "eventHandlers": [ + { + "enabled": true, + "trigger": "RatelimitExceeded", + "cooldownPeriod": "1s", + "type": "webhook", + "name": "My local webhook", + "url": "https://webhook.site/", + "method": "POST", + "headers": [ + { + "name": "X-Tyk", + "value": "example-local-webhook" + } + ], + "bodyTemplate": "templates/default_webhook.json" + } + ], + "listenPath": { + "strip": true, + "value": "/example-local-webhook/" + } + }, + "upstream": { + "rateLimit": { + "enabled": true, + "per": "10s", + "rate": 2 + }, + "url": "http://httpbin.org/" + } + } +} +``` + +In this example a local webhook has been registered to trigger when the `RatelimitExceeded` event is fired. The request rate limit has been set at 2 requests per 10 seconds, so simply make three requests in quick succession to trigger the webhook. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the local webhook feature. + +Note that to test this you will need to provide a valid target URL for your webhook to send the request; we've used `http://webhook.site`. + + +##### Global webhook configuration + +When using a *global webhook*, the event handler element in the `eventHandlers` object has the following configuration, which references the externally declared webhook using its `id`: +- `enabled`: enable the event handler +- `trigger`: the API event that will trigger the webhook +- `type`: the type of event handler, in this case should be set to `webhook` +- `cooldownPeriod`: the [webhook cooldown](/api-management/gateway-events#webhook-cooldown) for duplicate events (in duration format, e.g. 10s, 1m30s); use this to prevent flooding of the target endpoint when multiple events are fired in quick succession +- `id`: the *webhook id* assigned by Tyk to the global webhook when it was created (this can be determined using the [list webhooks](/api-management/dashboard-configuration#list-web-hooks) endpoint in the Tyk Dashboard API) + +For example: + +```json {hl_lines=["18-24"],linenos=true, linenostart=1} expandable +{ + "info": { + "title": "example-global-webhook", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": {}, + "components": {}, + "x-tyk-api-gateway": { + "info": { + "name": "example-global-webhook", + "state": { + "active": true + } + }, + "server": { + "eventHandlers": [ + { + "enabled": true, + "trigger": "RatelimitExceeded", + "cooldownPeriod": "1s", + "type": "webhook", + "id": "" + } + ], + "listenPath": { + "strip": true, + "value": "/example-global-webhook/" + } + }, + "upstream": { + "rateLimit": { + "enabled": true, + "per": "10s", + "rate": 2 + }, + "url": "http://httpbin.org/" + } + } +} +``` + +In this example a local webhook has been registered to trigger when the `RatelimitExceeded` event is fired. The request rate limit has been set at 2 requests per 10 seconds, so simply make three requests in quick succession to trigger the webhook. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the global webhook feature. + +Note, however, that to test this you will need to create a *global webhook* in your Tyk Dashboard and replace the value in `id` with the *webhook id* that Tyk Dashboard has allocated to your webhook. You can find this by querying the [list webhooks](/api-management/dashboard-configuration#list-web-hooks) endpoint in the Tyk Dashboard API. +
+
+ + +When a *global webhook* is registered to a Tyk OAS API, Tyk will create a read-only copy of the webhook [configuration](#local-webhook-configuration) (`url`, `method`, `bodyTemplate`, `headers`) within the API definition. This is so that Tyk Gateway knows how to handle the event, as it does not have access to the store of *global webhooks* registered with Tyk Dashboard. +
+
+If the global webhook is subsequently deleted from the Tyk Dashboard, the webhook will automatically be converted to a local webhook in any API definition that was using it. + + + + +#### Set up a webhook event handler in the Tyk Dashboard + +It is very simple to register webhooks to be triggered in response to specific API events when using Tyk OAS APIs with the Tyk Dashboard. The API Designer in the Dashboard allows you to define *local webhooks* and to register *global webhooks* to handle events. + +If you want to use a *global webhook* then you'll need to declare it first, following [these instructions](/api-management/gateway-events#creating-a-global-webhook-definition-using-tyk-dashboard). + +1. **Add event handler** + + From the **Settings** tab in the API Designer, scroll down to the **Server** section to find the **Event Handlers** pane. Select **Add Event**. + + Add an event handler from the Server section + +2. **Choose the event to be handled** + + This will add an event handler to the API. You'll need to select which event you want to handle from the drop-down list. Note that currently Tyk OAS only supports webhook event handlers, so this will default to *webhook* type. + + Choose the event that will trigger the webhook + +3. **Choose and configure global webhook** + + If you want to use a webhook that you've already registered with Tyk Dashboard, ensure that the **Webhook source** is set to **Global webhook** then select from the drop-down list. + + The only other thing you'll need to configure is the cooldown period. + + Select from the list of available global webhooks + + Note that Tyk automatically retrieves the details of the *global webhook* and displays them (read-only) in the API designer. + + A fully configured global webhook + + Don't forget to select **Save API** to apply the changes. + +4. **Configure local webhook** + + If you don't want to use a shared *global webhook* but instead want to configure a *local webhook* only available to this API/event then you should ensure that the **Webhook source** is set to **Local webhook**. + + Ready to configure a local webhook + + Now you can complete the various fields to set up your *local webhook*. If you want to add custom headers to send with the HTTP request, select **New Header** then enter the header key and value. + + A fully configured global webhook + + Don't forget to select **Save API** to apply the changes. + +### Webhook event handlers with Tyk Classic APIs + +[Webhooks](/api-management/gateway-events#event-handling-with-webhooks) are event handlers that can +be registered against API Events. The webhook will be triggered when the corresponding event is fired and will send a +customisable fixed payload to any open endpoint. + +Webhooks are configured in the Tyk Classic API Definition. You can do this via the Tyk Dashboard API or in the API +Designer. + +If you're using the newer Tyk OAS APIs, then check out the [Tyk +OAS](/api-management/gateway-events#webhook-event-handlers-with-tyk-oas-apis) +page. + +#### Set up a webhook event handler in the Tyk Classic API Definition + +To add a webhook event handler you must add a new event handler object within the `event_handlers.events` section of the +API definition for the appropriate [API event](/api-management/gateway-events#event-types). + +The event handler object has the following configuration: + +- `handler_name`: this identifies the type of event handler and must be set to `eh_web_hook_handler` +- `handler_meta`: this structure configures the HTTP request that will be sent when the webhook is triggered + +The `handler_meta` object has the following configuration: + +- `method`: this can be any of `GET`, `PUT`, `POST`, `PATCH` or `DELETE` and will be the HTTP method used to send the + request; methods that do not support an encoded request body will not have the event metadata provided with the + request; we advise using `POST` where possible +- `target_path`: this is an **absolute URL** to which the request will be sent +- `template_path`: this is the path to the [webhook + template](/api-management/gateway-events#webhook-payload) that will be + used to construct the request body +- `header_map`: a map of custom headers to be provided with the request +- `event_timeout`: the [webhook + cooldown](/api-management/gateway-events#webhook-cooldown) for duplicate + events (in seconds); use this to prevent flooding of the target endpoint when multiple events are fired in quick succession + +For example: + +```json {linenos=true, linenostart=1} expandable +{ + "event_handlers": { + "events": { + "AuthFailure": [ + { + "handler_name": "eh_web_hook_handler", + "handler_meta": { + "method": "POST", + "target_path": "http://posttestserver.com/post.php?dir=tyk-event-test", + "template_path": "templates/default_webhook.json", + "header_map": { "X-Tyk-Test-Header": "Tyk v1.BANANA" }, + "event_timeout": 10 + } + } + ] + } + } +} +``` + +In this example, when the `AuthFailure` event is fired, the webhook event handler will send a request to +`POST http://posttestserver.com/post.php?dir=tyk-event-test` and then start a 10 second cooldown before another webhook +request can be sent. + +The request will have one custom header `X-Tyk-Test-Header: Tyk v1.BANANA` and the body will be constructed from the +webhook template located at `templates/default_webhook.json`. + + + +This manually configured webhook event handler is private to the API within which it has been defined, it is not a +[global +webhook](/api-management/gateway-events#using-webhooks-with-tyk-dashboard). + + + +#### Set up a webhook event handler in the Tyk Dashboard + +It is very simple to register webhooks to be triggered in response to specific API events when using Tyk Classic APIs +with the Tyk Dashboard. The API Designer in the Dashboard allows you to register _global webhooks_ to handle events. + +Note that Tyk Gateway does not have access to the _global webhook_ definitions registered with Tyk Dashboard and can +only operate on the configuration within the API definition. Dashboard will manage the conversion of _global webhooks_ +to [locally defined webhook handlers](#set-up-a-webhook-event-handler-in-the-tyk-classic-api-definition) within the Tyk +Classic API definition, automatically updating the configuration in each API definition when the APIs are reloaded to +the Gateway. + +1. **Define the webhook** + + Before you can configure a webhook event handler for your API, you must first create a global webhook from the + **Webhooks** screen in the **API Management** menu, as described + [here](/api-management/gateway-events#creating-a-global-webhook-definition-using-tyk-dashboard). + +2. **Register the webhook with the event** + + From the API Designer select the **Advanced Options** tab and locate the **Webhooks** panel: + + Webhook API Details + + Now: + + - select the _API Event_ for which you want to trigger the webhook from the dropdown list + - select the _Webhook to use_ when the event fires, again from the dropdown list + - finally, configure the required _Cooldown period_ + - click **Add** + + Note that you can register multiple webhooks to be triggered in response to a single event and you can register the same + webhook with multiple API events. + + Remember to click **Save** to save your changes. + +#### Set up a webhook event handler in Tyk Operator + +Tyk Operator supports event handler integration for Tyk Classic API Definition. Configuring the `event_handlers` field +in ApiDefinition Custom Resource Definition (CRD) enables webhooks to be triggered by [specific +API events](/api-management/gateway-events#event-types). + +The process for configuring webhook event handlers using Tyk Operator is similar to that explained in +[Set up a webhook event handler in the Tyk Classic API Definition](#set-up-a-webhook-event-handler-in-the-tyk-classic-api-definition). +The example API Definition below enables the event handler by setting `spec.event_handlers`. + +```yaml {hl_lines=["14-25"],linenos=true, linenostart=1} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: webhook-handler +spec: + name: webhook-handler + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /webhook-handler + strip_listen_path: true + event_handlers: + events: + AuthFailure: + - handler_name: "eh_web_hook_handler" + handler_meta: + method: "POST" + name: "webhook name" + target_path: "http://posttestserver.com/post.php?dir=tyk-event-test" + template_path: "templates/default_webhook.json" + header_map: + X-Tyk-Test-Header: "Tyk v1.BANANA" + event_timeout: 10 +``` + + +## Logging API events + +Tyk’s built-in logging event handler is designed primarily for debugging purposes and will store details of an API event to the configured logger output. + +The Tyk platform can be configured to log at various verbosity levels (info, debug, warn, error) and can be integrated with third-party log aggregation tools like Sentry, Logstash, Graylog, and Syslog. For full details on configuring the Tyk logger, see [this section](/api-management/logs-metrics#system-logs). + +
+ + +Logging event handlers are currently only supported by Tyk Classic APIs. + + + +### Configuring the event handler + +Registering a logging event handler to your Tyk Classic API is the same as adding any other event handler, within the `event_handlers` section of the API definition. + +The `handler_name` for the logging event handler should be set to: `eh_log_handler`. + +The `handler_meta` for the logging event handler contains a single field: +- `prefix` is a label that will be prepended to each log entry + +For example, to register event handlers to log the `AuthFailure` and `KeyExpired` events you might add the following to your API definition: + +```json expandable +{ + "event_handlers": { + "events": { + "AuthFailure": [ + { + "handler_name": "eh_log_handler", + "handler_meta": { + "prefix": "AuthFailureEvent" + } + } + ], + "KeyExpired": [ + { + "handler_name": "eh_log_handler", + "handler_meta": { + "prefix": "KeyExpiredEvent" + } + } + ] + } + } +} +``` + +In this example +- the `AuthFailure` event will trigger the event handler to generate a log with the prefix `AuthFailureEvent` +- the `KeyExpired` event will trigger the event handler to generate a log with the prefix `KeyExpiredEvent` + +When the event handler is triggered an entry will be made in the log containing the corresponding prefix, which can be useful for monitoring and debugging purposes. + +## Custom API event handlers + +Tyk supports you to script your own custom code in JavaScript (JS) that will be invoked in response to API events. This is executed asynchronously so you don't need to worry about it blocking the Gateway handling requests. Event handlers like this can be very powerful for automating session, user and API-level functions. + +It is important to note that unlike custom JavaScript [plugins](/api-management/plugins/javascript#), custom event handlers execute in a *global* JavaScript environment. This means that you need to be careful when naming the event handlers: if you use the same event handler name for different event handling code across two APIs, only one of them will execute, as the other will be overridden when loaded. + +Custom event handlers have access to the [JavaScript API](/api-management/plugins/javascript#javascript-api) which gives access to the session object and enables your code to make HTTP calls. This is particularly useful if you want to interface with another API with a complex request/response cycle. + +
+ + +Custom event handlers are currently only supported by Tyk Classic APIs. + + + +### Creating a custom event handler + +A custom event handler consists of a function that accepts two variables (`event` and `context`) and has no return value. + +Creating an event handler is very similar to [creating custom JS plugins](/api-management/plugins/javascript#using-javascript-with-tyk), simply invoke the correct constructors with a closure in the TykJS namespace: + +```js expandable +// ---- Sample custom event handler ----- +var sampleHandler = new TykJS.TykEventHandlers.NewEventHandler({}); + +sampleHandler.NewHandler(function(event, context) { + // You can log to Tyk console output by calling the built-in log() function: + log("This handler does nothing, but this will appear in your terminal") + + return +}); +``` + +#### The `event` object + +This contains the [event metadata](/api-management/gateway-events#event-metadata-1) in the following structure: + +```json expandable +{ + "EventType": "Event Type Code", + "EventMetaData": { + "Message": "My Event Description", + "Path": "/{{api_id}}/{{path}}", + "Origin": "1.1.1.1:PORT", + "Key": "{{Auth Key}}" + }, + "TimeStamp": "2024-01-01 23:59:59.111157073 +0000 UTC" +} +``` + +#### The `context` Variable + +Tyk injects a `context` object into your event handler giving access to more information about the request. This object has the following structure: + +```js +type JSVMContextGlobal struct { + APIID string + OrgID string +} +``` + +It is populated with the API ID and Org ID of the request that your custom function can use together with the `event` metadata to interact with the Tyk REST API functions, for example: + +```js +// Use the TykGetKeyData function to retrieve a session from the session store, use the context variable to give the APIID for the key. +var thisSession = JSON.parse(TykGetKeyData(event.EventMetaData.Key, context.APIID)) +log("Expires: " + thisSession.expires) +``` + +### Registering a custom event handler + +Registering a custom event handler to your Tyk Classic API is the same as adding any other event handler, within the `event_handlers` section of the API definition. + +The `handler_name` for a custom event handler should be set to: `eh_dynamic_handler`. + +The `handler_meta` for a custom event handler consists of two fields: +- `name` is the unique name of your middleware object +- `path` is the relative path to the file (it can be absolute) + +For example, to register a custom event handler with the name `sessionHandler` to be invoked in response to the `KeyExpired` event you would add the following to your API definition: + +```json expandable +{ + "event_handlers": { + "events": { + "KeyExpired": [ + { + "handler_name":"eh_dynamic_handler", + "handler_meta": { + "name": "sessionHandler", + "path": "event_handlers/session_editor.js" + } + } + ] + } + } +} +``` + +### Loading custom event handlers + +The JavaScript files are loaded on API reload into the global JSVM. If a hot-reload event occurs, the global JSVM is re-set and files are re-loaded. This could cause event handlers that are currently executing to get abandoned. This is a measured risk and should not cause instability, however it should be noted that because of this, in an environment where reloads occur frequently, there is risk that event handler may not fire correctly. + +## Monitoring quota consumption + +Tyk provides the ability to actively monitor both user and organization quotas, using a dedicated webhook to notify your stakeholders, your system stack or the requesting API client when certain thresholds have been reached for a token. + +Unlike API event [webhooks](/api-management/gateway-events#event-handling-with-webhooks) the quota monitor is configured at the Gateway level. + +
+ + +Advanced quota threshold monitoring is currently only supported by Tyk Classic APIs. + + + +### Configuring the quota consumption monitor + +To enable advanced quota monitoring you will need to add a new `monitor` section to your Tyk Gateway configuration file (`tyk.conf`). + +This has the following fields: +- `enable_trigger_monitors`: set to `true` to have the monitors start to measure quota thresholds +- `configuration`: a [webhook configuration](/api-management/gateway-events#event-handling-with-webhooks) object +- `global_trigger_limit`: this is a percentage of the quota that the key must consume for the webhook to be fired +- `monitor_user_keys`: set to `true` to monitor individual tokens (this may result in a large number of triggers as it scales with the number of user tokens that are issued) +- `monitor_org_keys`: set to `true` to monitor organization quotas + +For example: + +```json expandable +{ + "monitor": { + "enable_trigger_monitors": true, + "configuration": { + "method": "POST", + "target_path": "http://posttestserver.com/post.php?dir=tyk-monitor-drop", + "template_path": "templates/monitor_template.json", + "header_map": {"x-tyk-monitor-secret": "12345"}, + "event_timeout": 10 + }, + "global_trigger_limit": 80.0, + "monitor_user_keys": false, + "monitor_org_keys": true + } +} +``` + +With this configuration, a monitor is configured to issue a request to `POST http://posttestserver.com/post.php?dir=tyk-monitor-drop` when 80% of the API-level quota has been consumed. This request will have the `x-tyk-monitor-secret` header (set to a value of `12345`) and will provide the content of the template file found at `templates/monitor_template.json` in the request body. A minimum of 10 seconds will elapse between successive monitor webhooks being fired. + +
+ + +If you are using our [Classic Developer Portal](/tyk-developer-portal/tyk-portal-classic/portal-events-notifications), developers registered in the portal will also receive emails about quota threshold limits being reached. + + + +#### Setting advanced thresholds + +The default quota consumption monitor will be triggered at the same level of quota usage for all users. Sometimes you might want to have a more granular approach with different triggering thresholds per user or organization. Sometimes you might want to fire the event at multiple thresholds, for example when the user hits 50%, 75% and 90% of their allowed quota. + +You can set user specific trigger levels for a user by additionally adding a `monitor` section to the access key ([Session Object](/api-management/policies#what-is-a-session-object)). This has one field, which is an array of `trigger_limits` (thresholds) that must be in *descending* order and represent the percentage of the quota that must be reached in order for the trigger to be fired, for example: + +```yaml +"monitor": { + "trigger_limits": [90.0, 75.0, 50.0] +} +``` + +If this is included in the session object, then the quota threshold event will be fired and the monitor webhook triggered when the user hits 50%, then 75%, and then again at 90% consumption. + +You can configure advanced thresholds for all users in an organization by adding the `monitor` section to the organization session object. + +### Webhook payload + +When the quota consumption monitor is fired, the webhook request that is issued will have the following payload: + +```json expandable +{ + "event": "TriggerExceeded", + "message": "Quota trigger reached", + "org": "53ac07777cbb8c2d53000002", + "key": "", + "trigger_limit": "80", +} +``` + +- `trigger_limit` will indicate which threshold has been reached (as defined in the session object's `monitor` section). +- `org` will contain the OrgID for the user or organization that triggered the event +- `key` will contain the *raw API key* used in the request only if the event was triggered by a user quota + +*Note: if the webhook was triggered by an organization threshold, `key` will be blank.* + +
+ + +When the monitor is triggered by a user hitting their quota threshold, the raw API key is provided in the webhook payload. It is important to secure the webhook endpoint and to handle the payload securely on the receiving end. + + + +## Error Templates + +In v2.2 the error handler allowed the use a single JSON template to communicate errors to users (a default one is shipped with Tyk, it's located in `templates/error.json`). + +As of v2.3 it is possible to use different templates for specific `HTTP error codes`. The `content-type` header of the request is also checked, enabling the usage of different template formats, e.g. an XML template. + +Please note that it is not possible to override the default message for HTTP 404 errors. These errors indicate that the requested resource could not be found (e.g. the requested URL does not exist). + +### Use Cases + +#### JSON Request + +When a HTTP 500 error occurs, and the request is a JSON request, Tyk will follow this logic: + +* If `templates/error_500.json` exists, this template will be used. +* Otherwise, Tyk will use `templates/error.json`. + +#### XML Request + +When a HTTP 500 error occurs, and the request is a XML request, Tyk will follow this logic: + +* If `templates/error_500.xml` exists, this template will be used. +* If no specific template exists for this HTTP code, `templates/error.xml` will be used. +* If `error.xml` doesn't exist, `templates/error.json` will be used. + +#### Removing the X-Generator Header + +In case of an error, the Tyk Gateway adds the following fixed header and value: `X-Generator: tyk.io` +Please note that for `404 Not found` errors, Tyk will not return this header from a security perspective. To mitigate this issue, in case you want to better understand your clients and provide you, the manager of the platform, with error information, you can set `track_404_logs` to `true` in your `tyk.conf` which will then produce error logs showing the resources that were requested and not found. + +If you don't want to return our default X-Generator header (set to tyk.io) in your templates, set `hide_generator_header` to `true` in your `tyk.conf` file diff --git a/api-management/graphql.mdx b/api-management/graphql.mdx new file mode 100644 index 00000000..c111f187 --- /dev/null +++ b/api-management/graphql.mdx @@ -0,0 +1,2469 @@ +--- +title: "GraphQL" +'og:description': "How to configure GraphQL" +tags: ['GraphQL', 'Federation', 'Entities', 'Grapqhl Proxy', 'Validation', 'Schema', 'Complexity Limiting', 'Persisted Queries', 'Migration Guide', 'GraphQL Playground', 'GQL Headers'] +sidebarTitle: "GraphQL" +--- + +import { ButtonLeft } from '/snippets/ButtonLeft.mdx'; + +## Overview + +Tyk has **native** GraphQL support, so it doesn’t require any external services or middleware. +It fully complies with the latest GraphQL specifications, as outlined on the [GraphQL Foundation webpage](https://spec.graphql.org/), including: + +- **[Queries](https://spec.graphql.org/October2021/#sec-Query)** – Fetching data +- **[Mutations](https://spec.graphql.org/October2021/#sec-Mutations)** – Modifying data +- **[Subscriptions](https://spec.graphql.org/October2021/#sec-Subscription)** – Real-time updates + + +### What can you do with GraphQL and Tyk? + +You can securely expose existing GraphQL APIs using our [GraphQL core functionality](/api-management/graphql#create-a-graphql-api). + +In addition to this, you can also use Tyk's integrated GraphQL engine to build a [Universal Data Graph](/api-management/data-graph#overview). The Universal Data Graph (UDG) lets you expose existing services as one single combined GraphQL API. + +See our video on getting started with GraphQL. + + + +### What is GraphQL? + +> GraphQL is a query language for APIs and a runtime for fulfilling those queries with your existing data. GraphQL provides a complete and understandable description of the data in your API, gives clients the power to ask for exactly what they need and nothing more, makes it easier to evolve APIs over time, and enables powerful developer tools. + +source: [GraphQL Foundation website](https://graphql.org/) + +### Why would you want to use GraphQL? + +Since this is the documentation section, we won't get into a debate about GraphQL vs REST. The main benefits of using GraphQL are: +* **Reduced network traffic** One of the biggest benefits of GraphQL is that it allows clients to specify exactly what data they need. This means that you can avoid sending unnecessary data over the network, which can help reduce the amount of traffic and improve the performance of your application. +* **Flexibility** GraphQL is very flexible and can be used with many different programming languages and frameworks. It can also be used to retrieve data from multiple sources, such as databases, APIs, and even third-party services. +* **Simplified data fetching** With GraphQL, you can fetch all the data you need with a single request. This is because GraphQL allows you to specify exactly what data you need and how it should be structured, which can simplify the process of fetching data and reduce the amount of code you need to write. +* **Easy maintenance** Because GraphQL allows you to define a schema for your data, it can be easier to maintain and evolve your API over time. This is because changes to the schema can be made without breaking existing clients, as long as the changes are backward compatible. +* **Strong typing** GraphQL has a strong type system that allows you to define the shape of your data and ensure that the data you receive is of the correct type. This can help catch errors early on and make your code more reliable. +* **Better developer experience for certain use cases** Examples of those use cases mostly mentioned by developers are: APIs with multiple consumers that have very different requirements, public APIs with large groups of unknown users (like Shopify of Github), rapidly evolving APIs, backends for mobile applications, aggregating data from multiple microservices and development of data-driven products. + +Our team has also published some blog posts that go deeper into GraphQL discussions. You can check some of them here: +* [How Airbnb, Shopify, GitHub and more are winning with GraphQL](https://tyk.io/blog/how-airbnb-shopify-github-and-more-are-winning-with-graphql-and-why-you-may-need-it-too/) +* [Who is Tyk GraphQL functionality for](https://tyk.io/blog/using-tyks-new-graphql-functionality-whos-it-for-and-what-does-it-do/) +* [GraphQL: Performance is no longer a trade-off](https://tyk.io/blog/graphql-performance-is-no-longer-a-trade-off/) + +## Create a GraphQL API + +GraphQL API can be created in Tyk using: +* Tyk Dashboard UI +* Tyk Dashboard API +* Tyk Gateway API - for OSS users + +The process is very similar to [HTTP API creation](/api-management/gateway-config-managing-classic#create-an-api) with a few additional steps to cover GraphQL-specific functionalities. + +### Via Tyk Dashboard UI + +#### Prerequisites + +In order to complete the next steps, you need to have [Tyk Self Managed installed](/tyk-self-managed/install). You can also create a 5-week trial account in Tyk Cloud. + + + +#### Steps for Configuration + +1. **Select "APIs" from the "System Management" section** + + API Menu + +2. **Click "ADD NEW API"** + + Add API button location + +3. **Set up the Base Configuration for your API** + + Create GQL API + + - From the **Overview** section, add your **API Name** and your API **Type** (In this case it's GraphQL). + - From the **Details** section, add your **Target URL**. This will set the upstream origin that hosts the service you want to proxy to. As an example, you can use [https://countries.trevorblades.com/](https://countries.trevorblades.com/). + - In case your upstream GQL service is protected, tick the box next to **Upstream Protected** and provide authorization details, so that Tyk can introspect the GraphQL service. You can provide authorization details as a set of headers or a certificate. [Introspection](/api-management/graphql#introspection) of your upstream service is important for Tyk to correctly work with your GraphQL. + - If you would like to persist authorization information for future use you can tick the **Persist headers for future use** box. That way, if the upstream GQL schema changes in the future, you will be able to update it easily in Tyk. + - Click **Configure API** when you have finished + +4. **Set up the Authentication for your API** + + From the **Authentication** section: + + Authentication + + You have the following options: + + - **Authentication mode**: This is the security method to use with your API. First, you can set it to `Open(Keyless)`, but that option is not advised for production APIs. See [Client Authentication](/api-management/client-authentication) for more details on securing your API. + - **Strip Authorization Data**: Select this option to strip any authorization data from your API requests. + - **Auth Key Header Name**: The header name that will hold the token on inbound requests. The default for this is `Authorization`. + - **Allow Query Parameter As Well As Header**: Set this option to enable checking the query parameter as well as the header for an auth token. **This is a setting that might be important if your GQL includes subscription operations**. + - **Use Cookie Value**: It is possible to use a cookie value as well as the other two token locations. + - **Enable client certificate**: Select this to use Mutual TLS. See [Mutual TLS](/basic-config-and-security/security/mutual-tls/client-mtls#why-use-mutual-tls) for details on implementing mutual TLS. + +5. **Save the API** + + Click **SAVE** + + Save button + + Once saved, you will be taken back to the API list, where the new API will be displayed. + + To see the URL given to your API, select the API from the list to open it again. The API URL will be displayed at the top of the editor: + + API URL location + + Your GQL API is now secured and ready to use. + +### Via Tyk Dashboard API + +#### Prerequisites + +It is possible to create GQL APIs using [Tyk Dashboard APIs](/api-management/dashboard-configuration#manage-apis-api-definition). To make things easier you can use our [Postman collection](https://www.postman.com/tyk-technologies/workspace/tyk-public-workspace/overview). + +You will need an API key for your organization and one command to create a GQL API and make it live. + +#### Steps for Configuration + +1. **Obtain your Tyk Dashboard API Access Credentials key & Dashboard URL** + + From the Tyk Dashboard, select "Users" from the "System Management" section. + Click **Edit** for your user, then scroll to the bottom of the page. Your **Tyk Dashboard API Access Credentials** key is the first entry: + + API key location + + Store your Dashboard Key, Dashboard URL & Gateway URL as environment variables so you don't need to keep typing them in: + + ```bash expandable + export DASH_KEY=db8adec7615d40db6419a2e4688678e0 + + # Locally installed dashboard + export DASH_URL=http://localhost:3000/api + + # Tyk's Cloud Dashboard + export DASH_URL=https://admin.cloud.tyk.io/api + + # Locally installed gateway + export GATEWAY_URL=http://localhost:8080 + + # Your Cloud Gateway + export GATEWAY_URL=https://YOUR_SUBDOMAIN.cloud.tyk.io + ``` + +2. **Query the `/api/apis` endpoint to see what APIs are loaded** + + ```curl + curl -H "Authorization: ${DASH_KEY}" ${DASH_URL}/apis + {"apis":[],"pages":1} + ``` + + For a fresh install, you will see that no APIs currently exist. + +3. **Create your first GQL API** + + This example API definition configures the Tyk Gateway to reverse proxy to the [https://countries.trevorblades.com/](https://countries.trevorblades.com/) public GraphQL service. + + To view the raw API definition object, you may visit: https://bit.ly/3zmviZ3 + + ```curl + curl -H "Authorization: ${DASH_KEY}" -H "Content-Type: application/json" ${DASH_URL}/apis \ + -d "$(wget -qO- https://bit.ly/3zmviZ3)" + {"Status":"OK","Message":"API created","Meta":"64270eccb1821e3a5c203d98"} + ``` + + Take note of the API ID returned in the meta above - you will need it later. + + ``` + export API_ID=64270eccb1821e3a5c203d98 + ``` + +4. **Test your new GQL API** + + ```curl + curl --location ${GATEWAY_URL}/trevorblades/ + --header 'Content-Type: application/json' + --data '{"query":"query {\n countries {\n name\n capital\n awsRegion\n }\n}","variables":{}}' + ``` + + You just sent a request to the gateway on the listen path `/trevorblades`. Using this path-based-routing, the gateway was able to identify the API the client intended to target. + + The gateway stripped the listen path and reverse-proxied the request to https://countries.trevorblades.com/ + +5. **Protect your API** + + Let's grab the API definition we created before and store the output in a file locally. + + ```curl + curl -s -H "Authorization: ${DASH_KEY}" -H "Content-Type: application/json" ${DASH_URL}/apis/${API_ID} | python -mjson.tool > api.trevorblades.json + ``` + + We can now edit the `api.trevorblades.json` file we just created, and modify a couple of fields to enable authentication. + + Change `use_keyless` from `true` to `false`. + + Change `auth_configs.authToken.auth_header_name` to `apikey`. + + Then send a `PUT` request back to Tyk Dashboard to update its configurations. + + ```curl + curl -H "Authorization: ${DASH_KEY}" -H "Content-Type: application/json" ${DASH_URL}/apis/${API_ID} -X PUT -d "@api.trevorblades.json" + {"Status":"OK","Message":"Api updated","Meta":null} + ``` + +6. **Test protected API** + + Send request without any credentials + + ```curl + curl -I ${GATEWAY_URL}/trevorblades/ \ + --header 'Content-Type: application/json' \ + --data '{"query":"query {\n countries {\n name\n capital\n awsRegion\n }\n}","variables":{}}' + + HTTP/1.1 401 Unauthorized + Content-Type: application/json + X-Generator: tyk.io + Date: Wed, 04 Dec 2019 23:35:34 GMT + Content-Length: 46 + ``` + + Send a request with incorrect credentials + + ```curl + curl -I ${GATEWAY_URL}/trevorblades/ \ + --header 'Content-Type: application/json' \ + --data '{"query":"query {\n countries {\n name\n capital\n awsRegion\n }\n}","variables":{}}' \ + -H 'apikey: somekey' + + HTTP/1.1 403 Forbidden + Content-Type: application/json + X-Generator: tyk.io + Date: Wed, 04 Dec 2019 23:36:16 GMT + Content-Length: 57 + ``` + + Congratulations - You have just created your first keyless GQL API, and then protected it using Tyk! + +### Via Tyk Gateway API + +#### Prerequisites + +In order to complete the next steps, you need to have the [Tyk OSS](/tyk-oss-gateway) installed. + + + +#### Creation Methods + +With Tyk OSS, it is possible to create GQL APIs using Tyk's Gateway API or to generate a file with the same object and store it in the `/apps` folder of the Tyk Gateway installation folder. This is demonstrated [in the file-based configuration section](/api-management/manage-apis/deploy-apis/deploy-apis-overview#file-based-configuration). + + +#### Steps for Configuration + + + +A generated API ID will be added to the Tyk API definition if it's not provided while creating a GQL API with Tyk Gateway API. + + + +See our video for adding an API to the Open Source Gateway via the Gateway API and Postman: + + + +You can also use our [Postman collection](https://www.postman.com/tyk-technologies/workspace/tyk-public-workspace/overview) to make things easier. + +In order to use the Gateway API you will need an API key for your Gateway and one command to create the API and make it live. + +1. **Make sure you know your API secret** + + Your Tyk Gateway API secret is stored in your `tyk.conf` file, the property is called `secret`, you will need to use this as a header called `x-tyk-authorization` to make calls to the Gateway API. + +2. **Create a GQL API** + + To create a GQL API, let's send a definition to the `apis` endpoint, which will return the status and version of your Gateway. Change the `x-tyk-authorization` value and `curl` domain name and port to be the correct values for your environment. + + This example API definition configures the Tyk Gateway to reverse proxy to the [https://countries.trevorblades.com/](https://countries.trevorblades.com/) public GraphQL service. + + To view the raw API definition object, you may visit: https://bit.ly/3nt8KDa + + ```curl + curl --location --request POST 'http://{your-tyk-host}:{port}/tyk/apis' \ + --header 'Content-Type: application/json' \ + --header 'Accept: application/json' \ + --header 'X-Tyk-Authorization: {your-secret}' \ + --data "$(wget -qO- https://bit.ly/3nt8KDa)" + ``` + + If the command succeeds, you will see: + ```json + { + "key": "trevorblades", + "status": "ok", + "action": "added" + } + ``` + + **What did we just do?** + + We just sent an API definition to the Tyk `/apis` endpoint. API definitions are discussed in detail in the API section of this documentation. These objects encapsulate all of the settings for an API within Tyk Gateway. + + + +Notice that when creating a GQL API you need to include your GQL service schema in the API definition. Tyk Gateway doesn't have the capacity to introspect your GQL service on its own. + +Including the correct schema allows Tyk Gateway to validate incoming requests against it. More on validation can be found [here](/api-management/graphql#validation) + + + + **Restart or hot reload** + + After generating the file, you must either restart the Gateway or initiate a hot reload through an API call to the gateway, as outlined below: + ```curl + curl -H "x-tyk-authorization: {your-secret}" -s http://{your-tyk-host}:{port}/tyk/reload/group + ``` + + This command will hot-reload your API Gateway(s) and the new GQL API will be loaded, if you take a look at the output of the Gateway (or the logs), you will see that it should have loaded [Trevorblades API](https://countries.trevorblades.com/) on `/trevorblades/`. + + Your GraphQL API is now ready to use. We recommend securing any GraphQL API before publishing it. + + Check the following docs for more on GraphQL-specific security options: + * [Field based permissions](/api-management/graphql#field-based-permissions) + * [Complexity limiting](/api-management/graphql#complexity-limiting-1) + * [Introspection](/api-management/graphql#introspection) + +## GraphQL Proxy Only + +### What is GraphQL Proxy Only + +GraphQL Proxy Only is a GraphQL API with a single data source and a read-only schema. The schema is automatically loaded from the GraphQL upstream, which must support introspection queries. +Like other APIs, the GraphQL API supports policies, but with more advanced settings. + +### Creating a GraphQL API via the Dashboard UI + +1. Log in to the Dashboard and go to APIs > Add New API > GraphQL. + +Creating GraphQL Proxy Only API + +2. Choose a name for your API and provide an upstream URL + + + + + In case your upstream URL is protected, select **Upstream Protected** and provide authorization details (either Header or Certificate information). + + + +3. In this case, the upstream is protected with Basic Authentication, so we add an Authorization header. + + + + + **Persist headers for future use** checkbox is selected. That way, you will not need to provide the auth headers anymore as they will be persisted in the API definition. + + + +Adding Auth Header for GraphQL Proxy Only API + + +4. Once done, click **Configure API**, and the Dashboard API designer will show up. + +5. Configure your API and click **save**, Your API will now be saved. + +### Managing GQL Schema + +There can be a need to update/sync the schema on your GraphQL API, say when the schema on the upstream is updated. +The Dashboard UI can show the last time your API schema was synced with the upstream schema. + +schema last updated screenshot + +If you click the **Get latest version**, the gateway will make an introspection query to your upstream to fetch the schema. +You need to click **Update** on the top right button, to update your API. + + + +If you upstream is protected, you will need to provide an Authorization Header. In the Dashboard go to your API > Advanced Options > Upstream Auth headers +and fill in your credentials + + + +### Policies, Keys, and Developer Portal + +#### Field-based permission + +You may want to allow different consumers access to your GraphQL API without exposing all data to them. So for example this could be a schema for a GraphQL API: +```graphql +type Query { + accounts: [Account!] +} + +type Account { + owner: String! + number: ID! + balance: Float! +} +``` + +and you don't want some associate with a certain key to access the `balance` field on type `Account`, the gateway will respond with: +```json +{ + "errors": [ + { + "message": "field: balance is restricted on type: Account" + } + ] +} +``` +Check the [Setup field-based permission](/api-management/graphql#setup-field-based-permissions-in-dashboard) section, to learn how to configure them. + + +#### Complexity Limiting + +The complexity of a GraphQL query is about its depth. checkout this query: +```graphql +{ + continents { + countries { + continent { + countries { + continent { + countries { + name + } + } + } + } + } + } +} +``` expandable + +The above query has a depth of seven since the nested queries are seven. + +Tyk offers a solution to limit the depth of a query. +Check out [this link](/api-management/graphql#query-depth-limit) on how to set query depth. + +#### Developer Portal + +As of Tyk v3.0.0, you can now publish GraphQL APIs to the Tyk Developer Portal. +[This section](/tyk-developer-portal/tyk-portal-classic/graphql) will show how you can expose a GraphQL API to the developer portal. + +## Introspection + +### Overview + +A GraphQL server can provide information about its schema. This functionality is called **introspection** and is achievable by sending an **introspection query** to the GraphQL server. + +If **introspection** is a completely new concept for you, browse through the official [GraphQL Specification](https://spec.graphql.org/October2021/#sec-Introspection) published by the GrapQL Foundation to find out more. + +When [creating a GraphQL proxy](/api-management/graphql#create-a-graphql-api) in Tyk Dashboard an introspection query is used to fetch the schema from the GraphQL upstream and display it in the schema tab. + + + +When using a GraphQL proxy the introspection query is always sent to the GraphQL upstream. This means that changes in the Tyk schema won't be reflected in the introspection response. You should keep the schemas synchronised to avoid confusion. + + + +#### Introspection for protected upstreams + +When you are creating a GQL API using Tyk Dashboard and your target GQL API is protected, you need to provide authorization details, so that Tyk Gateway can obtain your schema. + +In the *Create new API* screen you have to tick the **Upstream Protected** option under your Upstream URL. + + Upstream protected + + - From the **Upstream protected by** section choose the right option for your case: Headers or Certificate. + - Choosing **Headers** will allow you to add multiple key/value pairs in *Introsopection headers* section. + - You can also **Persist headers for future use** by ticking that option. This will save information you provided in case in the future your schema changes and you need to sync it again. To understand better where this information will be saved, go to [GQL Headers](/api-management/graphql#graphql-apis-headers). To read more about schema syncing go [here](/api-management/graphql#syncing-gql-schema). +- Choosing **Certificate** will allow you to provide *Domain* details and either *Select certificate* or *Enter certificate ID*. + +#### Turning off introspection + +The introspection feature should primarily be used as a discovery and diagnostic tool for development purposes. + +Problems with introspection in production: + +* It may reveal sensitive information about the GraphQL API and its implementation details. +* An attacker can discover potentially malicious operations. + +You should note that if the *Authentication Mode* is *Open(Keyless)*, GraphQL introspection is enabled and it cannot be turned off. + +GraphQL introspection is enabled in Tyk by default. You can disable the introspection per key or security policy using: +* Tyk Dashboard +* Tyk Dashboard and Gateway API + + + + +First, check the general information on [how to create a security policy with Tyk](/api-management/gateway-config-managing-classic#secure-an-api) + +For GraphQL APIs the *API ACCESS* section will show additional, GQL-specific options that can be enabled. + +Disable introspection + +You can diable introspection by changing the switch position. + +Because introspection control in Tyk works on Policy and Key level, it means you can control each of your consumer's access to introspection. You can have keys that allow introspection, while also having keys that disallow it. + + + + +First, you need to learn [how to create a security policy with Tyk API](/api-management/gateway-config-managing-classic#secure-an-api) or [how to create an API Key with Tyk API](/api-management/policies#access-key-level-security). + +Once you learn how to utilize the API to create a security policy or a key, you can use the following snippet: + +```bash +{ + "access_rights": { + "{API-ID}": { + "api_id": "{API-ID}", + "api_name": "{API-NAME}", + "disable_introspection": true, + "allowed_types": [], + "restricted_types": [] + } + } +} +``` + +With this configuration, we set `true` to `disable_introspection` field. When you try to run an introspection query on your API, you will receive an error response *(403 Forbidden)*: + +```bash +{ + "error": "introspection is disabled" +} +``` expandable + + + + + + +Introspection also works for the [Universal Data Graph](/api-management/data-graph#overview). + +### Introspection Queries + +Any GraphQL API can be introspected with the right introspection query. Here's some examples on what introspection queries can look like and what information you can learn about the GraphQL service using them. + +#### Introspecting all types + +This query will respond with information about all types and queries defined in the schema. Additional information like *name*, *description* and *kind* will also be provided. + +```graphql +query { + __schema { + types { + name + description + kind + } + queryType { + fields { + name + description + } + } + } + } + +``` + +#### Introspecting single type details + +If you want to know more about a certain type in the schema, you can use the following query: + +```graphql + query { + __type(name: "{type name}") { + ...FullType + } + } + + fragment FullType on __Type { + kind + name + description + fields(includeDeprecated: true) { + name + description + args { + ...InputValue + } + type { + ...TypeRef + } + isDeprecated + deprecationReason + } + + inputFields { + ...InputValue + } + + interfaces { + ...TypeRef + } + + enumValues(includeDeprecated: true) { + name + description + isDeprecated + deprecationReason + } + + possibleTypes { + ...TypeRef + } + } + + fragment InputValue on __InputValue { + name + description + type { + ...TypeRef + } + defaultValue + } + + fragment TypeRef on __Type { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + } + } + } + } + } + } + } + } +``` + +#### Introspecting types associated with an interface + +The query to introspect a single type can be used for any type, but you might prefer a simpler response for types such as `interface`. With this query you can get a list of objects that implements a specific `interface`. + +```graphql +query { +__type(name: "{interface name}") { + name + kind + description + possibleTypes { + name + kind + description + } +} +} +``` + +#### Introspecting ENUM values + +An `enum` type defines a set of discrete values. With this query you can get a complete list of those values for a chosen `enum`. + +```graphql +query { +__type(name: "{enum name}") { + name + kind + description + enumValues { + name + description + } +} +} +``` + +#### Introspecting query definitions + +GraphQL requires queries to be defined in a special type `Query` in the schema. You can use the below introspection query to find out more about a query operations of the graph. + +```graphql + query { + __type(name: "Query") { + ...QueryType + } + } + + fragment QueryType on __Type { + fields { + name + description + type { + name + kind + } + args { + name + description + type { + name + kind + } + } + } + } +``` expandable + + + +You might find GQL APIs where the `Query` type is called `QueryRoot`. In those cases the above introspection query needs to be modified in line 2 to: `__type(name: "QueryRoot")` + + + +#### Introspecting mutation and subscription definitions + +You should use the same introsopection query as you would for `Query` type, just change the name argument to `Mutation` or `Subscription`. + +#### Full introspection + +If you prefer to introspect GraphQL all at once, you can do that by sending this query: + +```graphql + + query IntrospectionQuery { + __schema { + + queryType { name } + mutationType { name } + subscriptionType { name } + types { + ...FullType + } + directives { + name + description + + locations + args { + ...InputValue + } + } + } + } + + fragment FullType on __Type { + kind + name + description + + fields(includeDeprecated: true) { + name + description + args { + ...InputValue + } + type { + ...TypeRef + } + isDeprecated + deprecationReason + } + inputFields { + ...InputValue + } + interfaces { + ...TypeRef + } + enumValues(includeDeprecated: true) { + name + description + isDeprecated + deprecationReason + } + possibleTypes { + ...TypeRef + } + } + + fragment InputValue on __InputValue { + name + description + type { ...TypeRef } + defaultValue + + + } + + fragment TypeRef on __Type { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + ofType { + kind + name + } + } + } + } + } + } + } + } + +``` expandable + +Tyk also allows you to block introspection queries for security reasons if you wish to do so. More information on how to do that is provided [here](/api-management/graphql#turning-off-introspection). + +## Validation + +In order to prevent errors happening during request processing or sending invalid queries to the upstream Tyk supports the validation of GraphQL queries and schemas. + +### Query Validation +Tyk's native GraphQL engine supports validating GraphQL queries based on the [GraphQL Specification](https://spec.graphql.org/October2021/). + +Both the GraphQL engine in front of your existing GraphQL API as well as any Universal Data Graph you build gets protected with a validation middleware. + +This means, no invalid request will be forwarded to your upstream. +The Gateway will catch the error and return it to the client. + +### Schema Validation +A broken schema can lead to undesired behaviors of the API including queries not being processed by the GraphQL middleware. As the search for the root cause for +such a malfunction can be tedious, Tyk provides schema validation. + + + +Schema validation is only available when using the Dashboard or Dashboard API. + + + +The schema validation will prevent you from saving or updating an API with a broken schema. This includes schemas breaking the following rules: + - No duplicated operation types (Query, Mutation, Subscription) + - No duplicated type names + - No duplicated field names + - No duplicated enum values + - No usage of unknown types + +When using the [Dashboard API](/api-management/dashboard-configuration#manage-apis-api-definition) the response for a broken schema will be a *400 Bad Request* with a body containing the validation errors. For example: + +```json +{ + "Status": "Error", + "Message": "Invalid GraphQL schema", + "Meta": null, + "Errors": [ + "field 'Query.foo' can only be defined once" + ] +} +``` expandable + +## GraphQL APIs headers + +Users can set up two kinds of headers when configuring GraphQL APIs: + +- Introspection headers +- Request headers + +Both types of headers can be set in the Advanced Options tab in Tyk Dashboard. + +### Introspection headers + +Tyk Dashboard can introspect any upstream GraphQL API and download a copy of the GQL schema. That schema will be displayed in the Schema tab. + +For protected upstreams that require authorization for introspection, Tyk allows you to persist authorization headers within the GraphQL API configuration using **Introspection headers**. + +Introspection headers + +Any header key/value pair defined in **Introspection headers** will only be used while making an introspection call from Tyk Dashboard to the upstream. Those headers will not be used while proxying requests from consumers to the upstream. + +**Introspection headers** can also be configured in the raw API definition: + +```json +... +"graphql": { + "execution_mode": "proxyOnly", + "proxy": { + "auth_headers": { + "admin-auth": "token-value" + } + } +} +``` expandable + +### Request headers + +You can enrich any GraphQL request proxied through Tyk Gateway with additional information in the headers by configuring **Request headers** in the Tyk Dashboard. + +Request headers + +**Request headers** values can be defined as context variables. To know how to refer to request context variables check [this page](/api-management/traffic-transformation/request-context-variables). + +Any header key/value pair defined in **Request headers** will only be used to inject headers into requests proxied through the Gateway. It will not be used to introspect the upstream schema from Tyk Dashboard. + +**Request headers** can also be configured in the raw API definition: + +```bash +... +"graphql": { + "execution_mode": "proxyOnly", + "proxy": { + "request_headers": { + "context-vars-metadata": "$tyk_context.path", + "static-metadata": "static-value" + } + } +} +``` expandable + +## Syncing GQL Schema + +A GraphQL Proxy API maintains a copy of the upstream GraphQL schema. When the upstream schema changes, these updates need to be reflected in the proxy schema. + +To manage this, Tyk Dashboard stores the timestamp of the last schema change each time a GraphQL API is updated. This timestamp helps identify whether the schema is outdated and needs to be synced with the upstream version. You can find this information above the schema editor. + +To sync the schema, click the **Resync** button. + + + + +Syncing schemas is only available for proxy-only GraphQL APIs and **not** for UDG. + + + +Sync Schema Button + +If your upstream is protected then you need to make sure you provide Tyk with the authorization details to execute the introspection query correctly. You can add those detail while [creating GQL API](/api-management/graphql#introspection-for-protected-upstreams) or using [Introspection headers](/api-management/graphql#introspection-headers) later on. + + + +## Persisting GraphQL queries + +Tyk Gateway `4.3.0` release includes a way to expose GraphQL queries as REST endpoints. For now, this can only be configured via the raw API definition, Tyk Dashboard support is coming soon. + +### How to persist GraphQL query + +The ability to expose a GraphQL query as a REST endpoint can be enabled by adding the `persist_graphql` section of the `extended_paths` on an HTTP type in any API version you intend to use to serve as the GraphQL query to REST endpoint proxy. + +Here is a sample REST API proxy for the HTTP type API: + +```json +{ + "name": "Persisted Query API", + "api_id": "trevorblades", + "org_id": "default", + "use_keyless": true, + "enable_context_vars": true, + "definition": { + "location": "header", + "key": "x-api-version" + }, + "proxy": { + "listen_path": "/trevorblades/", + "target_url": "https://countries.trevorblades.com", + "strip_listen_path": true + } +} +``` + +The target URL should point to a GraphQL upstream although this is a REST proxy. This is important for the feature to work. + +#### Adding versions + +On its own, this isn’t particularly remarkable. To enable GraphQL to REST middleware, modify the Default version like so: + +```json +{ + "name": "Persisted Query API", + "definition": { + "location": "header", + "key": "x-api-version" + }, + ... + "version_data": { + "not_versioned": true, + "default_version": "", + "versions": { + "Default": { + "name": "Default", + "expires": "", + "paths": { + "ignored": [], + "white_list": [], + "black_list": [] + }, + "use_extended_paths": true, + "global_headers": {}, + "global_headers_remove": [], + "global_response_headers": {}, + "global_response_headers_remove": [], + "ignore_endpoint_case": false, + "global_size_limit": 0, + "override_target": "", + "extended_paths": { + "persist_graphql": [ + { + "method": "GET", + "path": "/getContinentByCode", + "operation": "query ($continentCode: ID!) {\n continent(code: $continentCode) {\n code\n name\n countries {\n name\n }\n }\n}", + "variables": { + "continentCode": "EU" + } + } + ] + } + } + } + } +} +``` expandable + +The vital part of this is the `extended_paths.persist_graphql` field. The `persist_graphql` object consists of three fields: + +`method`: The HTTP method used to access that endpoint, in this example, any GET requests to `/getContinentByCode` will be handled by the *persist graphql* middleware + +`path`: The path the middleware listens to + +`operation`: This is the GraphQL operation (`query` in this case) that is sent to the upstream. + +`variables`: A list of variables that should be included in the upstream request. + +If you run a request to your proxy, you should get a response similar to this: + +```json +{ + "data": { + "continent": { + "code": "EU", + "name": "Europe", + "countries": [ + { + "name": "Andorra" + }, + ... + ] + } + } +} +``` + +#### Dynamic variables + +We have seen support for passing static variable values via the API definition, but there will be cases where we want to extract variables from the request header or URL. More information about available request context variables in Tyk can be found [here](/api-management/traffic-transformation/request-context-variables) + +Below is an examples of using an incoming `code` header value as a variable in `persist_graphql` middleware configuration: + +```json +{ + "method": "GET", + "path": "/getCountryByCode", + "operation": "query ($countryCode: ID!) {\n country(code: $countryCode) {\n code\n name\n }\n}", + "variables": { + "countryCode": "$tyk_context.headers_Code" + } +} +``` + +Making a request to that endpoint and providing header `"code": "UK"`, should result in a response similar to this: + +```json +{ + "data": { + "country": { + "code": "UK", + "name": "United Kingdom" + } + } +} +``` + +Similarly, you can also pass variables in the request URL. Modify your `persist_graphql` block to this: + +```json +{ + "method": "GET", + "path": "/getCountryByCode/{countryCode}", + "operation": "query ($countryCode: ID!) {\n country(code: $countryCode) {\n code\n name\n }\n}", + "variables": { + "countryCode": "$path.countryCode" + } +} +``` + +If you now make a request to `/getCountryByCode/NG` you should get a result similar to this: + +```json +{ + "data": { + "country": { + "code": "NG", + "name": "Nigeria" + } + } +} +``` + +## Complexity Limiting + +Depending on the GraphQL schema an operation can cause heavy loads on the upstream by using deeply nested or resource-expensive operations. Tyk offers a solution to this issue by allowing you to control query depth and define its max value in a policy or directly on a key. + +### Deeply nested query + +Even if you have a simple GraphQL schema, that looks like this: + +```graphql +type Query { + continents: [Continent!]! +} + +type Continent { + name: String! + countries: [Country!]! +} + +type Country { + name: String! + continent: Continent! +} +``` + +There is a potential risk, that a consumer will try to send a deeply nested query, that will put a lot of load on your upstream service. An example of such query could be: + +```graphql +query { + continents { + countries { + continent { + countries { + continent { + countries { + continent { + countries { + name + } + } + } + } + } + } + } + } +} +``` + +### Query depth limit +Deeply nested queries can be limited by setting a query depth limitation. The depth of a query is defined by the highest amount of nested selection sets in a query. + +Example for a query depth of `2`: +```json +{ + continents { + name + } +} +``` + +Example for a query depth of `3`: +```json +{ + continents { + countries { + name + } + } +} +``` + +When a GraphQL operation exceeds the query depth limit the consumer will receive an error response (*403 Forbidden*): +```json +{ + "error": "depth limit exceeded" +} +``` expandable + +#### Enable depth limits from the Dashboard + +Query depth limitation can be applied on three different levels: + +* **Key/Policy global limits and quota section. (`Global Limits and Quota`)** The query depth value will be applied on all APIs attached on a Key/Policy. + 1. *Optional:* Configure a Policy from **System Management > Policies > Add Policy**. + 2. From **System Management > Keys > Add Key** select a policy or configure directly for the key. + 3. Select your GraphQL API (marked as *GraphQL*). (if Policy is not applied on Key) + 4. Change the value for **Query depth**, from `Global Limits and Quota` by unchecking the *Unlimited query depth* checkmark and insert the maximum allowed query depth. + +query-depth-limit + +* **API limits and quota. (`Set per API Limits and Quota`)** This value will overwrite any value registered for query depth limitation on global Key/Policy level, and will be applied on all fields for Query and Mutation types defined within the API schema. + 1. *Optional:* Configure a Policy from **System Management > Policies > Add Policy**. + 2. From **System Management > Keys > Add Key** select a policy or configure directly for the key. + 3. Select your GraphQL API (marked as *GraphQL*). (if Policy is not applied on Key) + 4. Enable `Set per API Limits and Quota` section. + 5. Change the value for **Query depth**, from API level, by unchecking the *Unlimited query depth* checkmark and insert the maximum allowed query depth + +query-depth-limit + +* **API per query depth limit. (`Set per query depth limits`)** By setting a query depth limit value on a specific Query/Mutation type field, will take highest priority and all values set on first 2 steps will be overwritten. + 1. *Optional:* Configure a Policy from **System Management > Policies > Add Policy**. + 2. From **System Management > Keys > Add Key** select a policy or configure directly for the key. + 3. Select your GraphQL API (marked as *GraphQL*). (if Policy is not applied on Key) + 4. Enable `Set per query depth limits` section. + 5. Add as many queries you want to apply depth limitation on. + +query-depth-limit + + +#### Enable depth limits using Tyk APIs + +You can set the same query depth limits using the Tyk Gateway API (for open-source users) or Tyk Dashboard API. To make it easier we have [Postman collections](https://www.postman.com/tyk-technologies/workspace/tyk-public-workspace/overview) you can use. + +**Global query depth limit for Key/Policy** + +In the key/policy json you need to make sure this section has your desired `max_query_depth` set: + +```yaml +{... + "rate": 1000, + "per": 60, + "max_query_depth": 5 +...} +``` + +**Per API depth limits** + +In the key/policy json you need to make sure that this section is set correctly: + +```yaml +{ + ... + "access_rights_array": [ + { + "api_name": "trevorblades", + "api_id": "68496692ef5a4cb35a2eac907ec1c1d5", + "versions": [ + "Default" + ], + "allowed_urls": [], + "restricted_types": [], + "allowed_types": [], + "disable_introspection": false, + "limit": { + "rate": 1000, + "per": 60, + "throttle_interval": -1, + "throttle_retry_limit": -1, + "max_query_depth": 3, + "quota_max": -1, + "quota_renews": 0, + "quota_remaining": 0, + "quota_renewal_rate": -1, + "set_by_policy": false + }, + "field_access_rights": [], + "allowance_scope": "" + } + ] + ... +} +``` + +**API per query depth limits** + +If you have more than one query in your schema and you want to set different depth limits for each of those, Tyk also allows you to do that. In this case you need to make sure, that `field_access_rights` per API are set correctly: + +```yaml +{ + ... + "access_rights_array": [ + { + "api_name": "trevorblades", + "api_id": "68496692ef5a4cb35a2eac907ec1c1d5", + "versions": [ + "Default" + ], + "allowed_urls": [], + "restricted_types": [], + "allowed_types": [], + "disable_introspection": false, + "limit": null, + "field_access_rights": [ + { + "type_name": "Query", + "field_name": "continents", + "limits": { + "max_query_depth": 3 + } + }, + { + "type_name": "Query", + "field_name": "countries", + "limits": { + "max_query_depth": 5 + } + } + ], + "allowance_scope":"" + } + ] + ... +} +``` + + + +Setting the depth limit to `-1` in any of the above examples will allow *Unlimited* query depth for your consumers. + + + +## Field Based Permissions + +You may want to allow different consumers access to your GraphQL API without exposing all data to them. So for example this could be a schema for a GraphQL API: + +```graphql +type Query { + accounts: [Account!] +} + +type Account { + owner: String! + number: ID! + balance: Float! +} +``` + +For one type of consumer, it will be fine to query all data the schema exposes, while for another type of consumer it should not be allowed to retrieve the `balance` for example. + +Field access can be restricted by setting up *field based permissions* in a policy or directly on a key. + +When a field is restricted and used in a GraphQL operation, the consumer will receive an error response (*400 Bad Request*): + +```yaml +{ + "errors": [ + { + "message": "field: balance is restricted on type: Account" + } + ] +} +``` +### Field based permissions with the list of allowed types +Field access can be restricted by setting up an allowed types list in a policy or directly on a key. If new fields are added to the GraphQL schema, you don't need to update the field-based permissions. This is because the fields that are not in the list of allowed types are automatically access-restricted. + +First, you need to learn [how to create a security policy with the API](/api-management/gateway-config-managing-classic#secure-an-api) or [how to create an API Key with the API](/api-management/gateway-config-managing-classic#secure-an-api). + +Once you learn how to utilize the API to create a security policy or key, you can use the following snippet: + +```yaml +{ + "access_rights": { + "{API-ID}": { + "api_id": "{API-ID}", + "api_name": "{API-NAME}", + "allowed_types": [ + { + "name": "Query", + "fields": ["accounts"] + }, + { + "name": "Account", + "fields": ["owner"] + } + ] + } + } +} +``` +With this configuration, a consumer can only access the field called the `owner`. When any other fields are used in a GraphQL operation, the consumer will receive an error response *(400 Bad Request)*: + +```yaml +{ + "errors": [ + { + "message": "field: balance is restricted on type: Account" + } + ] +} +``` +It's important to note that once you set a list of allowed types, Tyk will use this list to control access rights and disable the list of restricted types. The same behavior will occur if an asterisk operator is used to control access. + +### Allow or restrict all fields with the asterisk operator + +You can allow or restrict all fields of a type by using an asterisk (*) operator. Any new fields of that type will be allowed or blocked by default. For example: + +```yaml +{ + "access_rights": { + "{API-ID}": { + "api_id": "{API-ID}", + "api_name": "{API-NAME}", + "allowed_types": [ + { + "name": "Query", + "fields": ["*"] + }, + { + "name": "Account", + "fields": ["*"] + } + ] + } + } +} +``` +With this configuration, the consumers are allowed to access all current and future fields of the `Query` and `Account` types. Please note that the asterisk operator does not work recursively. For example, in the example below, the asterisk operator only allows access to fields of the `Query` type. Fields of the `Account` type remain restricted. + +```yaml +{ + "access_rights": { + "{API-ID}": { + "api_id": "{API-ID}", + "api_name": "{API-NAME}", + "allowed_types": [ + { + "name": "Query", + "fields": ["*"] + } + ] + } + } +} +``` +The asterisk operator also works for the list of restricted types: + +```yaml +{ + "access_rights": { + "{API-ID}": { + "api_id": "{API-ID}", + "api_name": "{API-NAME}", + "restricted_types": [ + { + "name": "Query", + "fields": ["accounts"] + }, + { + "name": "Account", + "fields": ["*"] + } + ] + } + } +} +``` expandable + +The configuration above restricts access to all fields of the `Account` type. + +Please note that the list of allowed types overrides the list of restricted types. + + + +### Setup field based permissions in Dashboard + +Restricted and allowed types and fields can also be set up via Tyk Dashboard. + +1. *Optional:* Configure a Policy from **System Management > Policies > Add Policy**. +2. From **System Management > Keys > Add Key** select a policy or configure directly for the key. +3. Select your GraphQL API (marked as *GraphQL*). +4. Enable either **Block list** or **Allow list**. By default, both are disabled. It's not possible to have both enabled at the same time - enabling one switch automatically disables the other. + +#### Block list + +By default all *Types* and *Fields* will be unchecked. By checking a *Type* or *Field* you will disallow to use it for any GraphQL operation associated with the key. + +For example, the settings illustrated below would block the following: +- `code` and `countries` fields in `Continent` type. +- `latt` and `longt` fields in `Coordinates` type. + +field-based-permissions + +#### Allow list + +By default all *Types* and *Fields* will be unchecked. By checking a *Type* or *Field* you will allow it to be used for any GraphQL operation associated with the key. + +For example, the settings illustrated below would only allow the following: +- `code` field in `Continent` type. +- `code` and `name` fields in `Language` type. + +Note that the `Query` type is unchecked, which indicates that all fields in `Query` type are unchecked. Subsequently, you will not be able to run any query. + +field-based-permissions + +## GraphQL Federation + +### Overview + +#### Federation Version Support + +Tyk supports Federation v1 + +#### What is federation? + +Ease-of-use is an important factor when adopting GraphQL either as a provider or a consumer. Modern enterprises have dozens of backend services and need a way to provide a unified interface for querying them. Building a single, monolithic GraphQL service is not the best option. It leads to a lot of dependencies, over-complication and is hard to maintain. + +To remedy this, Tyk, with release 4.0 offers GraphQL federation that allows you to divide GQL implementation across multiple back-end services, while still exposing them all as a single graph for the consumers. + +GraphQL federation flowchart + +#### Subgraphs and supergraphs + +**Subgraph** is a representation of a back-end service and defines a distinct GraphQL schema. It can be queried directly as a separate service or it can be federated into a larger schema of a supergraph. + +**Supergraph** is a composition of several subgraphs that allows the execution of a query across multiple services in the backend. + +#### Subgraphs examples + +**Users** +```graphql +extend type Query { + me: User +} + +type User @key(fields: "id") { + id: ID! + username: String! +} +``` + +**Products** + +```graphql +extend type Query { + topProducts(first: Int = 5): [Product] +} + +extend type Subscription { + updatedPrice: Product! + updateProductPrice(upc: String!): Product! + stock: [Product!] +} + +type Product @key(fields: "upc") { + upc: String! + name: String! + price: Int! + inStock: Int! +} +``` + +**Reviews** + +```graphql +type Review { + body: String! + author: User! @provides(fields: "username") + product: Product! +} + +extend type User @key(fields: "id") { + id: ID! @external + username: String! @external + reviews: [Review] +} + +extend type Product @key(fields: "upc") { + upc: String! @external + reviews: [Review] +} +``` expandable + +#### Subgraph conventions + +- A subgraph can reference a type that is defined by a different subgraph. For example, the Review type defined in the last subgraph includes an `author` field with type `User`, which is defined in a different subgraph. + +- A subgraph can extend a type defined in another subgraph. For example, the Reviews subgraph extends the Product type by adding a `reviews` field to it. + +- A subgraph has to add a `@key` directive to an object’s type definition so that other subgraphs can reference or extend that type. The `@key` directive makes an object type an entity. +#### Supergraph schema + +After creating all the above subgraphs in Tyk, they can be federated in your Tyk Gateway into a single supergraph. The schema of that supergraph will look like this: + +```graphql +type Query { + topProducts(first: Int = 5): [Product] + me: User +} + +type Subscription { + updatedPrice: Product! + updateProductPrice(upc: String!): Product! + stock: [Product!] +} + +type Review { + body: String! + author: User! + product: Product! +} + +type Product { + upc: String! + name: String! + price: Int! + inStock: Int! + reviews: [Review] +} + +type User { + id: ID! + username: String! + reviews: [Review] +} +``` expandable + +#### Creating a subgraph via the Dashboard UI + +1. Log in to the Dashboard and go to APIs > Add New API > Federation > Subgraph. +Add federation subgraph + +2. Choose a name for the subgraph and provide an upstream URL. + + + + + Note + + In case your upstream URL is protected, select **Upstream Protected** and provide authorization details (either Header or Certificate information). + + + +Add upstream URL + +3. Go to Configure API and configure your subgraph just as you would any other API in Tyk. + + + + + Note + + In v4.0, subgraphs will be set to **Internal** by default. + + + +4. Once you have configured all the options, click Save. The subgraph is now visible in the list of APIs. +Subgraph API listing + +#### Creating a supergraph via the Dashboard UI +1. Log in to the Dashboard and go to APIs > Add New API > Federation > Supergraph. +Add supergraph API + +2. In the Details section, select all the subgraphs that will be included in your supergraph. +Select subgraphs + +3. Go to Configure API and configure your supergraph just as you would any other API in Tyk. +4. Once you configure all the options, click Save. The supergraph is now available in your list of APIs. +Supergraph API listing + +#### Defining Headers +In v4.0 you can define global (Supergraph) headers. Global headers are forwarded to all subgraphs that apply to the specific upstream request. + +##### Setting a Global Header + +1. After creating your supergraph, open the API in your Dashboard. +2. From the Subgraphs tab, click Global Headers. +Global Header setup for a supergraph + +3. Enter your header name and value. You can add more headers by clicking Add Headers. +Add further Global headers in a supergraph + +4. Click **Update** to save the header. +5. On the pop-up that is displayed, click Update API. +6. If you want to delete a global header, click the appropriate bin icon for it. +7. You can update your headers by repeating steps 2-5. + +### Entities + +#### Defining the base entity + +- Must be defined with the @key directive. +- The "fields" argument of the @key directive must reference a valid field that can uniquely identify the entity. +- Multiple primary keys are possible. + +An example is provided below: + +**Subgraph 1 (base entity)** + +```graphql +type MyEntity @key(fields: "id") @key(fields: "name") { + id: ID! + name: String! +} +``` expandable + +#### Extending entities + +Entities cannot be shared types (be defined in more than one single subgraph; see **Entity stubs** below). + +The base entity remains unaware of fields added through extension; only the extension itself is aware of them. + +Attempting to extend a non-entity with an extension that includes the @key directive or attempting to extend a base entity with an extension that does not include the @key directive will both result in errors. + +The primary key reference should be listed as a field with the @external directive. + +Below is an example extension for **MyEntity** (which was defined above in **Subgraph 1**): + +**Subgraph 2 (extension):** + +```graphql +extend type MyEntity @key(fields: "id") { + id: ID! @external + newField: String! +} +``` expandable + +#### Entity stubs +If one subgraph references a base entity (an entity defined in another subgraph) without adding new fields, that reference must be declared as a stub. In **federation v1**, stubs appear similar to extensions but do not add any new fields. + +An entity stub contains the minimal amount of information necessary to identify the entity (referencing exactly one of the primary keys from the base entity regardless of whether there are multiple primary keys on the base entity). + +The identifying primary key should feature the @external directive. + +For example, a stub of **MyEntity** (which was defined above in **Subgraph 1**): + +**Subgraph 3 (stub):** + +```graphql +extend type MyEntity @key(fields: "id") { + id: ID! @external +} +``` expandable + +##### What is a shared type? +Types that are identical by name and structure and feature in more than one subgraph are shared types. + +##### Can I extend a shared type? +Subgraphs are normalized before federation. This means you can extend a type if the resolution of the extension after normalization is exactly identical to the resolution of the type after normalization in other subgraphs. + +Unless the resolution of the extension in a single subgraph is exactly identical to all other subgraphs, extension is not possible. + +Here is a valid example where both subgraphs resolve to identical enums after normalization: + +**Subgraph 1:** + +```graphql +enum Example { + A, + B +} + +extend enum Example { + C +} +``` + +**Subgraph 2:** + +```graphql +enum Example { + A, + B, + C +} +``` + +Here, the enum named Example in **Subgraph 1** resolves to be identical to the enum named Example in **Subgraph 2**. + +However, if we were to include **Subgraph 3**, which does not feature the β€œC” value, the enum is no longer identical in all 3 subgraphs. Consequently, federation would fail. + +**Subgraph 3:** + +```graphql +enum Example { + A, + B +} +``` expandable + + + +### Extension Orphans + +#### What is an extension orphan? + +An extension orphan is an unresolved extension of a type after federation has completed. This will cause federation to fail and produce an error. + +#### How could an extension orphan occur? + +You may extend a type within a subgraph where the base type (the original definition of that type) is in another subgraph. This means that it is only after the creation of the supergraph that it can be determined whether the extension was valid. If the extension was invalid or was otherwise unresolved, an β€œextension orphan” would remain in the supergraph. + +For example, the type named Person does not need to be defined in **Subgraph 1**, but it must be defined in exactly one subgraph (see **Shared Types**: extension of shared types is not possible, so extending a type that is defined in multiple subgraphs will produce an error). + +**Subgraph 1** + +```graphql +extend type Person { + name: String! +} +``` expandable + +If the type named Person were not defined in exactly one subgraph, federation will fail and produce an error. + + + +## GraphQL WebSockets + +Tyk supports GraphQL via WebSockets using the protocols _graphql-transport-ws_ or _graphql-ws_ between client and Tyk Gateway. + +Before this feature can be used, WebSockets need to be enabled in the Tyk Gateway configuration. To enable it set [http_server_options.enable_websockets](/tyk-oss-gateway/configuration#http_server_optionsenable_websockets) to `true` in your `tyk.conf` file. + + + +You can find the full documentation of the _graphql-transport-ws_ protocol itself [here](https://github.com/enisdenjo/graphql-ws/tree/master). + +In order to upgrade the HTTP connection for a GraphQL API to WebSockets by using the _graphql-transport-ws_ protocol, the request should contain following headers: + +``` +Connection: Upgrade +Upgrade: websocket +Sec-WebSocket-Key: +Sec-WebSocket-Version: 13 +Sec-WebSocket-Protocol: graphql-transport-ws +``` + +**Messages** + +The connection needs to be initialised before sending Queries, Mutations, or Subscriptions via WebSockets: + +``` +{ "type": "connection_init" } +``` + +Always send unique IDs for different Queries, Mutations, or Subscriptions. + +For Queries and Mutations, the Tyk Gateway will respond with a `complete` message, including the GraphQL response inside the payload. + +```json +{ "id": "1", "type": "complete" } +``` expandable + +For Subscriptions, the Tyk Gateway will respond with a stream of `next` messages containing the GraphQL response inside the payload until the data stream ends with a `complete` message. It can happen infinitely if desired. + + + +Be aware of those behaviors: + - If no `connection_init` message is sent after 15 seconds after opening, then the connection will be closed. + - If a duplicated ID is used, the connection will be closed. + - If an invalid message type is sent, the connection will be closed. + + + +**Examples** + +**Sending queries** + +``` +{"id":"1","type":"subscribe","payload":{"query":"{ hello }"}} +``` + +**Sending mutations** + +``` +{"id":"2","type":"subscribe","payload":{"query":"mutation SavePing { savePing }"}} +``` + +**Starting and stopping Subscriptions** + +``` +{"id":"3","type":"subscribe","payload":{"query":"subscription { countdown(from:10) }" }} +``` +``` +{"id":"3","type":"complete"} +``` + + +In order to upgrade the HTTP connection for a GraphQL API to WebSockets by using the _graphql-ws_ protocol, the request should contain following headers: + +``` +Connection: Upgrade +Upgrade: websocket +Sec-WebSocket-Key: +Sec-WebSocket-Version: 13 +Sec-WebSocket-Protocol: graphql-ws +``` + +**Messages** + +The connection needs to be initialised before sending Queries, Mutations, or Subscriptions via WebSockets: + +``` +{ "type": "connection_init" } +``` + +Always send unique IDs for different Queries, Mutations, or Subscriptions. + +For Queries and Mutations, the Tyk Gateway will respond with a `complete` message, including the GraphQL response inside the payload. + +For Subscriptions, the Tyk Gateway will respond with a stream of `data` messages containing the GraphQL response inside the payload until the data stream ends with a `complete` message. It can happen infinitely if desired. + +**Examples** + +**Sending queries** + +``` +{"id":"1","type":"start","payload":{"query":"{ hello }"}} +``` + +**Sending mutations** + +``` +{"id":"2","type":"start","payload":{"query":"mutation SavePing { savePing }"}} +``` + +**Starting and stopping Subscriptions** + +``` +{"id":"3","type":"start","payload":{"query":"subscription { countdown(from:10) }" }} +``` +``` +{"id":"3","type":"stop"} +``` + + + +### Upstream connections + +For setting up upstream connections (between Tyk Gateway and Upstream) please refer to the [GraphQL Subscriptions Key Concept](/api-management/graphql#graphql-subscriptions). + + +## GraphQL Subscriptions + +Tyk **natively** supports also GraphQL subscriptions, so you can expose your full range of GQL operations using Tyk Gateway. Subscriptions support was added in `v4.0.0` in which *graphql-ws* protocol support was introduced. + +With the release of Tyk `v4.3.0` the number of supported subscription protocols has been extended. + +In Tyk subscriptions are using the [WebSocket transport](https://developer.mozilla.org/en-US/docs/Web/API/WebSockets_API) for connections between the client and Gateway. For connections between Gateway and upstream WebSockets or [SSE](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events) can be used. + +### Supported transports and protocols + +| Transport | Protocol | +| :------------ | :-------------------------------------------------------------------------------------------------------------------------- | +| WebSockets | [graphql-ws](http://github.com/apollographql/subscriptions-transport-ws) (default, no longer maintained) | +| WebSockets | [graphql-transport-ws](http://github.com/enisdenjo/graphql-ws) | +| HTTP | [Server-Sent Events (SSE)](https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events) | + +#### Setting up subscription types via API definition +Subscription types or subscription transports/protocols are set inside the graphql section of the API definition. + +Depending on whether you want to configure GraphQL proxy-only, UDG, or GraphQL Federation there are different places for the configuration option. + +The values for subscription types are the same on all API types: +- `graphql-ws` +- `graphql-transport-ws` +- `sse` (Server-Sent Events) + +##### GraphQL Proxy + +``` expandable +{ + ..., + "graphql": { + ..., + "proxy": { + ..., + "subscription_type": "graphql-ws" + } + } +} +``` + +##### Universal Data Graph + +``` expandable +{ + ..., + "graphql": { + ..., + "engine": { + ..., + "data_sources": [ + ..., + { + ..., + "subscription_type": "sse" + } + ] + } + } +} +``` + +##### Federation + +``` expandable +{ + ..., + "graphql": { + ..., + "supergraph": { + ..., + "subgraphs": [ + ..., + { + ..., + "subscription_type": "graphql-transport-ws" + } + ] + } + } +} +``` + + + +If the upstream subscription GraphQL API is protected please enable the authentication via query params to pass the header through. + + + +There is no need to enable subscriptions separately. They are supported alongside GraphQL as a standard. The only requirement for subscriptions to work is to [enable WebSockets](/api-management/graphql#graphql-websockets) in your Tyk Gateway configuration file. + +Here's a general sequence diagram showing how subscriptions in Tyk work exactly: + +Tyk Subscriptions workflow + +## GraphQL playground + +When you are creating or editing your GraphQL API, any change you make can be tested using Tyk Dashboard built-in GraphiQL Playground. + +Playground + +At the top of the Playground itself, you can switch between Dark and Light theme using the `Set theme` dropdown. + +There's also a built in `Explorer` to help with query building and a `Prettify` button that helps to make the typed out operation easier to read. + +The GraphiQL try-out playground comes with a series of features by default, which can be very useful while configuring the API: + 1. Syntax highlighting. + 2. Intelligent type ahead of fields, arguments, types, and more. + 3. Real-time error highlighting and reporting for queries and variables. + 4. Automatic query and variables completion. + 5. Automatically adds required fields to queries. + 6. Documentation explorer, search, with markdown support. + 7. Query History using local storage + 8. Run and inspect query results using any promise that resolves JSON results. 9. HTTPS or WSS not required. + 10. Supports full GraphQL Language Specification: Queries, Mutations, Subscriptions, Fragments, Unions, directives, multiple operations per query, etc + +### GraphQL Playgrounds in Tyk + +Tyk offers you two types of Playgrounds, depending on who should be authorized to use them. + +* **Playground** tab in `API Designer`, that's only accessible via Tyk Dashboard and is always enabled. You need to log into the Tyk Dashboard to be able to use it. +* **Public Playground** that you can enable for any GraphQL API and that is accessible for any consumer interacting with your GQL API. This playground will follow all security rules you set for your GQL API - authentication, authorization, etc. + + + + + The Public Playground relies on assets in the `playground` folder under [template_path](/tyk-oss-gateway/configuration#template_path) (default: `/opt/tyk-gateway/templates`). + If you change this path, be sure to copy the `playground` folder to the new location to preserve functionality. + + + +#### Enabling Public GraphQL Playground + + + + +To enable a Public GraphQL Playground for one of your GQL APIs follow these few simple steps: + +1. Navigate to `Core Settings` tab in `API designer` +2. Change the setting in `Enable API Playground` section. +3. Provide `Playground path`. By default, this path is set to `/playground` but you can change it. + +Headers + +Your `Public Playground` will be available at `http://{API-URL}/playground`. + + + + +To enable Public GraphQL Playground using just Tyk API definition, you need to set the following: + +```bash expandable +... +"graphql": { + "playground": { + "enabled": true, + "path": "/playground" + } + } +... +``` + +You can choose yourself the `path` name. + +Your `Public Playground` will be available at `http://{API-URL}/playground`. + + + + +#### Query variables + +You can pass query variables in two different ways, both are fully supported in Tyk Dashboard. + +##### Using inline arguments in GraphiQL Playground + +A query or mutation string in this case, would be written like in the example below and there would be no other requirements for executing an operation like this: + +```graphql expandable +mutation createUser { + createUser(input: { + username: "test", + email: "test@test.cz", + phone: "479332973", + firstName: "David", + lastName: "Test" + }) { + user { + id + username + email + phone + firstName + lastName + } + } +} +``` + +##### Using query variables in GraphiQL Playground + +For complex sets of variables, you might want to split the above example into two parts: GQL operation and variables. + +The operation itself would change to: + +```graphql expandable +mutation createUser($input: CreateUserInput!) { + createUser(input: $input) { + user { + id + username + email + phone + firstName + lastName + } + } +} +``` + +The values for variables would need be provided in the `Query variables` section of the Playground like this: + +```graphql expandable +{ + "input": { + "username": "test", + "email": "test@test.cz", + "phone": "479332973", + "firstName": "David", + "lastName": "Test" + } +} +``` + +#### Headers + +Debugging a GraphQL API might require additional headers to be passed to the requests while playing with the GraphiQL interface (i.e. `Authorization` header in case of Authentication Token protection over the API). This can be done using the dedicated headers tab in the Graphiql IDE. + +Headers + +You can also [forward headers](/api-management/graphql#graphql-apis-headers) from your client request to the upstream data sources. + + +#### Logs + + + +GraphQL request logs described below are **only available in Tyk Dashboard**. + + + +Besides the results displayed in the GraphiQL playground, Tyk also provides you with a full list of logs of the triggered request, which can help a lot when debugging the API functionality. + +Logs + +The Request Logs can be seen under the playground itself. When no logs are present, there will be no option to expand the logs, and the filter buttons (top right) will be disabled: + +Logs Bar + +After creating and sending a query, the logs will automatically expand, and the filter buttons will display the number of logs for its respective level (category). + +Logs table + +##### Contents of the logs + +There are four levels (categories) of logs: `Info`, `Debug`, `Warning`, and `Error`, and each log belongs to one of these levels. + +The first column of the table displays the color-coded `β€œlevel”` property of the log. A log should never be absent of a level. The second column displays the log `β€œmsg”` (message) property, if any. The third column displays the `β€œmw” `(middleware) property, if any. + +##### Expansion/collapse of Request Logs + +The Request Logs can be expanded or collapsed, using the chevron on the left side to toggle these states. + +##### Filter buttons and states + +Filter buttons have two states: active and inactive; the default of which is active. A solid background color of the button indicates that a filter is active. + +In the below picture, the `info` and `error` filters buttons are both active. If there are no logs for a particular level of log, the button will appear as a gray and disabled, as shown by the `Warning` filter button. + +Logs navigation + +Here's an example where there is at least one log, but all the filter buttons are in the inactive state. If the cursor (not shown) hovers over an inactive filter button, the button background will change to solid, and the tooltip will display `β€œShow”`. + +If all filter buttons are inactive, a message asking whether the user would like to reset all filters will display. Clicking this text will activate all available filters. + +Logs empty + +## Migrating to 3.2 + +As of 3.2 GraphQL schema for Tyk API definitions (i.e `api_definition.graphql`) changed significantly, hence GraphQL API definitions created in previous beta versions are not supported via the UI and need to go through a manual migration. + + + +Before you continue, we strongly advise to simply create a new API and avoid migration of the API definition. You'll achieve results faster and can avoid typos and errors that happens with the manual migration. + + + + + +Old API definitions will continue to work for the Tyk Gateway + + + + +### The changes +- To improve performance now a single Data Source can be used to link to multiple fields instead of having an independent data source for every field hence `graphql.type_field_configurations` is now obsolete and new data sources can be defined under `graphql.engine.data_sources` (see example below). + +- Data Source kind are `REST` or `GraphQL` regardless of your API being internal or not. + +- In case of internal APIs that are accessed via `tyk://`scheme, the `graphql.engine.data_sources[n].internal` property is set to true. + +- Each dataSources needs to be defined with a unique name `graphql.engine.data_sources[n].name`. + +- Each field connected to the data source is expected to be configured for mapping under `graphql.engine.field_configs` regardless of it requiring mapping or not. + +- It is important that all new GraphQL APIs have the version `graphql.version` property set to `2`. + +### Examples + +#### Old Data Source Config + +```json expandable +"type_field_configurations": [ + { + "type_name": "Query", + "field_name": "pet", + "mapping": { + "disabled": true, + "path": "" + }, + "data_source": { + "kind": "HTTPJSONDataSource", + "data_source_config": { + "url": "https://petstore.swagger.io/v2/pet/{{.arguments.id}}", + "method": "GET", + "body": "", + "headers": [], + "default_type_name": "Pet", + "status_code_type_name_mappings": [ + { + "status_code": 200, + "type_name": "" + } + ] + } + } + }, + { + "type_name": "Query", + "field_name": "countries", + "mapping": { + "disabled": false, + "path": "countries" + }, + "data_source": { + "kind": "GraphQLDataSource", + "data_source_config": { + "url": "https://countries.trevorblades.com", + "method": "POST" + } + } + }, +] +``` + +#### New Data Source Config + +```json expandable +"engine": { + "field_configs": [ + { + "type_name": "Query", + "field_name": "pet", + "disable_default_mapping": true, + "path": [ + "" + ] + }, + { + "type_name": "Query", + "field_name": "countries", + "disable_default_mapping": false, + "path": [ + "countries" + ] + }, + ], + "data_sources": [ + { + "kind": "REST", + "name": "PetStore Data Source", + "internal": false, + "root_fields": [ + { + "type": "Query", + "fields": [ + "pet" + ] + } + ], + "config": { + "url": "https://petstore.swagger.io/v2/pet/{{.arguments.id}}", + "method": "GET", + "body": "", + "headers": {}, + } + }, + { + "kind": "GraphQL", + "name": "Countries Data Source", + "internal": false, + "root_fields": [ + { + "type": "Query", + "fields": [ + "countries" + ] + } + ], + "config": { + "url": "https://countries.trevorblades.com", + "method": "POST", + "body": "" + } + } + ] +}, +``` + +#### Example of new graphql definition + +``` json expandable +"graphql" : { + "schema": "type Mutation {\n addPet(name: String, status: String): Pet\n}\n\ntype Pet {\n id: Int\n name: String\n status: String\n}\n\ntype Query {\n default: String\n}\n", + "enabled": true, + "engine": { + "field_configs": [ + { + "type_name": "Mutation", + "field_name": "addPet", + "disable_default_mapping": true, + "path": [""] + }, + { + "type_name": "Pet", + "field_name": "id", + "disable_default_mapping": true, + "path": [""] + }, + { + "type_name": "Query", + "field_name": "default", + "disable_default_mapping": false, + "path": ["default"] + } + ], + "data_sources": [ + { + "kind": "REST", + "name": "Petstore", + "internal": false, + "root_fields": [ + { + "type": "Mutation", + "fields": ["addPet"] + } + ], + "config": { + "url": "https://petstore.swagger.io/v2/pet", + "method": "POST", + "body": "{\n \"name\": \"{{ .arguments.name }}\",\n \"status\": \"{{ .arguments.status }}\"\n}", + "headers": { + "qa": "{{ .request.header.qa }}", + "test": "data" + }, + } + }, + { + "kind": "REST", + "name": "Local Data Source", + "internal": false, + "root_fields": [ + { + "type": "Pet", + "fields": ["id"] + } + ], + "config": { + "url": "http://localhost:90909/graphql", + "method": "HEAD", + "body": "", + "headers": {}, + } + }, + { + "kind": "GraphQL", + "name": "asd", + "internal": false, + "root_fields": [ + { + "type": "Query", + "fields": ["default"] + } + ], + "config": { + "url": "http://localhost:8200/{{.arguments.id}}", + "method": "POST", + } + } + ] + }, + "execution_mode": "executionEngine", + "version": "2", + "playground": { + "enabled": false, + "path": "" + }, + "last_schema_update": "2021-02-16T15:05:27.454+05:30" +} +``` + diff --git a/api-management/logs-metrics.mdx b/api-management/logs-metrics.mdx new file mode 100644 index 00000000..80490e80 --- /dev/null +++ b/api-management/logs-metrics.mdx @@ -0,0 +1,1662 @@ +--- +title: "API Observability - Configuring Logs and Metrics" +tags: ['Metrics', 'Traces', 'Logs', 'System Logs', 'API Traffic', 'Opentelemetry', 'Datadog', 'Dynatrace', 'New Relic', 'Elastic Search', 'Jaeger', 'Monitoring', 'Observability'] +sidebarTitle: "Metrics and Logs" +--- + +## Introduction + +API observability is the process of monitoring and analyzing APIs to gain insights into developer and end-user experience and to ensure the reliability of your system. + +You can achieve API observability by using a combination of telemetry signals such as traces, metrics, and logs. Each of these signals serves a specific purpose in monitoring and troubleshooting API issues: + +### Logs + +Logs provide detailed records of events and activities within the API processing and associated services. Logs are invaluable for debugging issues and understanding what happened at a specific point in time. Here's how you can use logs for API observability: + +- **Error Identification:** Use logs to identify errors, exceptions, and warning messages that indicate issues with the API's behavior. + +- **Debugging:** Logs help developers troubleshoot and debug issues by providing detailed information about the sequence of events leading up to a problem. + +- **Security Monitoring:** Monitor logs for security-related events, such as authentication failures, access control violations and suspicious activities. + +- **Audit Trail:** Maintain an audit trail of important actions and changes to the API, including configuration changes, access control changes and data updates. + + +Tyk allows you to capture and analyze logs related to API requests and responses in the [Log Browser](/api-management/dashboard-configuration#activity-logs) . You can optionally enable detailed recording for the requests per API level or per Key level to store inbound request and outbound response data. You can [enable debug modes](/api-management/troubleshooting-debugging#capturing-detailed-logs) for selected APIs and send the detail logs to one or more Pump backend instances. + +To achieve comprehensive API observability, it is essential to integrate traces, metrics and logs into the observability tools that the team in charge of the APIs are already using. Those tools should allow users to query and visualize data, set up alerts and provide an intuitive interface for monitoring and troubleshooting API issues effectively. See also our 7 observability anti-pattern to avoid when working with APIs: [Bad API observability](https://tyk.io/blog/bad-api-observability/). + +### Metrics + +Metrics provide aggregated, quantitative data about the performance and behavior of an API over time. They offer insights into the overall health of the system. Here's how you can leverage metrics for API observability: + +- **Key Performance Indicators (KPIs):** Define and track essential metrics such as request rate, response time, error rate and resource utilization to monitor the overall health and performance of the API. + +- **Custom Metrics:** Create custom metrics that are specific to your API's functionality or business objectives. For example, track the number of successful payments processed or the number of users signed up. + +- **Threshold Alerts:** Set up alerts based on predefined thresholds for metrics to receive notifications when API performance deviates from the expected norm. + +- **Trend Analysis:** Analyze metric trends over time to identify long-term performance patterns, plan for scaling and detect anomalies. + +Tyk Dashboard offers a [traffic analytics](/api-management/dashboard-configuration#traffic-analytics) function that provides insights into API usage, traffic patterns and response times. The built-in metrics allow you to track overall API traffic, detailed API analytics including: request count, response time distribution and error rates. API usage can be tracked on a per-client (per-key) basis. + +This analysis uses the [traffic logs](/api-management/logs-metrics#api-traffic-logs) generated by Tyk Gateway from API requests and responses. Tyk Pump is used to aggregate and transfer the logs to Tyk Dashboard's [aggregate analytics storage](/api-management/dashboard-configuration#data-storage-solutions). + +You can also use Tyk Pump to export those metrics to [different back-ends](/api-management/tyk-pump#external-data-stores). Here is an example of using Tyk Pump to send [API analytics metrics to Prometheus and Grafana](https://tyk.io/blog/service-level-objectives-for-your-apis-with-tyk-prometheus-and-grafana/). + +You can also leverage the OpenTelemetry spans exported from Tyk Gateway to calculate and export [span metrics](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/connector/spanmetricsconnector/README.md) from the OpenTelemetry collector. + +### Distributed Tracing + +Distributed traces provide a detailed, end-to-end view of a single API request or transaction as it traverses through various services and components. Traces are crucial for understanding the flow of requests and identifying bottlenecks or latency issues. Here's how you can make use of traces for API observability: + +- **End-to-end request tracing:** Implement distributed tracing across your microservices architecture to track requests across different services and gather data about each service's contribution to the overall request latency. + +- **Transaction Flow:** Visualize the transaction flow by connecting traces to show how requests move through different services, including entry points (e.g., API gateway), middleware and backend services. + +- **Latency Analysis:** Analyze trace data to pinpoint which service or component is causing latency issues, allowing for quick identification and remediation of performance bottlenecks. + +- **Error Correlation:** Use traces to correlate errors across different services to understand the root cause of issues and track how errors propagate through the system. + + +Since v5.2, Tyk Gateway has supported the [OpenTelemetry](/api-management/logs-metrics#opentelemetry) standard for distributed tracing. You can configure Tyk to work with an [OpenTelemetry collector](https://opentelemetry.io/docs/collector/) or integrate it with any [observability vendor supporting OpenTelemetry](https://opentelemetry.io/ecosystem/vendors/) to capture traces of API requests as they flow through Tyk Gateway and any upstream services. + +Explore our guides for [Datadog](/api-management/logs-metrics#datadog), [Dynatrace](/api-management/logs-metrics#dynatrace), [Jaeger](/api-management/logs-metrics#using-docker) and [New Relic](/api-management/logs-metrics#new-relic) for further info on how to integrate with 3rd party observability vendors. + +Tyk also supports the legacy [OpenTracing](/api-management/logs-metrics#opentracing-deprecated) approach (now deprecated), but we recommend users to adopt OpenTelemetry for a comprehensive, vendor-neutral technology with wide industry support. + +## Logging + +Tyk Gateway generates two different types of logs for various operational aspects: + +- **System logs** capture internal gateway events, typically used for monitoring and debugging. +- **API traffic logs**, also known as transaction logs, record details of every request and response handled by the gateway and are stored in Redis. They are typically processed by Tyk Pump to create aggregated data that are then transferred to persistent storage. Tyk Pump can also be used to transfer the raw logs to 3rd Party analysis tools. + +While system logs focus on the gateway's internal operations and errors, API traffic logs provide insights into API usage, security events, and performance trends. Logging verbosity and format can be customized to suit different operational needs. + +### System Logs + +Tyk will log **system events** to `stderr` and `stdout`. + +In a typical installation, these will be handled or redirected by the service manager running the process, and depending on the Linux distribution, will either be output to `/var/log/` or `/var/log/upstart`. + +Tyk will try to output structured logs, and so will include context data around request errors where possible. + +[Custom logging event handlers](/api-management/gateway-events#logging-api-events-1) can be registered against **Gateway events** to customise the logs that are generated for those events. + +When contacting support, you may be asked to change the logging level as part of the support handling process. See [Support Information](/api-management/troubleshooting-debugging#support-information) for more details. + +#### Log verbosity + +Tyk can generate system logs at four levels of verbosity: +- `error` is the most minimal level of logging, reporting only errors +- `warn` will log warnings and errors +- `info` logs errors, warnings and some additional information and is the default logging level +- `debug` generates a high volume of logs for maximum visibility of what Tyk is doing when you need to debug an issue + + + + + Debug log level generates a significant volume of data and is not recommended except when debugging. You can enable Debug mode reporting by adding the `--debug` flag to the process run command. + + + +You can set the logging verbosity for each Tyk Component using the appropriate `log_level` setting in its configuration file (or the equivalent environment variable). Note that there is no independent log level setting for Tyk Dashboard. + +| Tyk component | Config option | Environment variable | Default value if unset | +| :---------------- | :--------------- | :---------------------- | :-------------------------- | +| All components (except EDP) | | `TYK_LOGLEVEL` | `info` | +| [Tyk Gateway](/tyk-oss-gateway/configuration#log_level) | `log_level` | `TYK_GW_LOGLEVEL` | `info` | +| [Tyk Pump](/tyk-pump/tyk-pump-configuration/tyk-pump-environment-variables#log_level) | `log_level` | `TYK_PMP_LOGLEVEL` | `info` | +| [Tyk MDCB](/tyk-multi-data-centre/mdcb-configuration-options#log_level) | `log_level` | `TYK_MDCB_LOGLEVEL` | `info` | +| [Tyk Enterprise Developer Portal](/product-stack/tyk-enterprise-developer-portal/deploy/configuration#portal_log_level) | `logLevel` | `PORTAL_LOG_LEVEL` | `info` | + +For example, setting [TYK_GW_LOGLEVEL](/tyk-oss-gateway/configuration#log_level) environment variable to `debug` will enable verbose debug for the Gateway. + +Tyk support can advise you which level of verbosity to use for your deployment. + +#### Log format (only available for the Gateway) + +As of Tyk Gateway `v5.6.0`, you can control the format in which logs will be generated - either `default` or `json` - using the `TYK_LOGFORMAT` environment variable. As a general performance tip, the `json` output format incurs less memory allocation overhead than the `default` format. For optimal performance, it's recommended to configure logging in the JSON format. + +This is an example of the `default` logging format: +``` +time="Sep 05 09:04:12" level=info msg="Tyk API Gateway v5.6.0" prefix=main +``` + +And an example of `json` logging format: +```json +{"level":"info","msg":"Tyk API Gateway v5.6.0","prefix":"main","time":"2024-09-05T09:01:23-04:00"} +``` + +#### Exporting Logs to Third-Party Tools + +Tyk can be configured to send log data to a range of 3rd party tools for aggregation and analysis. + +The following targets are supported: +- [Sentry](#sentry) +- [Logstash](#logstash) +- [Graylog](#graylog) +- [Syslog](#syslog) + +##### Sentry + +To enable Sentry as a log aggregator, update these settings in both your `tyk.conf` and your `tyk_analytics.conf`: + +* `use_sentry`: Set this to `true` to enable the Sentry logger, you must specify a Sentry DSN under `sentry_code`. + +* `sentry_code`: The Sentry-assigned DSN (a kind of URL endpoint) that Tyk can send log data to. + +##### Logstash + +To enable Logstash as a log aggregator, update these settings in your `tyk.conf`: + +* `use_logstash`: Set this to `true` to enable the Logstash logger. + +* `logstash_transport`: The Logstash transport to use, should be `"tcp"`. + +* `logstash_network_addr`: Set to the Logstash client network address, should be in the form of `hostname:port`. + +##### Graylog + +To enable Graylog as a log aggregator, update these settings in your `tyk.conf`: + +* `use_graylog`: Set this to `true` to enable the Graylog logger. + +* `graylog_network_addr`: The Graylog client address in the form of `:`. + +##### Syslog + +To enable Syslog as a log aggregator, update these settings in your `tyk.conf`: + +* `use_syslog`: Set this to `true` to enable the Syslog logger. + +* `syslog_transport`: The Syslog transport to use, should be `"udp"` or empty. + +* `syslog_network_addr`: Set to the Syslog client network address, should be in the form of `hostname:port` + +### API Traffic Logs + +When a client makes a request to the Tyk Gateway, the details of the request and response are captured and stored in a temporary Redis list. In Tyk these transaction logs are also referred to as traffic analytics or simply analytics. This list is read (and then flushed) every 10 seconds by the [Tyk Pump](/tyk-pump). + +The Pump processes the records that it has read from Redis and forwards them to the required data sinks (e.g. databases or other tools) using the pumps configured in your system. You can set up multiple pumps and configure them to send different data to different sinks. The Mongo Aggregate and SQL Aggregate pumps perform aggregation of the raw analytics records before storing the aggregated statistics in the MongoDB or SQL database respectively. + +#### When to use API Traffic Logging + +1. **API usage trends** + + Monitoring the usage of your APIs is a key functionality provided by any API Management product. Traffic analytics give you visibility of specific and aggregated accesses to your services which you can monitor trends over time. You can identify popular and underused services which can assist with, for example, determining the demand profile for your services and thus appropriate sizing of the upstream capacity. + +2. **Security monitoring** + + Tracking requests made to security-critical endpoints, like those used for authentication or authorization, can help in identifying and mitigating potential security threats. Monitoring these endpoints for unusual activity patterns is a proactive security measure. + +3. **Development and testing** + + Enabling tracking during the development and testing phases can provide detailed insights into the API's behavior, facilitating bug identification and performance optimization. Adjustments to tracking settings can be made as the API transitions to production based on operational requirements. + +#### How API Traffic Logging Works + +API traffic logging must be enabled at the Gateway level in the startup configuration using the [enable_analytics](/tyk-oss-gateway/configuration#enable_analytics) field (or by setting the equivalent environment variable `TYK_GW_ENABLEANALYTICS`). + +The transaction records generated by the Gateway are stored in Redis, from which Tyk Pump can be configured to transfer them to the desired persistent storage. When using Tyk Dashboard, the [Aggregate Pump](/api-management/tyk-pump#tyk-dashboard) can be used to collate aggregated data that is presented in the [analytics](/api-management/dashboard-configuration#traffic-analytics) screens of the Tyk Dashboard. + +The Gateway will not, by default, include the request and response payloads in the transaction records. This minimizes the size of the records and also avoids logging any sensitive content. The [detailed recording](/api-management/logs-metrics#capturing-detailed-logs) option is provided if you need to capture the payloads in the records. + +You can suppress the generation of transaction records for any endpoint by enabling the [do-not-track middleware](/api-management/traffic-transformation/do-not-track) for that endpoint. This provides granular control over request tracking. + +You can find details of all the options available to you when configuring analytics in the Gateway in the [reference documentation](/tyk-oss-gateway/configuration#analytics_config). + + + +For the Tyk Dashboard's analytics functionality to work, you must configure both per-request and aggregated pumps for the database platform that you are using. For more details see the [Setup Dashboard Analytics](/api-management/tyk-pump#setup-dashboard-analytics) section. + + + +#### Capturing Detailed Logs + +The Gateway will not, by default, include the request and response payloads in traffic logs. This minimizes the size of the records and also minimises the risk of logging sensitive content. + +You can, however, configure Tyk to capture the payloads in the transaction records if required. This can be particularly useful during development and testing phases or when debugging an issue with an API. + +This is referred to as detailed recording and can be enabled at different levels of granularity. The order of precedence is: +1. [API level](/api-management/logs-metrics#configure-at-api-level) +2. [Key level](/api-management/logs-metrics#configure-at-key-level) +3. [Gateway level](/api-management/logs-metrics#configure-at-gateway-level) + +Consequently, Tyk will first check whether the API definition has detailed recording enabled to determine whether to log the request and response bodies. If it does not, then it will check the key being used in the request and finally it will check the Gateway configuration. + + + +Be aware that enabling detailed recording greatly increases the size of the records and will require significantly more storage space as Tyk will store the entire request and response in wire format. +
+
+Tyk Cloud users can enable detailed recording per-API following the instructions on this page or, if required at the Gateway level, via a support request. The traffic logs are subject to the subscription's storage quota and so we recommend that detailed logging only be enabled if absolutely necessary to avoid unnecessary costs. + + + +##### Configure at API level + +You can enable detailed recording for an individual API by setting the [server.detailedActivityLogs.enabled](/api-management/gateway-config-tyk-oas#detailedactivitylogs) flag within the Tyk Vendor Extension. + +In the Dashboard UI, you can configure detailed recording using the **Enable Detailed Activity Logs** option in the API Designer. + +Enabling detailed activity logs for a Tyk OAS API + +**Tyk Classic APIs** + +When working with Tyk Classic APIs, you should configure the equivalent `enable_detailed_recording` flag in the root of the API definition. + +In the Tyk Classic API Designer, the **Enable Detailed Logging** option can be found in **Core Settings**. + +Enabling detailed activity logs for a Tyk Classic API + +When using Tyk Operator with Tyk Classic APIs, you can enable detailed recording by setting `spec.enable_detailed_recording` to `true`, as in this example: + +```yaml {linenos=true, linenostart=1, hl_lines=["10-10"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin +spec: + name: httpbin + use_keyless: true + protocol: http + active: true + enable_detailed_recording: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true +``` + +##### Configure at Key Level +An alternative approach to controlling detailed recording is to enable it only for specific [access keys](/api-management/policies#what-is-a-session-object). This is particularly useful for debugging purposes where you can configure detailed recording only for the key(s) that are reporting issues. + +You can enable detailed recording for a key simply by adding the following to the root of the key's JSON file: + +``` +"enable_detailed_recording": true +``` + + + +This will enable detailed recording only for API transactions where this key is used in the request. + + + +##### Configure at Gateway Level +Detailed recording can be configured at the Gateway level, affecting all APIs deployed on the Gateway, by enabling the [detailed recording](/tyk-oss-gateway/configuration#analytics_configenable_detailed_recording) option in `tyk.conf`. + +```.json expandable +{ + "enable_analytics" : true, + "analytics_config": { + "enable_detailed_recording": true + } +} +``` + +#### Enabling API Request Access Logs in Tyk Gateway + +As of Tyk Gateway `v5.8.0`, you can configure the Gateway to log individual API request transactions. To enable this feature, set the `TYK_GW_ACCESSLOGS_ENABLED` environment variable to `true`. + +##### Configuring output fields + +You can specify which fields are logged by configuring the `TYK_GW_ACCESSLOGS_TEMPLATE` environment variable. Below are the available values you can include: + +- `api_key`: Obfuscated or hashed API key used in the request. +- `client_ip`: IP address of the client making the request. +- `host`: Hostname of the request. +- `method`: HTTP method used in the request (e.g., GET, POST). +- `path`: URL path of the request. +- `protocol`: Protocol used in the request (e.g., HTTP/1.1). +- `remote_addr`: Remote address of the client. +- `upstream_addr`: Full upstream address including scheme, host, and path. +- `upstream_latency`: Roundtrip duration between the gateway sending the request to the upstream server and it receiving a response. +- `latency_total`: Total time taken for the request, including upstream latency and additional processing by the gateway. +- `user_agent`: User agent string from the client. +- `status`: HTTP response status code. + +To configure, set `TYK_GW_ACCESSLOGS_TEMPLATE` environment variable with the desired values in the format: `["value1", "value2", ...]`. + +##### Default log example + +Configuration using `tyk.conf` + +```json +{ + "access_logs": { + "enabled": true + } +} +``` + +Configuration using environment variables: + +``` +TYK_GW_ACCESSLOGS_ENABLED=true +``` + +Output: + +``` +time="Jan 29 08:27:09" level=info api_id=b1a41c9a89984ffd7bb7d4e3c6844ded api_key=00000000 api_name=httpbin client_ip="::1" host="localhost:8080" latency_total=62 method=GET org_id=678e6771247d80fd2c435bf3 path=/get prefix=access-log protocol=HTTP/1.1 remote_addr="[::1]:63251" status=200 upstream_addr="http://httpbin.org/get" upstream_latency=61 user_agent=PostmanRuntime/7.43.0 +``` + +##### Custom template log example + +Configuration using `tyk.conf` + +```json expandable +{ + "access_logs": { + "enabled": true, + "template": [ + "api_key", + "remote_addr", + "upstream_addr" + ] + } +} +``` + +Configuration using environment variables: + +``` +TYK_GW_ACCESSLOGS_ENABLED=true +TYK_GW_ACCESSLOGS_TEMPLATE="api_key,remote_addr,upstream_addr" +``` + +Output: + +``` +time="Jan 29 08:27:48" level=info api_id=b1a41c9a89984ffd7bb7d4e3c6844ded api_key=00000000 api_name=httpbin org_id=678e6771247d80fd2c435bf3 prefix=access-log remote_addr="[::1]:63270" upstream_addr="http://httpbin.org/get" +``` + +##### Performance Considerations + +Enabling access logs introduces some performance overhead: + +- **Latency:** Increases consistently by approximately 4%–13%, depending on CPU allocation and configuration. +- **Memory Usage:** Memory consumption increases by approximately 6%–7%. +- **Allocations:** The number of memory allocations increases by approximately 5%–6%. + + + + + While the overhead of enabling access logs is noticeable, the impact is relatively modest. These findings suggest the performance trade-off may be acceptable depending on the criticality of logging to your application. + + + +#### Aggregated analytics + +The traffic logs that Tyk Gateway generates are stored in the local [Redis](/api-management/logs-metrics#how-api-traffic-logging-works) temporal storage. They must be transferred to a persistent data store (such as MongoDB or PostgreSQL) for use by analytics tools, typically using Tyk Pump. Tyk Pump can also generate aggregated statistics from these data using the dedicated [Mongo Aggregate](/api-management/tyk-pump#mongodb) and [SQL Aggregate](/api-management/tyk-pump#sql) pumps. These offload processing from Tyk Dashboard and reduce storage requirements compared with storing all of the raw logs. + +The aggregate pumps calculate statistics from the analytics records, aggregated by hour, for the following keys in the traffic logs: + +| Key | Analytics aggregated by | Dashboard screen | +| :---------------- | :---------------------------------- | :------------------------------------------------------------- | +| `APIID` | API proxy | [Activity by API](/api-management/dashboard-configuration#activity-by-api) | +| `TrackPath` | API endpoint | [Activity by endpoint](/api-management/dashboard-configuration#activity-by-endpoint) | +| `ResponseCode` | HTTP status code (success/error) | [Activity by errors](/api-management/dashboard-configuration#activity-by-error) | +| `APIVersion` | API version | n/a | +| `APIKey` | Client access key/token | [Activity by Key](/api-management/dashboard-configuration#activity-by-key) | +| `OauthID` | OAuth client (if OAuth used) | [Traffic per OAuth Client](/api-management/dashboard-configuration#activity-by-oauth-client) | +| `Geo` | Geographic location of client | [Activity by location](/api-management/dashboard-configuration#activity-by-location) | + +##### Custom aggregation keys + +Whereas Tyk Pump will automatically produce aggregated statistics for the keys in the previous section, you can also define custom aggregation keys using Tyk's custom analytics tag feature which identifies specific HTTP request headers to be used as aggregation keys. This has various uses, for example" + +- You need to record additional information from the request into the analytics but want to avoid [detailed logging](/api-management/logs-metrics#capturing-detailed-logs) due to the volume of traffic logs. +- You wish to track a group of API requests, for example: + - Show me all API requests where `tenant-id=123` + - Show me all API requests where `user-group=abc` + +The Traffic Log middleware is applied to all endpoints in the API and so configuration is found in the `middleware.global` section of the Tyk Vendor Extension, within the `trafficLogs` section. Custom aggregation tags are specified as a list of HTTP headers in [middleware.global.trafficLogs.tagHeaders](/api-management/gateway-config-tyk-oas#trafficlogs) that Tyk should use for generation of custom aggregation tags for the API. + +For example if we include the header name `x-user-id` in the list of headers, then Tyk will create an aggregation key for each different value observed in that header. These aggregation keys will be given the name `-`, for example `x-user-id-1234` if the request contains the HTTP header `"x-user-id":1234`. + +**Tyk Classic APIs** + +If you are using Tyk Classic APIs, then the equivalent field in the API definition is [tag_headers](/api-management/gateway-config-tyk-classic#traffic-logs). + +In the Tyk Classic API Designer, the **Tag Headers** option can be found in **Advanced Options**. + +Tag Headers + +When using Tyk Operator with Tyk Classic APIs, you can configure custom analytics tags by setting `spec.tag_headers` to `true`, as in this example: + +```yaml {linenos=true, linenostart=1, hl_lines=["10-12"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-tag-headers +spec: + name: httpbin-tag-headers + use_keyless: true + protocol: http + active: true + tag_headers: + - Host + - User-Agent + proxy: + target_url: http://httpbin.org + listen_path: /httpbin-tag-headers + strip_listen_path: true +``` + +In this example we can see that the `Host` and `User-Agent` headers exist within the `tag_headers` array. For each incoming request Tyk will add `host-` and `user-agent-` tags to the list of tags in the traffic log. + +###### Suppressing generation of aggregates for custom keys + +If you don't want or need aggregated analytics for the headers you record with `tagHeaders`, you can configure Tyk Pump (or Tyk MDCB if it is performing the pump functionality) to discard those statistics when writing to the persistent analytics store. + +For both cases, you simply add the tags you want to ignore, or their prefixes, to the `ignore_tag_prefix_list` field in the appropriate configuration file or environment variable: + +- [Hybrid Pump config](/tyk-pump/tyk-pump-configuration/tyk-pump-environment-variables#pumpshybridmetaignore_tag_prefix_list) +- [Mongo Pump config](/tyk-pump/tyk-pump-configuration/tyk-pump-environment-variables#pumpsmongoaggregatemetaignore_tag_prefix_list) +- [Splunk Pump config](/tyk-pump/tyk-pump-configuration/tyk-pump-environment-variables#pumpssplunkmetaignore_tag_prefix_list) +- [SQL Pump config](/tyk-pump/tyk-pump-configuration/tyk-pump-environment-variables#pumpssqlaggregatemetaignore_tag_prefix_list) +- [MDCB config](/tyk-multi-data-centre/mdcb-configuration-options#ignore_tag_prefix_list) + + + + + If you add headers to the tags list that are unique to each request, such as a timestamp or unique request Id, then Tyk Gateway will essentially create an aggregation point _per request_ and the number of these tags in an hour will be equal to the number of requests. Since there's no real value in aggregating something that has a total of one, we recommend that you add such headers to the ignore list. + + + +## Metric Collection + +Metrics collection and analysis are key components of an Observability strategy, providing real-time insight into system behaviour and performance. + +Tyk Gateway, Pump and Dashboard have been instrumented for [StatsD](https://github.com/etsy/statsd) monitoring. + +Additionally, Tyk Gateway has also been instrumented for [New Relic](https://newrelic.com) metrics. + +### StatsD Instrumentation + +StatsD is a network daemon that listens for statistics, like counters and timers, sent over UDP or TCP and sends aggregates to one or more pluggable backend services. It's a simple yet powerful tool for collecting and aggregating application metrics. + +#### Configuring StatsD instrumentation + +To enable instrumentation for StatsD, you must set the environment variable: `TYK_INSTRUMENTATION=1` and then configure the `statsd_connection_string` field for each component. + +`statsd_connection_string` is a formatted string that specifies how to connect to the StatsD server. It typically includes information such as the host address, port number, and sometimes additional configuration options. + +Optionally you can set `statsd_prefix` to a custom prefix value that will be applied to each metric generated by Tyk. For example, you can configure separate prefixes for your production and staging environments to make it easier to differentiate between the metrics in your analysis tool. + +#### StatsD Keys + +There are plenty of keys (metrics) available when you enable the StatsD instrumentation, but these are the basics: + +- API traffic handled by Gateway: `gauges..Load.rps` (requests per second) +- Tyk Gateway API: `counters..SystemAPICall.called.count` (calls count) and `timers..SystemAPICall.success` (response time) +- Tyk Dashboard API: `counters..SystemAPICall.SystemCallComplete.count` (requests count), `counters..DashSystemAPIError.*` (API error reporting) +- Tyk Pump records: `counters..record.count` (number of records processed by pump) + + +### New Relic Instrumentation + +Tyk Gateway has been instrumented for New Relic metrics since v2.5. Simply add the following config section to `tyk.conf` to enable the instrumentation and generation of data: + +```json expandable +{ + "newrelic": { + "app_name": "", + "license_key": "" + } +} +``` + +## OpenTelemetry + +Starting from Tyk Gateway version 5.2, you can leverage the power of [OpenTelemetry](https://opentelemetry.io/docs/what-is-opentelemetry/), an open-source observability framework designed for cloud-native software. This enhances your API monitoring with end-to-end distributed tracing. At this time, Tyk does not support OpenTelemetry metrics or logging, but we have these on our roadmap for future enhancement of the product. + +This documentation will guide you through the process of enabling and configuring OpenTelemetry in Tyk Gateway. You'll also learn how to customize trace detail levels to meet your monitoring requirements. + +For further guidance on configuring your observability back-end, explore our guides for [Datadog](/api-management/logs-metrics#datadog), [Dynatrace](/api-management/logs-metrics#dynatrace), [Jaeger](/api-management/logs-metrics#jaeger) and [New Relic](/api-management/logs-metrics#new-relic). + +All the configuration options available when using Tyk's OpenTelemetry capability are documented in the [Tyk Gateway configuration guide](/tyk-oss-gateway/configuration#opentelemetry). + +### Using OpenTelemetry with Tyk + +OpenTelemetry support must be enabled at the Gateway level by adding the following to the Tyk Gateway configuration file (typically `tyk.conf`): + +```json + { + "opentelemetry": { + "enabled": true + } + } +``` + +Alternatively you can set the corresponding environment variable `TYK_GW_OPENTELEMETRY_ENABLED` to `true`. + + + +By default, OpenTelemetry spans are exported to the collector using the `gRPC` protocol to `localhost:4317`. You can choose between HTTP and gRPC protocols by configuring the [opentelemetry.exporter](/tyk-oss-gateway/configuration#opentelemetryexporter) field to `http` or `grpc`. You can specify an alternative target using the [opentelemetry.endpoint](/tyk-oss-gateway/configuration#opentelemetryendpoint) control. + + + +Tyk Gateway will now generate two spans for each request made to your APIs, encapsulating the entire request lifecycle. These spans include attributes and tags but lack fine-grained details. The parent span represents the total time from request reception to response and the child span represent the time spent in the upstream service. + +Detailed Tracing Disabled + +#### Detailed Tracing + +You can generate more detailed traces for requests to an API by setting the [server.detailedTracing](/api-management/gateway-config-tyk-oas#detailedtracing) flag in the Tyk Vendor Extension of the API definition. + +For users of the Tyk Dashboard UI, the **Enable Detailed Tracing** option in the Tyk OAS API Designer allows you to set and unset this option for the API. + +Detailed Tracing Disabled + +When detailed tracing is enabled for an API, Tyk creates a span for each middleware involved in request processing. These spans offer detailed insights, including the time taken for each middleware execution and the sequence of invocations. + +Detailed Tracing Enabled + +By choosing the appropriate setting, you can customize the level of tracing detail to suit your monitoring needs. + +**Tyk Classic APIs** + +If you are using Tyk Classic APIs, then the equivalent field in the API definition is [detailed_tracing](/api-management/gateway-config-tyk-classic#opentelemetry). + +### Understanding The Traces + +Tyk Gateway exposes a helpful set of *span attributes* and *resource attributes* with the generated spans. These attributes provide useful insights for analyzing your API requests. A clear analysis can be obtained by observing the specific actions and associated context within each request/response. This is where span and resource attributes play a significant role. + +#### Span Attributes + +A span is a named, timed operation that represents an operation. Multiple spans represent different parts of the workflow and are pieced together to create a trace. While each span includes a duration indicating how long the operation took, the span attributes provide additional contextual metadata. + +Span attributes are key-value pairs that provide contextual metadata for individual spans. Tyk automatically sets the following span attributes: + +- `tyk.api.name`: API name. +- `tyk.api.orgid`: Organization ID. +- `tyk.api.id`: API ID. +- `tyk.api.path`: API listen path. +- `tyk.api.tags`: If tagging is enabled in the API definition, the tags are added here. + +#### Resource Attributes + +Resource attributes provide contextual information about the entity that produced the telemetry data. Tyk exposes following resource attributes: + +#### Service Attributes + +The service attributes supported by Tyk are: + +| Attribute | Type | Description | +| :--------------------- | :-------- | :- | +| `service.name` | String | Service name for Tyk API Gateway: `tyk-gateway` | +| `service.instance.id` and `tyk.gw.id` | String | The Node ID assigned to the gateway. Example `solo-6b71c2de-5a3c-4ad3-4b54-d34d78c1f7a3` | +| `service.version` | String | Represents the service version. Example `v5.2.0` | +| `tyk.gw.dataplane` | Bool | Whether the Tyk Gateway is hybrid (`slave_options.use_rpc=true`) | +| `tyk.gw.group.id` | String | Represents the `slave_options.group_id` of the gateway. Populated only if the gateway is hybrid. | +| `tyk.gw.tags` | []String | Represents the gateway `segment_tags`. Populated only if the gateway is segmented. | + +By understanding and using these resource attributes, you can gain better insights into the performance of your API Gateways. + +#### Common HTTP Span Attributes + +Tyk follows the OpenTelemetry semantic conventions for HTTP spans. You can find detailed information on common attributes [here](https://github.com/open-telemetry/semantic-conventions/blob/main/docs/http/http-spans.md#common-attributes). + +Some of these common attributes include: + +- `http.method`: HTTP request method. +- `http.scheme`: URL scheme. +- `http.status_code`: HTTP response status code. +- `http.url`: Full HTTP request URL. + +For the full list and details, refer to the official [OpenTelemetry Semantic Conventions](https://github.com/open-telemetry/semantic-conventions/blob/main/docs/http/http-spans.md#common-attributes). + +### Advanced OpenTelemetry Capabilities + +#### Context Propagation + +This setting allows you to specify the type of context propagator to use for trace data. It's essential for ensuring compatibility and data integrity between different services in your architecture. The available options are: + +- **tracecontext**: This option supports the [W3C Trace Context](https://www.w3.org/TR/trace-context/) format. +- **b3**: This option serializes `SpanContext` to/from the B3 multi Headers format. [Here](https://github.com/openzipkin/b3-propagation) you can find more information of this propagator. + +The default setting is `tracecontext`. To configure this setting, you have two options: + +- **Environment Variable**: Use `TYK_GW_OPENTELEMETRY_CONTEXTPROPAGATION` to specify the context propagator type. +- **Configuration File**: Navigate to the `opentelemetry.context_propagation` field in your configuration file to set your preferred option. + +#### Sampling Strategies + +Tyk supports configuring the following sampling strategies via the Sampling configuration structure: + +##### Sampling Type + +This setting dictates the sampling policy that OpenTelemetry uses to decide if a trace should be sampled for analysis. The decision is made at the start of a trace and applies throughout its lifetime. By default, the setting is `AlwaysOn`. + +To customize, you can either set the `TYK_GW_OPENTELEMETRY_SAMPLING_TYPE` environment variable or modify the `opentelemetry.sampling.type` field in the Tyk Gateway configuration file. Valid values for this setting are: + +- **AlwaysOn**: All traces are sampled. +- **AlwaysOff**: No traces are sampled. +- **TraceIDRatioBased**: Samples traces based on a specified ratio. + +##### Sampling Rate + +This field is crucial when the `Type` is configured to `TraceIDRatioBased`. It defines the fraction of traces that OpenTelemetry will aim to sample, and accepts a value between 0.0 and 1.0. For example, a `Rate` set to 0.5 implies that approximately 50% of the traces will be sampled. The default value is 0.5. To configure this setting, you have the following options: + +- **Environment Variable**: Use `TYK_GW_OPENTELEMETRY_SAMPLING_RATE`. +- **Configuration File**: Update the `opentelemetry.sampling.rate` field in the configuration file. + +##### ParentBased Sampling + +This option is useful for ensuring the sampling consistency between parent and child spans. Specifically, if a parent span is sampled, all it's child spans will be sampled as well. This setting is particularly effective when used with `TraceIDRatioBased`, as it helps to keep the entire transaction story together. Using `ParentBased` with `AlwaysOn` or `AlwaysOff` may not be as useful, since in these cases, either all or no spans are sampled. The default value is `false`. Configuration options include: + +- **Environment Variable**: Use `TYK_GW_OPENTELEMETRY_SAMPLING_PARENTBASED`. +- **Configuration File**: Update the `opentelemetry.sampling.parent_based` field in the configuration file. + +### OpenTelemetry Backends for Tracing + +#### Datadog + +This guide explains how to configure Tyk API Gateway and the OpenTelemetry Collector to collect distributed traces in Datadog. It follows the reference documentation from [Datadog](https://docs.datadoghq.com/opentelemetry/otel_collector_datadog_exporter/?tab=onahost). + +While this tutorial demonstrates using an OpenTelemetry Collector running in Docker, the core concepts remain consistent regardless of how and where the OpenTelemetry collector is deployed. + +Whether you're using Tyk API Gateway in an open-source (OSS) or commercial deployment, the configuration options remain identical. + +##### Prerequisites + +- [Docker installed on your machine](https://docs.docker.com/get-docker/) +- Tyk Gateway v5.2.0 or higher +- OpenTelemetry Collector Contrib [docker image](https://hub.docker.com/r/otel/opentelemetry-collector-contrib). Make sure to use the Contrib distribution of the OpenTelemetry Collector as it is required for the [Datadog exporter](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/datadogexporter). + +##### Steps for Configuration + +1. **Configure the OpenTelemetry Collector** + + You will need: + - An [API key from Datadog](https://docs.datadoghq.com/account_management/api-app-keys/#add-an-api-key-or-client-token). For example, `6c35dacbf2e16aa8cda85a58d9015c3c`. + - Your [Datadog site](https://docs.datadoghq.com/getting_started/site/#access-the-datadog-site). Examples are: `datadoghq.com`, `us3.datadoghq.com` and `datadoghq.eu`. + + Create a new YAML configuration file named `otel-collector.yml` with the following content: + + ```yaml expandable + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + processors: + batch: + send_batch_max_size: 100 + send_batch_size: 10 + timeout: 10s + exporters: + datadog: + api: + site: "YOUR-DATADOG-SITE" + key: "YOUR-DATAGOG-API-KEY" + service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [datadog] + + ``` + +2. **Configure a test API** + + If you don't have any APIs configured yet, create a subdirectory called `apps` in the current directory. Create a new file `apidef-hello-world.json` and copy this very simple API definition for testing purposes: + + ```json + { + "name": "Hello-World", + "slug": "hello-world", + "api_id": "Hello-World", + "org_id": "1", + "use_keyless": true, + "detailed_tracing": true, + "version_data": { + "not_versioned": true, + "versions": { + "Default": { + "name": "Default", + "use_extended_paths": true + } + } + }, + "proxy": { + "listen_path": "/hello-world/", + "target_url": "http://httpbin.org/", + "strip_listen_path": true + }, + "active": true + } + ``` + +3. **Create the Docker-Compose file** + + Save the following YAML configuration to a file named `docker-compose.yml`. + + ```yaml + version: "2" + services: + # OpenTelemetry Collector Contrib + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + volumes: + - ./otel-collector.yml:/etc/otel-collector.yml + command: ["--config=/etc/otel-collector.yml"] + ports: + - "4317" # OTLP gRPC receiver + networks: + - tyk + + # Tyk API Gateway, open-source deployment + tyk: + image: tykio/tyk-gateway:v5.2 + ports: + - 8080:8080 + environment: + - TYK_GW_OPENTELEMETRY_ENABLED=true + - TYK_GW_OPENTELEMETRY_EXPORTER=grpc + - TYK_GW_OPENTELEMETRY_ENDPOINT=otel-collector:4317 + volumes: + - ./apps:/opt/tyk-gateway/apps + depends_on: + - redis + networks: + - tyk + + redis: + image: redis:4.0-alpine + ports: + - 6379:6379 + command: redis-server --appendonly yes + networks: + - tyk + + networks: + tyk: + ``` + + + To start the services, go to the directory that contains the docker-compose.yml file and run the following command: + + ```bash + docker-compose up + ``` + +4. **Explore OpenTelemetry traces in Datadog** + + Begin by sending a few requests to the API endpoint configured in step 2: + `` + http://localhost:8080/hello-world/ + `` + + Next, log in to Datadog and navigate to the 'APM' / 'Traces' section. Here, you should start observing traces generated by Tyk: + + Tyk API Gateway distributed trace in Datadog + + Click on a trace to view all its internal spans: + + Tyk API Gateway spans in Datadog + + Datadog will generate a service entry to monitor Tyk API Gateway and will automatically compute valuable metrics using the ingested traces. + + Tyk API Gateway service monitoring in Datadog + +##### Troubleshooting + +If you do not observe any traces appearing in Datadog, consider the following steps for resolution: + +- Logging: Examine logs from Tyk API Gateway and from the OpenTelemetry Collector for any issues or warnings that might provide insights. +- Data Ingestion Delays: Be patient, as there could be some delay in data ingestion. Wait for 10 seconds to see if traces eventually appear, as this is the timeout we have configured in the batch processing of the OpenTelemetry collector within step 1. + +#### Dynatrace + +This documentation covers how to set up Dynatrace to ingest OpenTelemetry traces via the OpenTelemetry Collector (OTel Collector) using Docker. + +##### Prerequisites + +- [Docker installed on your machine](https://docs.docker.com/get-docker/) +- [Dynatrace account](https://www.dynatrace.com/) +- Dynatrace Token +- Gateway v5.2.0 or higher +- OTel Collector [docker image](https://hub.docker.com/r/otel/opentelemetry-collector) + +##### Steps for Configuration + +1. **Generate Dynatrace Token** + + 1. In the Dynatrace console, navigate to access keys. + 2. Click on _Create a new key_ + 3. You will be prompted to select a scope. Choose _Ingest OpenTelemetry_ traces. + 4. Save the generated token securely; it cannot be retrieved once lost. + + Example of a generated token ([taken from Dynatrace website](https://www.dynatrace.com/support/help/dynatrace-api/basics/dynatrace-api-authentication#token-format-example)): + + ```bash + dt0s01.ST2EY72KQINMH574WMNVI7YN.G3DFPBEJYMODIDAEX454M7YWBUVEFOWKPRVMWFASS64NFH52PX6BNDVFFM572RZM + ``` + +2. **Configuration Files** + + 1. **OTel Collector Configuration File** + + Create a YAML file named `otel-collector-config.yml`. In this file replace `` with the string from the address bar when you log into Dynatrace. Replace `` with the token you generated earlier. + + Here's a sample configuration file: + + ```yaml + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + processors: + batch: + exporters: + otlphttp: + endpoint: "https://.live.dynatrace.com/api/v2/otlp" + headers: + Authorization: "Api-Token " # You must keep 'Api-Token', just modify + extensions: + health_check: + pprof: + endpoint: :1888 + zpages: + endpoint: :55679 + service: + extensions: [pprof, zpages, health_check] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlphttp] + ``` + + 2. **Docker Compose File** + + Create a file named docker-compose.yml. + + Here is the sample Docker Compose file: + + ```yaml + version: "3.9" + services: + otel-collector: + image: otel/opentelemetry-collector:latest + volumes: + - ./configs/otel-collector-config.yml:/etc/otel-collector.yml + command: ["--config=/etc/otel-collector.yml"] + networks: + - tyk + ports: + - "1888:1888" # pprof extension + - "13133:13133" # health_check extension + - "4317:4317" # OTLP gRPC receiver + - "4318:4318" # OTLP http receiver + - "55670:55679" # zpages extension + networks: + tyk: + ``` + +3. **Testing and Viewing Traces** + + **1.** Launch the Docker containers: docker-compose up -d + + **2.** Initialize your Tyk environment. + + **3.** Configure a basic HTTP API on the Tyk Gateway or Dashboard. + + **4.** Use cURL or Postman to send requests to the API gateway. + + **5.** Navigate to Dynatrace -> Services -> Tyk-Gateway. + + Dynatrace Services + + **6.** Wait for 5 minutes and refresh. + + **7.** Traces, along with graphs, should appear. If they don't, click on the "Full Search" button. + + Dynatrace Metrics + +4. **Troubleshooting** + + - If traces are not appearing, try clicking on the "Full Search" button after waiting for 5 minutes. + Make sure your Dynatrace token is correct in the configuration files. + - Validate the Docker Compose setup by checking the logs for any errors: `docker-compose logs` + +And there you have it! You've successfully integrated Dynatrace with the OpenTelemetry Collector using Docker. + +#### Elasticsearch + +This quick start explains how to configure Tyk API Gateway (OSS, self-managed or hybrid gateway connected to Tyk Cloud) with the OpenTelemetry Collector to export distributed traces to [Elasticsearch](https://www.elastic.co/observability). + +##### Prerequisites + +Ensure the following prerequisites are met before proceeding: + +* Tyk Gateway v5.2 or higher +* OpenTelemetry Collector deployed locally +* Elasticsearch deployed locally or an account on Elastic Cloud with Elastic APM + +Elastic Observability natively supports OpenTelemetry and its OpenTelemetry protocol (OTLP) to ingest traces, metrics, and logs. + +OpenTelemetry support in Elasticsearch +Credit: Elasticsearch, [OpenTelemetry on Elastic](https://www.elastic.co/blog/opentelemetry-observability) + +##### Steps for Configuration + +1. **Configure Tyk API Gateway** + + To enable OpenTelemetry in Tyk API Gateway, follow these steps: + + For Tyk Helm Charts: + * Add the following configuration to the Tyk Gateway section: + + ```yaml + tyk-gateway: + gateway: + opentelemetry: + enabled: true + endpoint: {{Add your endpoint here}} + exporter: grpc + ``` + + For Docker Compose: + * In your docker-compose.yml file for Tyk Gateway, add the following environment variables: + + ```yaml + environment: + - TYK_GW_OPENTELEMETRY_ENABLED=true + - TYK_GW_OPENTELEMETRY_EXPORTER=grpc + - TYK_GW_OPENTELEMETRY_ENDPOINT={{Add your endpoint here}} + ``` + + Make sure to replace `` with the appropriate endpoint from your OpenTelemetry collector. + + After enabling OpenTelemetry at the Gateway level, you can activate [detailed tracing](/api-management/logs-metrics#opentelemetry) for specific APIs by editing their respective API definitions. Set the `detailed_tracing` option to either true or false. By default, this setting is false. + +2. **Configure the OpenTelemetry Collector to Export to Elasticsearch** + + To configure the OTel Collector with Elasticsearch Cloud, follow these steps: + + * Sign up for an [Elastic account](https://www.elastic.co/) if you haven't already + * Once logged in to your Elastic account, select "Observability" and click on the option "Monitor my application performance" + + Configure Elasticsearch + + * Scroll down to the APM Agents section and click on the OpenTelemetry tab + + Configure Elasticsearch + + * Search for the section "Configure OpenTelemetry in your application". You will need to copy the value of "OTEL_EXPORTER_OTLP_ENDPOINT" and "OTEL_EXPORTER_OTLP_HEADERS" in your OpenTelemetry Collector configuration file. + + Configure Elasticsearch + + * Update your OpenTelemetry Collector configuration, here's a simple example: + + ```yaml + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 # OpenTelemetry receiver endpoint + processors: + batch: + exporters: + otlp/elastic: + endpoint: "ELASTIC_APM_SERVER_ENDPOINT_GOES_HERE" #exclude scheme, e.g. HTTPS:// or HTTP:// + headers: + # Elastic APM Server secret token + Authorization: "Bearer ELASTIC_APM_SECRET_TOKEN_GOES_HERE" + service: + pipelines: + traces: + receivers: [otlp] + exporters: [otlp/elastic] + ``` + + If are running Elasticsearch locally, you will need to use your APM Server endpoint (elastic-apm-server:8200) and set-up [a secret token authorization in ElasticSearch](https://www.elastic.co/guide/en/observability/current/secret-token.html). + + You can refer to the [example configuration provided by Elastic](https://www.elastic.co/guide/en/observability/current/open-telemetry-direct.html#connect-open-telemetry-collector) for more guidance on the OpenTelemetry Collector configuration. + +3. **Explore OpenTelemetry Traces in Elasticsearch** + + * In Elasticsearch Cloud: + * Go to "Home" and select "Observability." + Configure Elasticsearch + * On the right menu, click on "APM / Services." + * Click on "tyk-gateway." + + You will see a dashboard automatically generated based on the distributed traces sent by Tyk API Gateway to Elasticsearch. + + Configure Elasticsearch + + Select a transaction to view more details, including the distributed traces: + + Configure Elasticsearch + +#### New Relic + +This guide provides a step-by-step procedure to integrate New Relic with Tyk Gateway via the OpenTelemetry Collector. At the end of this guide, you will be able to visualize traces and metrics from your Tyk Gateway on the New Relic console. + +##### Prerequisites + +- [Docker installed on your machine](https://docs.docker.com/get-docker/) +- [New Relic Account](https://newrelic.com/) +- New Relic API Key +- Gateway v5.2.0 or higher +- OTel Collector [docker image](https://hub.docker.com/r/otel/opentelemetry-collector) + +##### Steps for Configuration + +1. **Obtain New Relic API Key** + + 1. Navigate to your New Relic Console. + + 2. Go to `Profile β†’ API keys`. + + 3. Copy the key labeled as `INGEST-LICENSE`. + +
+ + + + + You can follow the [official New Relic documentation](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/) for more information. + + + + **Example token:** + + ```bash + 93qwr27e49e168d3844c5h3d1e878a463f24NZJL + ``` + +2. **Configuration Files** + + **OTel Collector Configuration YAML** + + 1. Create a file named `otel-collector-config.yml` under the configs directory. + 2. Copy the following template into that file: + + ```yaml + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + processors: + batch: + exporters: + otlphttp: + endpoint: "" + headers: + api-Key: "" + extensions: + health_check: + pprof: + endpoint: :1888 + zpages: + endpoint: :55679 + service: + extensions: [pprof, zpages, health_check] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlphttp] + ``` + + - Replace `` with your specific New Relic endpoint (`https://otlp.nr-data.net` for US or `https://otlp.eu01.nr-data.net` for EU). + - Replace `` with the API key obtained in Step 1. + + **Docker Compose configuration** + + 1. Create a file named docker-compose.yml at the root level of your project directory. + + 2. Paste the following code into that file: + + ```yaml + version: "3.9" + services: + otel-collector: + image: otel/opentelemetry-collector:latest + volumes: + - ./otel-collector-config.yml:/etc/otel-collector.yml + command: ["--config=/etc/otel-collector.yml"] + networks: + - tyk + ports: + - "1888:1888" # pprof extension + - "13133:13133" # health_check extension + - "4317:4317" # OTLP gRPC receiver + - "4318:4318" # OTLP http receiver + - "55670:55679" # zpages extension + + networks: + tyk: + ``` +
+ + + + + Replace the variable fields with the relevant data. + + + +3. **Testing and Verifying Traces** + + 1. Run `docker-compose up -d` to start all services. + + 2. Initialize your Tyk environment. + + 3. Create a simple `httpbin` API using Tyk Dashboard. You can follow the [Tyk Dashboard documentation](/api-management/gateway-config-managing-classic#create-an-api) for more information. + + 4. Send requests to the API using cURL or Postman. + + 5. Open New Relic Console. + + 6. Navigate to `APM & Services β†’ Services - OpenTelemetry β†’ tyk-gateway`. + + New Relic Services + + 7. Wait for about 5 minutes for the data to populate. + + Traces and graphs should now be visible on your New Relic console. + + New Relic Metrics + +
+ + + + + If traces are not showing, try refreshing the New Relic dashboard. + + + +##### Troubleshooting + +- If the traces aren't appearing, double-check your API key and endpoints. +- Ensure that your Tyk Gateway and New Relic are both running and connected. + +##### Conclusion + +You have successfully integrated New Relic with Tyk Gateway via the OpenTelemetry Collector. You can now monitor and trace your APIs directly from the New Relic console. + +#### Jaeger + +##### Using Docker + +This quick start guide offers a detailed, step-by-step walkthrough for configuring Tyk API Gateway (OSS, self-managed or hybrid gateway connected to Tyk Cloud) with OpenTelemetry and [Jaeger](https://www.jaegertracing.io/) to significantly improve API observability. We will cover the installation of essential components, their configuration, and the process of ensuring seamless integration. + +For Kubernetes instructions, please refer to [How to integrate with Jaeger on Kubernetes](#using-kubernetes). + +###### Prerequisites + +Ensure the following prerequisites are met before proceeding: + +- [Docker installed on your machine](https://docs.docker.com/get-docker/) +- Gateway v5.2.0 or higher + +###### Steps for Configuration + +1. **Create the Docker-Compose File for Jaeger** + + Save the following YAML configuration in a file named docker-compose.yml: + + ```yaml + version: "2" + services: + # Jaeger: Distributed Tracing System + jaeger-all-in-one: + image: jaegertracing/all-in-one:latest + ports: + - "16686:16686" # Jaeger UI + - "4317:4317" # OTLP receiver + ``` + + This configuration sets up Jaeger's all-in-one instance with ports exposed for Jaeger UI and the OTLP receiver. + +2. **Deploy a Test API Definition** + + If you haven't configured any APIs yet, follow these steps: + + - Create a subdirectory named apps in the current directory. + - Create a new file named `apidef-hello-world.json`. + - Copy the provided simple API definition below into the `apidef-hello-world.json` file: + + + ```json + { + "name": "Hello-World", + "slug": "hello-world", + "api_id": "Hello-World", + "org_id": "1", + "use_keyless": true, + "detailed_tracing": true, + "version_data": { + "not_versioned": true, + "versions": { + "Default": { + "name": "Default", + "use_extended_paths": true + } + } + }, + "proxy": { + "listen_path": "/hello-world/", + "target_url": "http://httpbin.org/", + "strip_listen_path": true + }, + "active": true + } + ``` + + This API definition sets up a basic API named Hello-World for testing purposes, configured to proxy requests to `http://httpbin.org/`. + +3. **Run Tyk Gateway OSS with OpenTelemetry Enabled** + + To run Tyk Gateway with OpenTelemetry integration, extend the previous Docker Compose file to include Tyk Gateway and Redis services. Follow these steps: + + - Add the following configuration to your existing docker-compose.yml file: + + ```yaml + # ... Existing docker-compose.yml content for jaeger + + tyk: + image: tykio/tyk-gateway:v5.2.0 + ports: + - 8080:8080 + environment: + - TYK_GW_OPENTELEMETRY_ENABLED=true + - TYK_GW_OPENTELEMETRY_EXPORTER=grpc + - TYK_GW_OPENTELEMETRY_ENDPOINT=jaeger-all-in-one:4317 + volumes: + - ${TYK_APPS:-./apps}:/opt/tyk-gateway/apps + depends_on: + - redis + + redis: + image: redis:4.0-alpine + ports: + - 6379:6379 + command: redis-server --appendonly yes + ``` + + - Navigate to the directory containing the docker-compose.yml file in your terminal. + - Execute the following command to start the services: + + ```bash + docker compose up + ``` + +4. **Explore OpenTelemetry Traces in Jaeger** + + - Start by sending a few requests to the API endpoint configured in Step 2: + ```bash + curl http://localhost:8080/hello-world/ -i + ``` + + - Access Jaeger at [http://localhost:16686](http://localhost:16686). + - In Jaeger's interface: + - Select the service named tyk-gateway. + - Click the *Find Traces* button. + + You should observe traces generated by Tyk Gateway, showcasing the distributed tracing information. + + Tyk API Gateway distributed trace in Jaeger + + Select a trace to visualize its corresponding internal spans: + + Tyk API Gateway spans in Jaeger + + +##### Using Kubernetes + +This quick start guide offers a detailed, step-by-step walkthrough for configuring Tyk Gateway OSS with OpenTelemetry and [Jaeger](https://www.jaegertracing.io/) on Kubernetes to significantly improve API observability. We will cover the installation of essential components, their configuration, and the process of ensuring seamless integration. + +For Docker instructions, please refer to [How to integrate with Jaeger on Docker](#using-docker). + + +###### Prerequisites + +Ensure the following prerequisites are in place before proceeding: + +- A functional Kubernetes cluster +- [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) and [helm](https://helm.sh/docs/intro/install/) CLI tools installed + +###### Steps for Configuration + +1. **Install Jaeger Operator** + + For the purpose of this tutorial, we will use jaeger-all-in-one, which includes the Jaeger agent, collector, query, and UI in a single pod with in-memory storage. This deployment is intended for development, testing, and demo purposes. Other deployment patterns can be found in the [Jaeger Operator documentation](https://www.jaegertracing.io/docs/1.51/operator/#deployment-strategies). + + + 1. Install the cert-manager release manifest (required by Jaeger) + + ```bash + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.2/cert-manager.yaml + ``` + + 2. Install [Jaeger Operator](https://www.jaegertracing.io/docs/1.51/operator/). + + ```bash + kubectl create namespace observability + kubectl create -f https://github.com/jaegertracing/jaeger-operator/releases/download/v1.51.0/jaeger-operator.yaml -n observability + + ``` + + 3. After the Jaeger Operator is deployed to the `observability` namespace, create a Jaeger instance: + + ```bash + kubectl apply -n observability -f - < + + + +Please make sure you are installing Redis versions that are supported by Tyk. Please refer to Tyk docs to get list of [supported versions](/tyk-self-managed#redis). + + + + + Tyk Gateway is now accessible through service gateway-svc-tyk-oss-tyk-gateway at port 8080 and exports the OpenTelemetry traces to the `jaeger-all-in-one-collector` service. + +3. **Deploy Tyk Operator** + + Deploy Tyk Operator to manage APIs in your cluster: + + ```bash + kubectl create namespace tyk-operator-system + kubectl create secret -n tyk-operator-system generic tyk-operator-conf \ + --from-literal "TYK_AUTH=$APISecret" \ + --from-literal "TYK_ORG=org" \ + --from-literal "TYK_MODE=ce" \ + --from-literal "TYK_URL=http://gateway-svc-tyk-otel-tyk-gateway.tyk.svc:8080" \ + --from-literal "TYK_TLS_INSECURE_SKIP_VERIFY=true" + helm install tyk-operator tyk-helm/tyk-operator -n tyk-operator-system + + ``` + +4. **Deploy a Test API Definition** + + Save the following API definition as `apidef-hello-world.yaml`: + + ```yaml + apiVersion: tyk.tyk.io/v1alpha1 + kind: ApiDefinition + metadata: + name: hello-world + spec: + name: hello-world + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org/ + listen_path: /hello-world + strip_listen_path: true + ``` + + To apply this API definition, run the following command: + + ```bash + kubectl apply -f apidef-hello-world.yaml + ``` + + This step deploys an API definition named hello-world using the provided configuration. It enables a keyless HTTP API proxying requests to http://httpbin.org/ and accessible via the path /hello-world. + +5. **Explore OpenTelemetry traces in Jaeger** + + You can use the kubectl `port-forward command` to access Tyk and Jaeger services running in the cluster from your local machine's localhost: + + For Tyk API Gateway: + + ```bash + kubectl port-forward service/gateway-svc-tyk-otel-tyk-gateway 8080:8080 -n tyk + ``` + + For Jaeger: + + ```bash + kubectl port-forward service/jaeger-all-in-one-query 16686 -n observability + ``` + + Begin by sending a few requests to the API endpoint configured in step 2: + + ```bash + curl http://localhost:8080/hello-world/ -i + ``` + + Next, navigate to Jaeger on `http://localhost:16686`, select the Β΄serviceΒ΄ called Β΄tyk-gatewayΒ΄ and click on the button Β΄Find tracesΒ΄. You should see traces generated by Tyk: + + Tyk API Gateway distributed trace in Jaeger + + Click on a trace to view all its internal spans: + + Tyk API Gateway spans in Jaeger + +## OpenTracing (deprecated) + + + +**Deprecation** + +The CNCF (Cloud Native Foundation) has archived the OpenTracing project. This means that no new pull requests or feature requests are accepted into OpenTracing repositories. + +We introduced support for [OpenTelemetry](/api-management/logs-metrics#opentelemetry) in Tyk v5.2. We recommend that users migrate to OpenTelemetry for better support of your tracing needs. + +OpenTracing is now deprecated in Tyk products. + + + +### OpenTracing tools with legacy Tyk integration + +- [Jaeger](/api-management/logs-metrics#jaeger-1) +- [Zipkin](/api-management/logs-metrics#zipkin) +- [New Relic](/api-management/logs-metrics#new-relic-1) + +### Enabling OpenTracing + +OpenTracing can be configured at the Gateway level by adding the following configuration to your Gateway configuration (typically via the `tyk.conf` file or equivalent [environment variables](/tyk-oss-gateway/configuration). + +```.json +{ + "tracing": { + "enabled": true, + "name": "${tracer_name}", + "options": {} + } +} +``` expandable + +Where: +- `name` is the name of the supported tracer +- `enabled`: set this to true to enable tracing +- `options`: key/value pairs for configuring the enabled tracer. See the + supported tracer documentation for more details. + +Tyk will automatically propagate tracing headers to APIs when tracing is enabled. + +### Jaeger + + + +Tyk's OpenTelemetry Tracing works with Jaeger and we recommend following our guide to [use OpenTelemetry with Jaeger](/api-management/logs-metrics#jaeger) rather than the following deprecated Open Tracing method. + + + +Prior to Tyk 5.2, you cannot use OpenTelemetry and so must use [OpenTracing](https://opentracing.io/) with the [Jaeger client libraries](https://www.jaegertracing.io/docs/1.11/client-libraries/) to send Tyk Gateway traces to Jaeger. + +**Configuring Jaeger** + +In `tyk.conf` on `tracing` setting + +```{.json} +{ + "tracing": { + "enabled": true, + "name": "jaeger", + "options": {} + } +} +``` + +`options` are settings that are used to initialise the Jaeger client. For more details about the options [see client libraries](https://www.jaegertracing.io/docs/1.11/client-libraries/) + +**Sample configuration** + +```{.json} +{ + "tracing": { + "enabled": true, + "name": "jaeger", + "options": { + "baggage_restrictions": null, + "disabled": false, + "headers": null, + "reporter": { + "BufferFlushInterval": "0s", + "collectorEndpoint": "", + "localAgentHostPort": "jaeger:6831", + "logSpans": true, + "password": "", + "queueSize": 0, + "user": "" + }, + "rpc_metrics": false, + "sampler": { + "maxOperations": 0, + "param": 1, + "samplingRefreshInterval": "0s", + "samplingServerURL": "", + "type": "const" + }, + "serviceName": "tyk-gateway", + "tags": null, + "throttler": null + } + } +} +``` expandable + +### New Relic + + + +Tyk's OpenTelemetry Tracing works with New Relic and we recommend following our guide to [use OpenTelemetry with New Relic](/api-management/logs-metrics#new-relic) rather than the following deprecated Open Tracing method. + + + +Prior to Tyk 5.2, you cannot use OpenTelemetry and so must use [OpenTracing](https://opentracing.io/) to send Tyk Gateway traces to [*New Relic*](https://newrelic.com/) using the *Zipkin* format.
+ +**Configuring New Relic** + +In `tyk.conf` under the `tracing` section + +```.json +{ + "tracing": { + "enabled": true, + "name": "zipkin", + "options": {} + } +} +``` + +In the `options` setting you can set the initialisation of the *Zipkin* client. + +**Sample configuration** + +```.json +{ + "tracing": { + "enabled": true, + "name": "zipkin", + "options": { + "reporter": { + "url": "https://trace-api.newrelic.com/trace/v1?Api-Key=NEW_RELIC_LICENSE_KEY&Data-Format=zipkin&Data-Format-Version=2" + } + } + } +} +``` + +`reporter.url` is the URL to the *New Relic* server, where trace data will be sent to. + +### Zipkin + +Prior to Tyk 5.2, you cannot use OpenTelemetry and so must use [OpenTracing](https://opentracing.io/) with the [Zipkin Go tracer](https://zipkin.io/pages/tracers_instrumentation) to send Tyk Gateway traces to Zipkin. + +**Configuring Zipkin** + +In `tyk.conf` on `tracing` setting + +```{.json} +{ + "tracing": { + "enabled": true, + "name": "zipkin", + "options": {} + } +} +``` + +`options` are settings that are used to initialise the Zipkin client. + +**Sample configuration** + +```{.json} +{ + "tracing": { + "enabled": true, + "name": "zipkin", + "options": { + "reporter": { + "url": "http:localhost:9411/api/v2/spans" + } + } + } +} +``` + +`reporter.url` is the URL to the Zipkin server, where trace data will be sent. diff --git a/api-management/manage-apis/api-operations/api-observability.mdx b/api-management/manage-apis/api-operations/api-observability.mdx new file mode 100644 index 00000000..1c0212a7 --- /dev/null +++ b/api-management/manage-apis/api-operations/api-observability.mdx @@ -0,0 +1,63 @@ +--- +title: "API Observability" +'og:description': "Explains how to achieve API observability through Open Telemetry signals such as traces, metrics and logs" +tags: ['API Observability', 'Distributed Tracing', 'Metrics', 'Logs', 'Logging', 'Open Telemetry', 'OTel'] +sidebarTitle: "Monitoring and Troubleshooting APIs" +--- + +API observability is the process of monitoring and analyzing APIs to gain insights into developer and end-user experience and to ensure the reliability of your system. + +You can achieve API observability by using a combination of telemetry signals such as traces, metrics, and logs. Each of these signals serves a specific purpose in monitoring and troubleshooting API issues: + +## Distributed tracing + +Distributed traces provide a detailed, end-to-end view of a single API request or transaction as it traverses through various services and components. Traces are crucial for understanding the flow of requests and identifying bottlenecks or latency issues. Here's how you can make use of traces for API observability: + +- **End-to-end request tracing:** Implement distributed tracing across your microservices architecture to track requests across different services and gather data about each service's contribution to the overall request latency. + +- **Transaction Flow:** Visualize the transaction flow by connecting traces to show how requests move through different services, including entry points (e.g., API gateway), middleware and backend services. + +- **Latency Analysis:** Analyze trace data to pinpoint which service or component is causing latency issues, allowing for quick identification and remediation of performance bottlenecks. + +- **Error Correlation:** Use traces to correlate errors across different services to understand the root cause of issues and track how errors propagate through the system. + + +From v5.2+, Tyk supports OpenTelemetry standard for tracing. You can configure Tyk to work with an [OpenTelemetry collector](https://opentelemetry.io/docs/collector/) or integrate it with any [observability vendor supporting OpenTelemetry](https://opentelemetry.io/ecosystem/vendors/) to capture traces of API requests as they flow through Tyk API Gateway and any upstream services. + +Explore our guides for [Datadog](/api-management/logs-metrics#datadog), [Dynatrace](/api-management/logs-metrics#dynatrace), [Jaeger](/api-management/logs-metrics#using-docker) and [New Relic](/api-management/logs-metrics#new-relic) for further info on how to integrate with 3rd party observability vendors. + +Tyk also supports OpenTracing (now deprecated), but we recommend users to start migrating to OpenTelemetry for a comprehensive, vendor-neutral technology with wide industry support. + +## Metrics + +Metrics provide aggregated, quantitative data about the performance and behavior of an API over time. They offer insights into the overall health of the system. Here's how you can leverage metrics for API observability: + +- **Key Performance Indicators (KPIs):** Define and track essential metrics such as request rate, response time, error rate and resource utilization to monitor the overall health and performance of the API. + +- **Custom Metrics:** Create custom metrics that are specific to your API's functionality or business objectives. For example, track the number of successful payments processed or the number of users signed up. + +- **Threshold Alerts:** Set up alerts based on predefined thresholds for metrics to receive notifications when API performance deviates from the expected norm. + +- **Trend Analysis:** Analyze metric trends over time to identify long-term performance patterns, plan for scaling and detect anomalies. + + +Tyk offers built-in metrics and analytics in [Tyk Dashboard](/api-management/dashboard-configuration#traffic-analytics) through Tyk API Gateway and Tyk Pump. These metrics provide insights into API usage, traffic patterns and response times. The built-in metrics allow you to track overall API traffic, detailed API analytics including: request count, response time distribution and error rates. Furthermore, API usage can be tracked on a per-key basis. + +You can also use Tyk Pump to export those metrics to [different back-ends](/api-management/tyk-pump#external-data-stores). Here is an example of using Tyk Pump to send [API analytics metrics to Prometheus and Grafana](https://tyk.io/blog/service-level-objectives-for-your-apis-with-tyk-prometheus-and-grafana/). From v5.2+, you can also leverage the OpenTelemetry spans exported from Tyk Gateway to calculate and export [span metrics](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/connector/spanmetricsconnector/README.md) from the OpenTelemetry collector. + +## Logs + +Logs provide detailed records of events and activities within the API and its associated services. Logs are invaluable for debugging issues and understanding what happened at a specific point in time. Here's how you can utilize logs for API observability: + +- **Error Identification:** Use logs to identify errors, exceptions, and warning messages that indicate issues with the API's behavior. + +- **Debugging:** Logs help developers troubleshoot and debug issues by providing detailed information about the sequence of events leading up to a problem. + +- **Security Monitoring:** Monitor logs for security-related events, such as authentication failures, access control violations and suspicious activities. + +- **Audit Trail:** Maintain an audit trail of important actions and changes to the API, including configuration changes, access control changes and data updates. + + +Tyk allows you to capture and analyze logs related to API requests and responses in the [Log Browser](/api-management/dashboard-configuration#activity-logs) . You can optionally enable detailed recording for the requests per API level or per Key level to store inbound request and outbound response data. You can [enable debug modes](/api-management/troubleshooting-debugging#capturing-detailed-logs) for selected APIs and send the detail logs to one or more Pump backend instances. + +To achieve comprehensive API observability, it is essential to integrate traces, metrics and logs into the observability tools that the team in charge of the APIs are already using. Those tools should allow users to query and visualize data, set up alerts and provide an intuitive interface for monitoring and troubleshooting API issues effectively. See also our 7 observability anti-pattern to avoid when working with APIs: [Bad API observability](https://tyk.io/blog/bad-api-observability/). diff --git a/api-management/manage-apis/deploy-apis/deploy-apis-overview.mdx b/api-management/manage-apis/deploy-apis/deploy-apis-overview.mdx new file mode 100644 index 00000000..c3346ad0 --- /dev/null +++ b/api-management/manage-apis/deploy-apis/deploy-apis-overview.mdx @@ -0,0 +1,54 @@ +--- +title: "API Creation Methods" +'og:description': "Different ways to create and manage APIs in Tyk" +sidebarTitle: "API Creation Methods" +tags: ['API Management', 'API Configuration', 'Dashboard', 'Tyk Sync', 'Tyk Operator'] +--- + +This page explains the different methods available for creating and managing APIs in Tyk, each suited to different use cases and workflow requirements. + +

File-based configuration

+Load API configurations directly to the `/apps` folder using JSON API specifications. This method is available for open source users and is ideal for testing gateway and API configurations. + +**Use case:** Testing and experimentation in development environments. + +**Learn more:** +* [Create an API in file-based mode](/api-management/gateway-config-managing-classic#create-an-api-in-file-based-mode) + +## Dashboard UI + +Create and configure APIs through the web-based Dashboard interface. Changes take effect immediately, making this method suitable for learning, testing, and proof-of-concept work. + +**Use case:** Manual API management, learning, and proof-of-concept projects. + +**Learn more:** +* [Create an API with the Dashboard](/api-management/gateway-config-managing-classic#create-an-api-with-the-dashboard) + +## Dashboard and Gateway API + +Programmatically create and manage APIs, policies, keys, and developer portals using REST APIs. This method provides flexibility for automation but requires imperative scripting. + +**Use case:** Programmatic API management and basic automation needs. + +**Learn more:** +- [Dashboard API](/api-management/dashboard-configuration#exploring-the-dashboard-api) +- [Gateway API](/tyk-gateway-api) + +## Tyk Sync + +Manage API configurations declaratively using version-controlled files. Tyk Sync enables GitOps workflows by maintaining API configurations as code that can be versioned and deployed through CI/CD pipelines. + +**Use case:** GitOps workflows and teams requiring version-controlled API configurations. + +**Learn more:** +- [Tyk Sync](/api-management/automations/sync) + +## Tyk Operator + +Kubernetes-native API management using Custom Resource Definitions (CRDs). Tyk Operator provides declarative configuration with automatic drift detection and reconciliation in Kubernetes environments. + +**Use case:** Kubernetes-native environments requiring automated API lifecycle management. + +**Learn more:** +- [Tyk Operator](/api-management/automations/operator#what-is-tyk-operator) +- [Using Tyk Operator to enable GitOps](/api-management/automations) diff --git a/api-management/mdcb.mdx b/api-management/mdcb.mdx new file mode 100644 index 00000000..f713c69f --- /dev/null +++ b/api-management/mdcb.mdx @@ -0,0 +1,984 @@ +--- +title: "Tyk Multi Data Center Bridge (MDCB): Centralized API Governance Across Distributed Environments" +'og:description': "How to configure Multi Data Center Bridge" +tags: ['MDCB', 'Multi Data Center Bridge', 'Control Plane', 'Data Plane', 'Synchroniser'] +sidebarTitle: "Manage Distributed Gateways" +--- + +## Overview + +Tyk’s Multi Data Center Bridge (MDCB) is a separately licensed extension to the Tyk control plane that performs management and synchronisation of logically or geographically distributed clusters of Tyk API Gateways. We use it ourselves to support our Tyk Cloud offering. + +### Challenges in Distributed Environment + +When your users are spread geographically and want to access your APIs from different parts of the world you can optimize the performance, value and utility of your APIs by deploying API Gateways in data centers local to them. + +Single API gateway + +Having localised gateways offers benefits to you and your users, such as: + +- Reduced latency (roundtrip time) for users by accessing a local data center +- Deployment close to backend services, reducing interconnect costs and latencies +- Increased availability across your estate - if one region goes offline the rest will continue to serve users +- Compliance with data residency and sovereignty regulations + +Distributed API gateways + +This distributed architecture, however, introduces challenges for you in terms of managing the configuration, synchronisation and resilience of the Gateways in each data center. + +- How do you configure each of the Tyk API Gateways to ensure that a user can access only their authorized APIs, but from any location? +- How can you ensure that the correct APIs are deployed to the right Gateways - and kept current as they are updated? + +As the complexity of your architecture increases, this maintenance becomes an increasingly difficult and expensive manual task. + +This is where Tyk’s Multi Data Center Bridge (MDCB) comes in. + +### How does Tyk Multi Data Center Bridge help? + +The Tyk MDCB makes it possible to manage federated global deployments easily, from a central Dashboard: you can confidently deploy a multi-data center, geographically isolated set of Tyk Gateway clusters for maximum redundancy, failover, latency optimization, and uptime. + +Combining Tyk Dashboard with MDCB, you are provided with a β€œsingle pane of glass” or control plane that allows you to centrally manage multiple Tyk Gateway clusters. This has many advantages over having separate gateways and corresponding dashboard/portals, which would require manual synchronisation to roll out any changes (e.g. new APIs) across all the individual gateways. + +By deploying MDCB, API Management with Tyk becomes a service that can be easily offered to multiple teams from a centralised location. + +Distributed API Gateways with MDCB + +### How does MDCB work? + +MDCB acts as a broker between the Tyk Gateway instances that you deploy in data centers around the world. A single Control Plane (Management) Gateway is used as reference: you configure APIs, keys and quotas in one central location; MDCB looks after the propagation of these to the Data Plane (or Worker) Gateways, ensuring the synchronisation of changes. + +MDCB is extremely flexible, supporting clusters of Tyk Gateways within or across data centers - so for example two clusters within the same data center could run different configurations of APIs, users etc. + +MDCB keeps your Tyk API Gateways highly available because all the Worker Gateways, where your users access your APIs, can be configured and run independently. If the MDCB link back to the Management Gateway goes down, the Workers will continue to service API requests; when the link is back up, MDCB will automatically refresh the Workers with any changes they missed. + +Multi Data Center Bridge is down + +What happens if the worst happens and Worker Gateways fail while the link to the Control Plane is down? We’ve thought of that: Tyk will automatically configure the new Workers that spin up using the last known set of API resources in the worker’s cluster, minimizing the impact on availability of your services. + +### When might you deploy MDCB? + +#### Managing geographically distributed gateways to minimize latency and protect data sovereignty + +Consider Acme Global Bank: they have customers in the USA and the EU. Due to compliance, security and performance requirements they need to deploy their Tyk API Gateways locally in each of those regions. They need to manage the deployment and synchronisation of APIs and associated resources (e.g. keys, policies and certificates) between the data centers to ensure global service for their customers. + +Acme Global Bank without MDCB + +Tyk MDCB enables Acme Global Bank to power this architecture by creating a primary data center with all the Tyk Control Plane components and secondary (worker) data centers that act as local caches to run validation and rate limiting operations to optimize latency and performance. + +Acme Global Bank with MDCB + +#### Managing a complex deployment of services with internal and externally facing APIs + +Consider Acme Telecoms: they have a large nationally distributed workforce and complex self-hosted IT systems; are using Tyk API Gateways to deploy internal and external APIs; and have different teams managing and consuming different sets of APIs across multiple sites. They need to ensure data segregation, availability, and access for internal and external users and partners. + +Acme Telecoms without MDCB + +Combining Tyk’s built-in multi-tenancy capability with MDCB enables Acme Telecoms to set up dedicated logical gateways for different user groups and different physical gateways to guarantee data segregation, with a single management layer for operational simplicity. + +Acme Telecoms with MDCB + +### Why Choose MDCB for Your API Infrastructure? + +Beyond the two usage scenarios described here, there are many others where MDCB will provide you with the power and flexibility you need to manage your own particular situation. + +Here are some examples of the benefits that deploying Tyk MDCB can bring: + +#### Flexible architecture + +- You can control geographic distribution of traffic, restricting traffic to data centers/regions of your choice. +- You can put your Tyk API Gateways close to users, but still have a single management layer. +- You have a single, simple, point of access for configuration of your complex API infrastructure and yet deploy multiple Developer Portals, if required, to provide access to different user groups (e.g. Internal and External). +- You can physically [segment teams and environments](/api-management/multiple-environments#gateway-sharding) within a single physical data center, giving each team full control of its own API gateway and server resources without the noisy neighbors you might experience in a standard self-managed deployment. +- You can deploy gateways with whichever mix of cloud vendors you wish. +- You can mix and match cloud and on premises data centers. + +#### Improved resiliency, security and uptime + +- Each Data Plane (Worker) Gateway operates autonomously using a locally stored copy of the API resources it needs. +- The Control Plane (Management) Gateway maintains synchronisation of these configurations across your Tyk deployment via the MDCB backbone link. +- If the Management Gateway or MDCB backbone fails, the Workers will continue to handle API requests, rejecting only new authorization tokens created on other Gateways. When connectivity is restored, the Worker Gateways will hot-reload to fetch any updated configurations (e.g. new authorization tokens) from the Control Plane. +- If a Worker Gateway fails, this does not impact the operation of the others: when it comes back online, if it is unable to contact the Control Plane, it will retrieve the β€œlast good” configuration held locally. +- The MDCB backbone runs on a resilient compressed RPC channel that is designed to handle ongoing and unreliable connectivity; all traffic on the backbone is encrypted and so safer to use over the open internet or inter-data center links. +- Improved data security through separation of traffic into completely separate clusters within your network. + +#### Reduced latency + +- Deploying Data Plane (Worker) Gateways close to your geographically distributed API consumers helps reduce their perceived request latency. +- Deploying Worker Gateways close to your backend services will minimize round trip time servicing API requests. +- The Worker Gateways cache keys and other configuration locally, so all operations can be geographically localised. +- All traffic to and from one Gateway cluster will have rate limiting, authentication and authorization performed within the data center rather than β€œcalling home” to a central control point; this reduces the API request round trip time. + +#### Improved Infrastructure Management + +- Due to the shared control plane, all Worker Gateways report into a single Tyk Dashboard. This provides a simple, consistent place to manage your APIM deployment. +- This allows a shared infra team to offer API management and API Gateways as a service, globally, across multiple clouds and Self-Managed regions, from a single pane of glass. + +#### Next Steps + +- [The components of an MDCB deployment](/api-management/mdcb#mdcb-components) +- [Run an MDCB Proof of Concept](/api-management/mdcb#minimizing-latency-with-mdcb) +- [MDCB reference guide](/tyk-multi-data-centre/mdcb-configuration-options) + +## MDCB Components + +### Overview + +Here we will give an overview of the main elements of a Tyk Multi Data Center (distributed) solution, clarifying the terminology used by Tyk. +A Tyk Multi Data Center Bridge deployment + +#### Tyk Gateway +- The workhorse of any deployment, Tyk’s lightweight Open Source API gateway that exposes your APIs for consumption by your users. It is a reverse proxy that secures your APIs, manages session and policies, monitors, caches and manipulates requests/responses when needed before/after it proxies them to and from the upstream. + +#### Tyk Dashboard +- Tyk’s management platform used to control the creation of API configurations, policies and keys in a persistent manner. It provides analytic information on the traffic the Gateways have processed which includes aggregated API usage and detailed information per transaction. + +#### Tyk Multi Data Center Bridge (MDCB) +- The backbone of the distributed Tyk deployment, connecting the distributed Data Plane deployments back to the Control Plane. + +#### Tyk Pump +- Tyk’s open source analytics purger that can be used to export transaction logs from the Tyk deployment to the visualisation tool or other data store of your choice + +#### Tyk Developer Portal +- The access point for your API Consumers where you publish your API catalog(s) and they obtain API keys. + +#### Redis +- An in-memory data store used as a database, cache and message broker. We use it as pub/sub broker for inter-Gateway communication, and as a cache for API configurations, keys, certificates, and temporary store for analytics records. + +#### MongoDB/SQL +- A persistent data store for API configurations, policies, analytics and aggregated analytics, Dashboard organizations, configurations, dashboard users, portal developers and configuration. + + +### Control Plane +The Tyk Control Plane + +The Control Plane must consist of the following elements: +- **Tyk Dashboard** (used to configure and control the whole Tyk installation) +- **Tyk Gateway** (used for creation of keys and certificates, this does not service API requests; it is important to ensure there is no public access to it and it must not be sharded (tagged) as it "belongs" to the whole Tyk installation) +- **Tyk MDCB** +- **Redis** (high availability Redis data store that should be backed up in case of failure; this [document](https://redis.io/docs/management/persistence/) gives recommendation on Redis persistency) +- **MongoDB or SQL** (a persistent data store that should be deployed and set up for redundancy and high availability) + +To improve resilience and availability, multiple instances of each Tyk component should be deployed and load balanced within the Control Plane. + +#### Optional Components +- One or more **Tyk Pumps** can be deployed within the Control Plane to export analytics data (request/response logs) to your [data sink of choice](/api-management/tyk-pump#external-data-stores) for further analytics and visualisation. +- A **Tyk Developer Portal** can be added to enhance the end-user experience when accessing your APIs. + +### Data Plane +The Tyk Data Plane + +The Data Plane deployment must consist of the following elements: +- **Tyk Gateway** (one or more Gateways specifically configured as Workers) +- **Redis** (a single Redis data store shared by all Gateways in the cluster) + +To provide resilience and availability, multiple Gateways should be deployed and load balanced within the cluster. +If you want this Data Plane deployment to be resilient, available, and independent from the Control Plane during a disconnection event, it is advised to make the Redis data store persistent. + +#### Optional Components +- A **Tyk Pump** specifically configured as a [Hybrid Pump](/product-stack/tyk-charts/tyk-data-plane-chart#hybrid-pump) can be deployed with the Data Plane gateways to export analytics data (request/response logs) to your [data sink of choice](/api-management/tyk-pump#external-data-stores) for further analytics and visualisation. + +## Setup MDCB Control Plane + +The [Tyk control plane](/api-management/mdcb#control-plane) contains all the +standard components of a standard Tyk Self-Managed installation with the addition of the Multi Data Center Bridge (MDCB). + +### Installing MDCB Component On Linux +The MDCB component must be able to connect to Redis and MongoDB/PostgreSQL directly from within the Control Plane deployment. It does not require access to the Tyk Gateway(s) or Dashboard application. + +The MDCB component will however, by default, expose an RPC service on port 9091, to which the [Tyk Data Plane](/api-management/mdcb#data-plane) data centers, i.e. the worker gateway(s) that serves API traffic, will need connectivity. + +#### Prerequisites +We will assume that your account manager has provided you with a valid MDCB and Dashboard License and the command to enable you to download the MDCB package. +We will assume that the following components are up and running in your Controller DC: + +* MongoDB or SQL (check [supported versions](/planning-for-production/database-settings)) +* Redis (check [supported versions](/tyk-self-managed#redis)) +* Tyk Dashboard +* Tyk Gateway / Gateways Cluster +* Working Tyk-Pro [Self-Managed installation](/tyk-self-managed/install) + + + + + When using SQL rather than MongoDB in a production environment, we only support PostgreSQL. + + + +#### Installing using RPM and Debian packages +To download the relevant MDCB package from PackageCloud: + +```curl +curl -s https://packagecloud.io/install/repositories/tyk/tyk-mdcb-stable/script.deb.sh | sudo bash +``` + +```curl +curl -s https://packagecloud.io/install/repositories/tyk/tyk-mdcb-stable/script.rpm.sh | sudo bash +``` + +After the relevant script for your distribution has run, the script will let you know it has finished with the following message: + +`The repository is setup! You can now install packages.` + +You will now be able to install MDCB as follows: + +```curl +sudo apt-get install tyk-sink +``` + +Or + +```curl +sudo yum install tyk-sink +``` + +### Installing in a Kubernetes Cluster with our Helm Chart + +The [Tyk Control Plane](/product-stack/tyk-charts/tyk-control-plane-chart) helm chart is pre-configured to install Tyk control plane for multi data center API management from a single Dashboard with the MDCB component. + +Below is a concise instruction on how to set up an MDCB Control Plane with Redis and PostgreSQL. + +To access the comprehensive installation instructions and configuration options, please see [Tyk Control Plane Helm Chart](/product-stack/tyk-charts/tyk-control-plane-chart). + +#### Prerequisites +- [Kubernetes 1.19+](https://kubernetes.io/docs/setup/) +- [Helm 3+](https://helm.sh/docs/intro/install/) +- MDCB and Dashboard license + +#### Quick Start + +1. **Setup required credentials** + + First, you need to provide Tyk Dashboard and MDCB license, admin email and password, and API keys. We recommend to store them in secrets. + + ```bash expandable + NAMESPACE=tyk-cp + + API_SECRET=changeit + ADMIN_KEY=changeit + ADMIN_EMAIL=admin@default.com + ADMIN_PASSWORD=changeit + DASHBOARD_LICENSE=changeit + MDCB_LICENSE=changeit + SECURITY_SECRET=changeit + OPERATOR_LICENSE=changeit + + kubectl create namespace $NAMESPACE + + kubectl create secret generic my-secrets -n $NAMESPACE \ + --from-literal=APISecret=$API_SECRET \ + --from-literal=AdminSecret=$ADMIN_KEY \ + --from-literal=DashLicense=$DASHBOARD_LICENSE \ + --from-literal=OperatorLicense=$OPERATOR_LICENSE + + kubectl create secret generic mdcb-secrets -n $NAMESPACE \ + --from-literal=MDCBLicense=$MDCB_LICENSE \ + --from-literal=securitySecret=$SECURITY_SECRET + + kubectl create secret generic admin-secrets -n $NAMESPACE \ + --from-literal=adminUserFirstName=Admin \ + --from-literal=adminUserLastName=User \ + --from-literal=adminUserEmail=$ADMIN_EMAIL \ + --from-literal=adminUserPassword=$ADMIN_PASSWORD + ``` + +2. **Install Redis (if you don't already have Redis installed)** + + If you do not already have Redis installed, you may use these charts provided by Bitnami. + + ```bash + helm upgrade tyk-redis oci://registry-1.docker.io/bitnamicharts/redis -n $NAMESPACE --install --version 19.0.2 + ``` + Follow the notes from the installation output to get connection details and password. The DNS name of your Redis as set by Bitnami is `tyk-redis-master.tyk-cp.svc:6379` (Tyk needs the name including the port) + + The Bitnami chart also creates a secret `tyk-redis` which stores the connection password in `redis-password`. We will make use of this secret in installation later. + + + +Ensure that you are installing Redis versions that are supported by Tyk. Please consult the list of [supported versions](/tyk-self-managed#redis) that are compatible with Tyk. + + + +3. **Install PostgreSQL (if you don't already have PostgreSQL installed)** + + If you do not already have PostgreSQL installed, you may use these charts provided by Bitnami. + + ```bash + helm upgrade tyk-postgres oci://registry-1.docker.io/bitnamicharts/postgresql --set "auth.database=tyk_analytics" -n $NAMESPACE --install --version 14.2.4 + ``` + + Follow the notes from the installation output to get connection details. + + We require the PostgreSQL connection string for Tyk installation. This can be stored in a secret and will be used in installation later. + + ```bash + POSTGRESQLURL=host=tyk-postgres-postgresql.$NAMESPACE.svc\ port=5432\ user=postgres\ password=$(kubectl get secret --namespace $NAMESPACE tyk-postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d)\ database=tyk_analytics\ sslmode=disable + + kubectl create secret generic postgres-secrets -n $NAMESPACE --from-literal=postgresUrl="$POSTGRESQLURL" + ``` + + + +Ensure that you are installing PostgreSQL versions that are supported by Tyk. Please consult the list of [supported versions](/api-management/dashboard-configuration#supported-database) that are compatible with Tyk. + + + +4. **Install Tyk Control Plane** + + ```bash + helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ + + helm repo update + + helm upgrade tyk-cp tyk-helm/tyk-control-plane -n $NAMESPACE \ + --install \ + --set global.adminUser.useSecretName=admin-secrets \ + --set global.secrets.useSecretName=my-secrets \ + --set tyk-mdcb.mdcb.useSecretName=mdcb-secrets \ + --set global.redis.addrs="{tyk-redis-master.$NAMESPACE.svc:6379}" \ + --set global.redis.passSecret.name=tyk-redis \ + --set global.redis.passSecret.keyName=redis-password \ + --set global.postgres.connectionStringSecret.name=postgres-secrets \ + --set global.postgres.connectionStringSecret.keyName=postgresUrl + ``` + +5. **Done!** + + Now Tyk Dashboard and Tyk MDCB should be accessible through service `dashboard-svc-tyk-cp-tyk-dashboard` at port `3000` and `mdcb-svc-tyk-cp-tyk-mdcb` at port `9091` respectively. You can login to Dashboard using the admin email and password to start managing APIs. + + You can use the MDCB connection details included in the installation output, to install the [MDCB Data Plane](/api-management/mdcb#setup-mdcb-data-plane). + +### Configuration +If you install MDCB component with package, modify your `/opt/tyk-sink/tyk_sink.conf` file as follows: + +#### Configuration Example +```json +{ + "listen_port": 9091, + "healthcheck_port": 8181, + "server_options": { + "use_ssl": false, + "certificate": { + "cert_file": "", + "key_file": "" + }, + "min_version": 771 + }, + "storage": { + "type": "redis", + "host": "localhost", + "port": 6379, + "username": "", + "password": "", + "enable_cluster": false, + "redis_use_ssl": false, + "redis_ssl_insecure_skip_verify": false + }, + "basic-config-and-security/security": { + "private_certificate_encoding_secret": "" + }, + "hash_keys": true, + "forward_analytics_to_pump": true, + "ignore_tag_prefix_list": [ + + ], + "analytics": { + "mongo_url": "mongodb://localhost/tyk_analytics", + "mongo_use_ssl": false, + "mongo_ssl_insecure_skip_verify": false + }, + "license": "MDCB_LICENSE_KEY" +} +``` + + + +From MDCB 2.0+, you can choose between Mongo or SQL databases to setup your `analytics` storage. In order to setup your PostgreSQL storage, you can use the same configuration from your [Tyk Dashboard main storage](/planning-for-production/database-settings#postgresql). + +For example, to set up a `postgres` storage the `analytics` configurations would be: + +```json +{ +... + ... + "analytics": { + "type": "postgres", + "connection_string": "user=postgres_user password=postgres_password database=dbname host=potgres_host port=postgres_port", + "table_sharding": false + }, +} +``` +This storage will work for fetching your organization data (APIs, Policies, etc) and for analytics. + + + +You should now be able to start the MDCB service, check that it is up and running and ensure that the service starts on system boot: + +```console +sudo systemctl start tyk-sink +``` + + +```console +sudo systemctl enable tyk-sink +``` expandable + +### Health check + +It is possible to perform a health check on the MDCB service. This allows you to determine if the service is running, so is useful when using MDCB with load balancers. + +Health checks are available via the HTTP port. This is defined by `http_port` configuration setting and defaults to `8181`. Do **not** use the standard MDCB listen port (`listen_port`) for MDCB health checks. + +From MDCB v2.7.0, there are 2 health check services available: +1. `/liveness` endpoint returns a `HTTP 200 OK` response when the service is operational. +2. `/readiness` endpoint returns a `HTTP 200 OK` response when MDCB is ready to accept requests. It ensures that dependent components such as Redis and data store are connected, and the gRPC server is ready for connection. + +See [MDCB API](/tyk-mdcb-api) for details of the endpoints. + +In MDCB v2.6.0 or earlier, MDCB only offers one health check endpoint at `/health` via the port defined by the `healthcheck_port` configuration setting. The default port is `8181`. The `/health` endpoint is also available on v2.7.0 or later for backward compatibility. + +To use the health check service, call the `/health` endpoint i.e. `http://my-mdcb-host:8181/health`. This will return a `HTTP 200 OK` response if the service is running. + +Please note that an HTTP 200 OK response from the `/health` endpoint merely indicates that the MDCB service is operational. However, it is important to note that the service may not yet be ready for use if it is unable to establish a connection with its dependent components (such as Redis and Data store) or if they are offline. Upgrade to v2.7.0 and later to have more accurate health checking. + +### Troubleshooting + +#### Check that the MDCB service is running + +```console +sudo systemctl status tyk-sink +``` + +Should Return: + +```console +tyk-sink.service - Multi Data Center Bridge for the Tyk API Gateway + + Loaded: loaded (/usr/lib/systemd/system/tyk-sink.service; enabled; vendor preset: disabled) + + Active: active (running) since Thu 2018-05-03 09:39:37 UTC; 3 days ago + Main PID: 1798 (tyk-sink) + + CGroup: /system.slice/tyk-sink.service + + └─1798 /opt/tyk-sink/tyk-sink -c /opt/tyk-sink/tyk_sink.conf +``` + +#### Check that MDCB is listening on port 9091 + +```console +sudo netstat -tlnp +``` + +Should Return: + +``` +Active Internet connections (only servers) +Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name +... +tcp6 0 0 :::9091 :::* LISTEN 1798/tyk-sink +``` + +#### Check the logs for MDCB + +```{.copyWrapper} +> sudo journalctl -u tyk-sink +``` + +Add the `-f` flag to follow the log. The command should return output similar to this: + +```console +-- Logs begin at Thu 2018-05-03 09:30:56 UTC, end at Mon 2018-05-07 08:58:23 UTC. -- +May 06 11:50:37 master tyk-sink[1798]: time="2018-05-06T11:50:37Z" level=info msg="RPC Stats:{\"RPCCalls\":0,\"RPCTime\":0,\"Byte +May 06 11:50:38 master tyk-sink[1798]: time="2018-05-06T11:50:38Z" level=info msg="RPC Stats:{\"RPCCalls\":0,\"RPCTime\":0,\"Byte +... +May 06 11:50:42 master tyk-sink[1798]: time="2018-05-06T11:50:42Z" level=info msg="Ping!" +``` + +#### Check MDCB configurations + +From MDCB v2.7.0, a secured HTTP endpoint `/config` can be enabled that allows you to query configuration of MDCB. + +To enable the secured HTTP endpoint, make sure you have the following in your `/opt/tyk-sink/tyk_sink.conf` config file. + +```json +{ + "security": { + "enable_http_secure_endpoints": true, + "secret": "" + }, + "http_server_options": { + "use_ssl": true, + "certificate": { + "cert_file": ..., + "key_file": ..., + "min_version": ... + } + } +} +``` + +Subsequently, you can issue a request to the `/config` endpoint to return a json representation of your MDCB config: + +```bash +curl -H "x-tyk-authorization: " https://my-mdcb-host:8181/config +``` + +Alternatively, you can issue a request to the `/env` endpoint to return your MDCB config in the form of environment variables settings: + +```bash +curl -H "x-tyk-authorization: " https://my-mdcb-host:8181/env +``` expandable + +### Enabling MDCB on Organization Object on Tyk Dashboard + +Before a worker gateway can connect to MDCB, it is important to enable the organization that owns all the APIs to be distributed to be allowed to utilize Tyk MDCB. To do this, the organization record needs to be modified with two flags using the [Tyk Dashboard Admin API](/dashboard-admin-api). + +To make things easier, we will first set a few [environment variables](/tyk-oss-gateway/configuration): + +1. `export DASH_ADMIN_SECRET=` + +You can find `` in `tyk_analytics.conf` file under `admin_secret` field or `TYK_DB_ADMINSECRET` environment variable. + +2. `export DASH_URL=` + +This is the URL you use to access the Dashboard (including the port if not using the default port). + +3. `export ORG_ID=` + +You can find your organization id in the Dashboard, under your user account details. + +Org ID + +4. Send a GET request to the Dashboard API to `/admin/organisations/$ORG_ID` to retrieve the organization object. In the example below, we are redirecting the output json to a file `myorg.json` for easy editing. + +```curl +curl $DASH_URL/admin/organisations/$ORG_ID -H "Admin-Auth: $DASH_ADMIN_SECRET" | python -mjson.tool > myorg.json +``` +5. Open `myorg.json` in your favorite text editor and add the following fields as follows. +New fields are between the `...` . + +```json {linenos=table,hl_lines=["5-12"],linenostart=1} +{ + "_id": "55780af69b23c30001000049", + "owner_slug": "portal-test", + ... + "hybrid_enabled": true, + "event_options": { + "key_event": { + "webhook": "https://example.com/webhook", + "email": "user@example.com", + "redis": true + }, + }, + ... + "apis": [ + { + "api_human_name": "HttpBin (again)", + "api_id": "2fdd8512a856434a61f080da67a88851" + } + ] +} +``` expandable + +In the example above it can be seen that the `hybrid_enabled` and `event_options` configuration fields have been added: + +- `hybrid_enabled:` Allows a worker gateway to login as an organization member into MDCB. +- `event_options:` The `event_options` object is optional. By default the update and removal of Redis keys (hashed and unhashed), API Definitions and policies are propagated to various instance zones. The `event_options` object contains a `key_event` object that allows configuration of the following additional features: + + - event notification mechanism for all Redis key (hashed and unhashed) events. Events can be notified via webhook by setting the `webhook` property to the value of the webhook URL. Similarly, events can be notified via email by setting the `email` property to the value of the target email address. + - enable propagation of events for when an OAuth token is revoked from Dashboard by setting the `redis` property to `true`. + + The `event_options` in the example above enables the following functionality: + + - events are propagated when OAuth tokens are revoked from Dashboard since `redis` is `true` + - events associated with Redis keys (hashed and unhashed) and revoking OAuth tokens via Dashboard are sent to webhook `https://example.com/webhook` and email address `user@example.com` + +6. Update your organization with a PUT request to the same endpoint, but this time, passing in your modified `myorg.json` file. + +```curl +curl -X PUT $DASH_URL/admin/organisations/$ORG_ID -H "Admin-Auth: $DASH_ADMIN_SECRET" -d @myorg.json +``` + +This should return: + +```json +{"Status":"OK","Message":"Org updated","Meta":null} +``` expandable + +## Setup MDCB Data Plane + +You may configure an unlimited number of [Tyk Data Planes](/api-management/mdcb#data-plane) containing Worker Gateways for ultimate High Availablity (HA). We recommend that you deploy your worker gateways as close to your upstream services as possible in order to reduce latency. + +It is a requirement that all your Worker Gateways in a Data Plane data center share the same Redis DB in order to take advantage of Tyk's DRL and quota features. +Your Data Plane can be in the same physical data center as the Control Plane with just a logical network separation. If you have many Tyk Data Planes, they can be deployed in a private-cloud, public-cloud, or even on bare-metal. + +### Installing in a Kubernetes Cluster with our Helm Chart + +The [Tyk Data Plane](/product-stack/tyk-charts/tyk-data-plane-chart) helm chart is pre-configured to install Tyk Gateway and Tyk Pump that connects to MDCB or Tyk Cloud, our SaaS MDCB Control Plane. After setting up Tyk Control Plane with Helm Chart, obtain the required connection details from installation output and configure data plane chart as below. For Tyk Cloud users, following [Tyk Cloud instructions](/tyk-cloud#deploy-hybrid-gateways) to deploy your hybrid gateways. + +#### Prerequisites + +* [Kubernetes 1.19+](https://kubernetes.io/docs/setup/) +* [Helm 3+](https://helm.sh/docs/intro/install/) +* Connection details to remote control plane from the tyk-control-plane installation output. + +The following quick start guide explains how to use the [Tyk Data Plane Helm chart](/product-stack/tyk-charts/tyk-data-plane-chart) to configure Tyk Gateway that includes: +- Redis for key storage +- Tyk Pump to send analytics to Tyk Control Plane and Prometheus + +At the end of this quickstart Tyk Gateway should be accessible through service `gateway-svc-tyk-dp-tyk-gateway` at port `8080`. Pump is also configured with Hybrid Pump which sends aggregated analytics to MDCB, and Prometheus Pump which expose metrics locally at `:9090/metrics`. + +1. **Set connection details** + + Set the below environment variables and replace values with connection details to your MDCB control plane. See [Tyk Data Plane](/product-stack/tyk-charts/tyk-data-plane-chart#obtain-remote-control-plane-connection-details-from-tyk-control-plane-chart) documentation on how to get the connection details. + + ```bash + USER_API_KEY=9d20907430e440655f15b851e4112345 + ORG_ID=64cadf60173be90001712345 + MDCB_CONNECTIONSTRING=mdcb-svc-tyk-cp-tyk-mdcb.tyk-cp.svc:9091 + GROUP_ID=your-group-id + MDCB_USESSL=false + ``` + +2. **Then use Helm to install Redis and Tyk** + + ```bash + NAMESPACE=tyk-dp + APISecret=foo + + helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ + helm repo update + + helm upgrade tyk-redis oci://registry-1.docker.io/bitnamicharts/redis -n $NAMESPACE --create-namespace --install + + helm upgrade tyk-dp tyk-helm/tyk-data-plane -n $NAMESPACE --create-namespace \ + --install \ + --set global.remoteControlPlane.userApiKey=$USER_API_KEY \ + --set global.remoteControlPlane.orgId=$ORG_ID \ + --set global.remoteControlPlane.connectionString=$MDCB_CONNECTIONSTRING \ + --set global.remoteControlPlane.groupID=$GROUP_ID \ + --set global.remoteControlPlane.useSSL=$MDCB_USESSL \ + --set global.secrets.APISecret="$APISecret" \ + --set global.redis.addrs="{tyk-redis-master.$NAMESPACE.svc.cluster.local:6379}" \ + --set global.redis.passSecret.name=tyk-redis \ + --set global.redis.passSecret.keyName=redis-password + ``` + +3. **Done!** + + Now Tyk Gateway should be accessible through service `gateway-svc-tyk-dp-tyk-gateway` at port `8080`. Pump is also configured with Hybrid Pump which sends aggregated analytics to MDCB, and Prometheus Pump which expose metrics locally at `:9090/metrics`. + + For the complete installation guide and configuration options, please see [Tyk Data Plane Chart](/product-stack/tyk-charts/tyk-data-plane-chart). + +### Configuring an existing Tyk Gateway +If you have Redis and a working Tyk Gateway deployed, follow below steps to configure your gateways to work in RPC mode. + + + +If you have deployed Gateway with `tyk-data-plane` Chart, you don't need to go through following steps to configure Tyk Gateway. The necessary configurations has been set in `tyk-data-plane` chart templates. + + + +#### Prerequisites +- Redis +- A working headless/open source Tyk Gateway deployed + +#### Worker Gateway Configuration + +Modify the Tyk Gateway configuration (`tyk.conf`) as follows: +`"use_db_app_configs": false,` + +Next, we need to ensure that the policy loader and analytics pump use the RPC driver: + +```{.json} +"policies": { + "policy_source": "rpc", + "policy_record_name": "tyk_policies" +}, +"analytics_config": { + "type": "rpc", + ... // remains the same +}, +``` + +Lastly, we add the sections that enforce the Worker mechanism: + +```{.json} +"slave_options": { + "use_rpc": true, + "rpc_key": "{ORGID}", + "api_key": "{APIKEY}", + "connection_string": "{MDCB_HOSTNAME:9091}", + "enable_rpc_cache": true, + "bind_to_slugs": false, + "group_id": "{ny}", + "use_ssl": false, + "ssl_insecure_skip_verify": true +}, +"auth_override": { + "force_auth_provider": true, + "auth_provider": { + "name": "", + "storage_engine": "rpc", + "meta": {} + } +} +``` + + +if you set `analytics_config.type` to `rpc` - make sure you don't have your Tyk Pump configured to send analytics via the `hybrid` Pump type. + + + + +As an optional configuration you can use `key_space_sync_interval` to set the period's length in which the gateway will check for changes in the key space, if this value is not set then by default it will be 10 seconds. + + +The most important elements here are: + +| Field | Description | +| :--------------- | :---------------- | +|`api_key` |This the API key of a user used to authenticate and authorize the Gateway's access through MDCB. The user should be a standard Dashboard user with minimal privileges so as to reduce risk if compromised. The suggested security settings are `read` for `Real-time notifications` and the remaining options set to `deny`.| +|`group_id` |This is the "zone" that this instance inhabits, e.g. the cluster/data center the gateway lives in. The group ID must be the same across all the gateways of a data center/cluster which are also sharing the same Redis instance. This id should also be unique per cluster (otherwise another gateway's cluster can pick up your keyspace events and your cluster will get zero updates). +|`connection_string` |The MDCB instance or load balancer.| +|`bind_to_slugs` | For all Tyk installations except for Tyk Classic Cloud this should be set to false.| + +Once this is complete, you can restart the Tyk Gateway in the Data Plane, and it will connect to the MDCB instance, load its API definitions, and is ready to proxy traffic. + +## Minimizing latency with MDCB + +### Overview + +As described [previously](/api-management/mdcb#managing-geographically-distributed-gateways-to-minimize-latency-and-protect-data-sovereignty), Acme Global Bank has operations and customers in both the EU and USA. + +To decrease the latency in response from their systems and to ensure that data remains in the same legal jurisdiction as the customers (data residency), they have deployed backend (or, from the perspective of the API gateway, β€œupstream”) services in two data centers: one in the US, the other in the EU. + +Without a dedicated solution for this multi-region use case, Acme Global Bank would deploy a Tyk Gateway cluster in each data center and then have the operational inconvenience of maintaining two separate instances of Tyk Dashboard to configure, secure and publish the APIs. + +By using Tyk's Multi-Data Center Bridge (MDCB), however, they are able to centralise the management of their API Gateways and gain resilience against failure of different elements of the deployment - data or control plane - improving the availability of their public APIs. + +In this example we will show you how to create the Acme Global Bank deployment using our example Helm charts. + +MDCB Proof of Concept - Acme Global Bank + +**Step-by-step instructions to deploy the Acme Global Bank scenario with Kubernetes in a public cloud (here we’ve used Google Cloud Platform):** + +### Pre-requisites and configuration + +1. What you need to install/set-up + - Tyk Pro license (Dashboard and MDCB keys - obtained from Tyk) + - Access to a cloud account of your choice, e.g. GCP + - You need to grab this Tyk Demo repository: [GitHub - TykTechnologies/tyk-k8s-demo](https://github.com/TykTechnologies/tyk-k8s-demo) + - You need to install `helm`, `jq`, `kubectl` and `watch` + +2. To configure GCP + - Create a GCP cluster + - Install the Google Cloud SDK + - install `gcloud` + - `./google-cloud-sdk/install.sh` + - Configure the Google Cloud SDK to access your cluster + - `gcloud auth login` + - `gcloud components install gke-gcloud-auth-plugin` + - `gcloud container clusters get-credentials <> β€”zone <>β€”project <>` + - Verify that everything is connected using `kubectl` + - `kubectl get nodes` + +3. You need to configure the Tyk build + - Create a `.env` file within tyk-k8s-demo based on the provided `.env.example` file + - Add the Tyk license keys to your `.env`: + - `LICENSE=` + - `MDCB_LICENSE=` + +### Deploy Tyk Stack to create the Control and Data Planes + +1. Create the Tyk Control Plane + - `./up.sh -r redis-cluster -e load-balancer tyk-cp` + +*Deploying the Tyk Control Plane* +Tyk Control Plane Deployed + +2. Create two logically-separate Tyk Data Planes (Workers) to represent Acme Global Bank’s US and EU operations using the command provided in the output from the `./up.sh` script: + - `TYK_WORKER_CONNECTIONSTRING= TYK_WORKER_ORGID= TYK_WORKER_AUTHTOKEN= TYK_WORKER_USESSL=false ./up.sh --namespace tyk-worker` + +Note that you need to run the same command twice, once setting `` to `tyk-worker-us`, the other to `tyk-worker-eu` (or namespaces of your choice) + +*Deploying the `tyk-worker-us` namespace (Data Plane #1)* +Deploying the tyk-worker-us namespace + +*Deploying the `tyk-worker-eu` namespace (Data Plane #2)* +Deploying the tyk-worker-eu namespace + +3. Verify and observe the Tyk Control and Data Planes + - Use `curl` to verify that the gateways are alive by calling their `/hello` endpoints + +observe Tyk K8s namespace console output + + - You can use `watch` to observe each of the Kubernetes namespaces + +*`tyk-cp` (Control Plane)* +Control Plane + +*`tyk-worker-us` (Data Plane #1)* +Data Plane #1 + +*`tyk-worker-eu` (Data Plane #2)* +Data Plane #2 + +### Testing the deployment to prove the concept +As you know, the Tyk Multi Data Center Bridge provides a link from the Control Plane to the Data Plane (worker) gateways, so that we can control all the remote gateways from a single dashboard. + +1. Access Tyk Dashboard + - You can log into the dashboard at the external IP address reported in the watch window for the Control Plane - in this example it was at `34.136.51.227:3000`, so just enter this in your browser + - The user name and password are provided in the `./up.sh` output: + - username: `default@example.com` + - password: `topsecretpassword` (or whatever you’ve configured in the `.env` file) + +Tyk Dashboard login + +2. Create an API in the dashboard, but don’t secure it (set it to `Open - keyless`); for simplicity we suggest a simple pass-through proxy to `httbin.org`. +3. MDCB will propagate this API through to the workers - so try hitting that endpoint on the two Data Plane gateways (their addresses are given in the watch windows: for example `34.173.240.149:8081` for my `tyk-worker-us` gateway above). +4. Now secure the API from the dashboard using the Authentication Token option. You’ll need to set a policy for the API and create a key. +5. If you try to hit the API again from the workers, you’ll find that the request is now rejected because MDCB has propagated out the change in authentication rules. Go ahead and add the Authentication key to the request header… and now you reach `httpbin.org` again. You can see in the Dashboard’s API Usage Data section that there will have been success and error requests to the API. +6. OK, so that’s pretty basic stuff, let’s show what MDCB is actually doing for you… reset the API authentication to be `Open - keyless` and confirm that you can hit the endpoint without the Authentication key from both workers. +7. Next we’re going to experience an MDCB outage - by deleting its deployment in Kubernetes: +
`kubectl delete deployment.apps/mdcb-tyk-cp-tyk-pro -n tyk` +8. Now there's no connection from the data placne to the control plane, but try hitting the API endpoint on either worker and you’ll see that they continue serving your users' requests regardless of their isolation from the Control Plane. +9. Back on the Tyk Dashboard make some changes - for example, re-enable Authentication on your API, add a second API. Verify that these changes **do not** propagate through to the workers. +10. Now we’ll bring MDCB back online with this command: +
`./up.sh -r redis-cluster -e load-balancer tyk-cp` +11. Now try hitting the original API endpoint from the workers - you’ll find that you need the Authorization key again because MDCB has updated the Data Planes with the new config from the Control Plane. +12. Now try hitting the new API endpoint - this will also have automatically been propagated out to the workers when MDCB came back up and so is now available for your users to consume. + +Pretty cool, huh? + +There’s a lot more that you could do - for example by deploying real APIs (after all, this is a real Tyk deployment) and configuring different Organization Ids for each Data Plane to control which APIs propagate to which workers (allowing you to ensure data localisation, as required by the Acme Global Bank). + +### Closing everything down +We’ve provided a simple script to tear down the demo as follows: +1. `./down.sh -n tyk-worker-us` +2. `./down.sh -n tyk-worker-eu` +3. `./down.sh` + +**Don’t forget to tear down your clusters in GCP if you no longer need them!** + +## Synchroniser feature with MDCB + +### Overview + +In order to process API requests the worker Gateways need resources such as API keys, certificates, and OAuth clients. To ensure high availability these resources need to be synchronised in worker Gateways. + +Prior to Tyk Gateway v4.1, the API keys, certificates and OAuth clients required by worker Gateways were synchronised from the controller Gateway on-demand. With Gateway v4.1 and MDCB v2.0.3 we introduced a new configurable option that user may choose to have proactive synchronisation of these resources to the worker Gateways when they start up. + +This change improves resilience in case the MDCB link or controller Gateway is unavailable, because the worker Gateways can continue to operate independently using the resources stored locally. There is also a performance improvement, with the worker Gateways not having to retrieve resources from the controller Gateway when an API is first called. + +Changes to keys, certificates and OAuth clients are still synchronised to the worker Gateways from the controller when there are changes and following any failure in the MDCB link. + +### How does worker Gateways get resources from MDCB control plane? + +**Without Synchroniser** + +If [Synchroniser](/tyk-multi-data-centre/mdcb-configuration-options#sync_worker_configenabled) is disabled, the resources were pulled by the worker Gateways on-demand and not in advance. It means that first it checks if the resource lives in the local Redis and if it doesn’t exist then it tries to pull it from the control plane to store it locally. + +Every time that a key is updated or removed the control plane emits a signal to all the cluster gateways to update the key accordingly. + +Considerations: +This introduces a single point of failure. When the MDCB or controller Gateway in the control plane fails then the worker Gateways are also affected. + +Without Synchroniser + +**With Synchroniser** + +If [Synchroniser](/tyk-multi-data-centre/mdcb-configuration-options#sync_worker_configenabled) is enabled, API keys, certificates and OAuth clients are synchronised and stored in the local redis server in advance. When one of those resources is created, modified or deleted, a signal will be emitted which allows the worker Gateways to respond accordingly. The transmitted information includes type of resource, action (create, update, delete), if hashed (in the case of keys), and resource ID so the changes are applied in the worker Gateways accordingly. + +Considerations: +- Size of local Redis storage: If there are a lot of keys / resources to be synchronised this will increase the size of local Redis storage. The data stored in Redis, including keys, OAuth clients, and certificates, is passed to the Redis instance of each data plane. This is a characteristic of the synchronisation mechanism and occurs regardless of whether these elements are being actively used on a given data plane. Keep in mind that even if certain resources are not being utilized in a specific data plane, they are still stored and maintained in sync by the Multi Data Center Bridge (MDCB). Therefore, if your system has a large volume of keys, OAuth clients, and certificates, this could increase the storage requirements and network traffic between your data planes. It's essential to consider these factors when designing and scaling your system. +- Data residency: The synchronization of resources does not support the application of this feature to specific groups. Instead, all keys, oauth-clients, etc. will be propagated to all Redis instances in the worker Gateways, without any differentiation based on groups. This should be considered for those customers who have a single control plane but multiple clusters of worker Gateways connected. In this scenario, all Redis instances in the Worker Gateways will receive all the keys. This aspect should be taken into account if you have specific data residency requirements. + +With Synchroniser + +### Configuring the Synchroniser for Tyk Self Managed + + + +The synchroniser feature is disabled by default. To enable it, please configure both the worker Gateways and MDCB control plane accordingly. + + + +1. **Worker Gateway configuration** + + First, configure the worker Gateway to enable synchroniser: + + `"slave_options":{ "synchroniser_enabled":true }` + + Please see [Gateway configuration options](/tyk-oss-gateway/configuration#slave_optionssynchroniser_enabled) for reference. + + To configure how often the worker Gateways read signals from MDCB control plane: + + `"slave_options":{ "key_space_sync_interval": 10 }` + + It configures the interval (in seconds) that the worker Gateway will take to check if there are any changes. If this value is not set then it will default to 10 seconds. + + Please see [Gateway configuration options](/tyk-oss-gateway/configuration#slave_optionskey_space_sync_interval) for reference. + + If you are running a cluster of Gateways, you must have a _GroupID_ configured for synchronisation to work properly and propagate keys. + + `"slave_options":{ "group_id": "FOOBAR" }` + + + Please see [Gateway configuration options](/tyk-oss-gateway/configuration#slave_optionsgroup_id) for reference + +2. **Control Plane configuration** + + Secondly, configure the control plane. The most simple configuration to enable this feature in the MDCB config file is: + + - MDCB: + + `"sync_worker_config":{ "enabled":true }` + + Please see [MDCB configuration options](/tyk-multi-data-centre/mdcb-configuration-options#sync_worker_config) for reference. + + If API keys were used and hash key is disabled, please also set these additional configurations for the following components: + + - MDCB: + + `"sync_worker_config":{ "enabled":true, "hash_keys": false }, "hash_keys": false` + + - Dashboard: + + `"hash_keys": false` + + - Controller Gateway: + + `"hash_keys": false` + + If certificates were used, please also set these additional configurations: + + - MDCB + + Set `"security.private_certificate_encoding_secret"` with the certificate encoding secret. This is required because MDCB would decode the certificate first before propagating it to worker gateways. The worker Gateways could encode the certificate with their own secret. + + Please see [MDCB configuration options](/tyk-multi-data-centre/mdcb-configuration-options#securityprivate_certificate_encoding_secret) for reference + +### Configuring the Synchroniser for Tyk Cloud + +Please [submit a support ticket](https://support.tyk.io/hc/en-gb) to us if you want to enable Synchroniser for your Tyk Cloud deployment. + +### Troubleshooting + +1. **How do I know when synchronisation happened?** + + You could check the MDCB log message to know about when synchronisation started and finished: + + ``` + Starting oauth clients sync worker for orgID... + Starting keys sync worker for orgID... + Starting keys sync worker for orgID... + + Sync APIKeys worker for orgID:... + Sync Certs worker for orgID:... + Sync oauth worker for orgID:... + ``` + +2. **Can I trigger a re-synchronisation?** + + Synchronisation will be triggered once the Time To Live (TTL) of a worker Gateway has expired. The default expiry duration is 3 minutes. The Time To Live (TTL) value can be set via [sync_worker_config.group_key_ttl](/tyk-multi-data-centre/mdcb-configuration-options#sync_worker_configgroup_key_ttl) diff --git a/api-management/migrate-from-tyk-classic.mdx b/api-management/migrate-from-tyk-classic.mdx new file mode 100644 index 00000000..077a1531 --- /dev/null +++ b/api-management/migrate-from-tyk-classic.mdx @@ -0,0 +1,197 @@ +--- +title: "Migrating from Tyk Classic APIs" +'og:description': "API Migration: Converting Tyk Classic APIs to Tyk OAS Format" +tags: ['Tyk OAS API', 'Tyk Classic API', 'Migrate', 'Convert', 'Migration', 'Tyk Classic', 'Tyk OAS', 'API definition'] +sidebarTitle: "Migrate from Tyk Classic" +--- + +## Overview + +From Tyk 5.8.0, you can convert your existing [Tyk Classic APIs](/api-management/gateway-config-tyk-classic) to the recommended [Tyk OAS API](/api-management/gateway-config-tyk-oas) format. + +The API Migration feature provides a powerful way to convert your existing Tyk Classic API definitions to the newer Tyk OAS format with various options to ensure a smooth transition. We've built support into the Tyk Dashboard's [API Designer](/api-management/migrate-from-tyk-classic#using-the-api-designer) to convert individual Tyk Classic APIs one by one. The Tyk Dashboard API [migrate endpoint](/api-management/migrate-from-tyk-classic#using-the-migration-api) allows you to migrate individual APIs or perform bulk migrations. + +### Features of Tyk Dashboard's Migration Feature + +- **Flexibility**: Migrate APIs using the API Designer or using the Tyk Dashboard API +- **Bulk Migration**: Convert multiple APIs in a single operation via the Dashboard API +- **Risk Mitigation**: Test migrations before applying changes +- **Phased Implementation**: Stage and test migrations before final deployment +- **Detailed Reporting**: Get comprehensive success, failure, and skip information + +## Migration Modes + +Tyk Dashboard supports four different migration modes to suit your needs: + +1. **Dry Run Mode**: Simulates the conversion process without making changes. This mode returns the converted API definitions in Tyk OAS format for your review, allowing you to verify the conversion before committing to it. + +2. **Stage Mode**: Perform a phased conversion by creating staged copies of your APIs in Tyk OAS format alongside the existing Tyk Classic APIs. You can then thoroughly test that the migrated APIs will behave as expected without affecting production traffic. When you are happy, you can **promote** the staged APIs. + + The **staged** API will be the same as the original Tyk Classic, with the following modifications to allow it to coexist: + + - The API ID will have the `staging-` prefix added + - `[staged] ` prefix will be added to the API name + - The listen path will be modified by the addition of the `/tyk-staging` prefix + - A reference will be added to link the staged API to the original Tyk Classic API's ID + +3. **Promote Mode**: Promote previously staged APIs, replacing their Tyk Classic counterparts. This process removes the staging prefixes from ID, name, and listen path. It then replaces the original Tyk Classic API with the Tyk OAS version - the Tyk OAS API will inherit both the API ID and database ID of the original API. + +4. **Direct Mode**: Directly migrates APIs from Tyk Classic to Tyk OAS format without staging; typically this will be used if testing was performed on a **dry run** copy of the original. This mode will also replace the original Tyk Classic API with the Tyk OAS version - the Tyk OAS API will inherit both the API ID and database ID of the original API. + + + + + Note that both Promote and Direct operations are destructive. The converted API will replace the original in the API database, inheriting both API Id and database ID. + + + +## Using the API Designer + +The API Designer provides a simple interface for migrating your Tyk Classic APIs to Tyk OAS, offering both **staged** and **direct** conversions. + +1. **Back Up Your APIs:** + + First ensure that you have taken a backup of your API, in case of accidental deletion - the direct and promote operations are destructive and will permanently delete the original Tyk Classic API definition from your database. You can export the API definition using the **Actions** menu or, if you want to backup all APIs you can do so via the Tyk Dashboard API by following [these instructions](/developer-support/upgrading#backup-apis-and-policies). + +2. **Start API Migration:** + + Find the API that you want to convert in the list of **Created APIs**. You can use the **Select API type** filter to show just Tyk Classic APIs. + + Applying a filter to see just the Tyk Classic APIs + + From the **Actions** menu select **Convert to Tyk OAS**. You can also find this option in the **Actions** menu within the Tyk Classic API Designer. + + Convert an API to Tyk OAS + + This will open the **Convert to Tyk OAS** mode selector. Choose whether to **stage** the conversion or perform a **direct** migration, then select **Convert API**. + + Choosing the migration path: staged or direct + +5. **Staging the API:** + + If you selected **stage** you'll be taken to the API Designer for the newly created staged Tyk OAS API. Note the prefixes that have been applied to the API name, API ID and API base path. + + The staged Tyk OAS API with prefixes + + For the staged API, you can now validate the configuration, make any required modifications and test that behavior is as expected. + +6. **Promote the Stage API:** + + When you are ready, you can promote the staged API to complete migration by selecting **Promote staged API** from the API Designer's **Actions** menu. Note that the **promote** option is not available from the API list, only within the API Designer for the specific API. This is to protect against accidentally promoting the wrong API. + + Promoting a staged API +
+ + Confirm the promotion +
+ + + +There is a known issue in Tyk 5.8.0 that the cancel button does not work in the promotion confirmation window, so once you have selected Promote staged API you will have to hit back in your browser if you do not want to complete the migration. This will be addressed in the next patch. + + + + +7. **Final Stage:** + + Following promotion, or if you selected **direct** conversion, you'll be taken to the API Designer for the newly created Tyk OAS API. Note that this has inherited the API ID from your Tyk Classic API and has replaced the Tyk Classic in your API database. + + Migration complete! + +## Using the Migration API + +You can use the Tyk Dashboard API to convert individual APIs, multiple API, or all APIs stored in your API storage database via the dedicated [/migrate](/api-management/migrate-from-tyk-classic#tyk-dashboard-api-migrate-endpoint) endpoint. + +### Tyk Dashboard API Migrate Endpoint + +`POST /api/apis/migrate` + +The payload for this request is: + +```json expandable +{ + "mode": "dryRun", // Required: Migration mode (dryRun, stage, promote, direct) + "apiIDs": ["api123", "api456"], // List of API IDs to migrate (cannot be used with 'all') + "all": false, // Migrate all APIs (cannot be used with 'apiIDs') + "abortOnFailure": false, // Stop migration process on first failure + "overrideStaged": false // When mode is 'stage', overwrite existing staged APIs +} +``` + +- Indicate the migration [mode] using the following options: + + - `dryRun` + - `stage` + - `promote` + - `direct` + +- You can convert specific APIs by providing their API IDs in the `apiIDs` array or convert all your Tyk Classic APIs in one go using the `all` option. + +- Set `abortOnFailure` to `true` if you want Tyk to stop processing if it encounters a failure while operating on a batch of APIs. + +- You can only have one **staged** version of a Tyk Classic API, so if you have already started the migration of an API and try again - for example after making changes to the original Tyk Classic API, the operation will fail. Use **overrideStaged** to delete the existing staged API and create a new one. + + +#### Example: Dry Run Migration + +``` expandable +curl -X POST \ + https://your-tyk-dashboard/api/apis/migrate \ + -H "Authorization: your-dashboard-api-key" \ + -H "Content-Type: application/json" \ + -d '{ + "mode": "dryRun", + "apiIDs": ["api123", "api456"], + "abortOnFailure": false + }' +``` + +#### Example: Migrate All APIs + +``` expandable +curl -X POST \ + https://your-tyk-dashboard/api/apis/migrate \ + -H "Authorization: your-dashboard-api-key" \ + -H "Content-Type: application/json" \ + -d '{ + "mode": "direct", + "all": true, + "abortOnFailure": false + }' +``` + +## Known Limitations of Migration + +There are some differences between the way Tyk Gateway works with Tyk Classic and Tyk OAS APIs, so you are advised to take the following into account and, if your API uses any of these features, to stage (or dry run) your migrations and ensure you perform appropriate testing on the interim versions. + +1. **Handling of Regular Expression Path Parameters** + + When migrating APIs with regex-based path parameters, be aware that: + + - In Tyk Classic, the Gateway only routes requests matching the regex pattern + - In Tyk OAS, the Gateway routes all requests matching the pattern structure by default + + Recommended action: Enable the [Validate Request](/api-management/traffic-transformation/request-validation#request-validation-using-tyk-oas) middleware in your migrated Tyk OAS API to maintain the same behavior. + +2. **Location of Mock Response Middleware** + + The position of mock response middleware in the request processing chain [differs between Tyk Classic and Tyk OAS](/api-management/traffic-transformation/mock-response#middleware-execution-order-during-request-processing): + + - In Tyk Classic, it appears at the start of the request processing chain (before authentication) + - In Tyk OAS, it appears at the end of the request processing chain + + During migration, the system automatically adds the [ignore authentication](/api-management/traffic-transformation/ignore-authentication#ignore-authentication-overview) middleware to endpoints with mock responses to maintain similar behavior. Note, however, that any other middleware configured for that endpoint or at the API level will be applied for the Tyk OAS API (which was not the case for the Tyk Classic API). + +3. **Enhanced Request Validation** + + Tyk OAS uses the more advanced [Validate Request](/api-management/traffic-transformation/request-validation#request-validation-using-tyk-oas) middleware, whereas Tyk Classic is limited to the [Validate JSON](/api-management/traffic-transformation/request-validation#request-validation-using-classic) middleware. The migration will configure Validate Request to check the request body (as performed by Validate JSON). + +## Recommended Migration Strategy + +For a safe migration approach, we recommend following these steps: +1. **Back Up Your APIs**: Perform a full [back-up](/developer-support/upgrading#backup-apis-and-policies) of your API Definitions - remember that the final migration is destructive, as the Tyk OAS APIs will inherit the database and API IDs of the originals +2. **Start with Dry Run**: Use the `dryRun` mode to validate the migration before making changes +3. **Stage Critical APIs**: For important APIs, use the `stage` mode to create test versions +4. **Test Thoroughly**: Verify all functionality in the staged APIs +5. **Promote Gradually**: Once testing is complete, use the `promote` mode to complete the migration of original APIs in batches +6. **Monitor Performance**: After migration, closely monitor the performance of migrated APIs diff --git a/api-management/multiple-environments.mdx b/api-management/multiple-environments.mdx new file mode 100644 index 00000000..a5efe6bf --- /dev/null +++ b/api-management/multiple-environments.mdx @@ -0,0 +1,401 @@ +--- +title: "Manage Multiple Environments" +'og:description': "How to Manage Multiple Environments" +tags: ['TLS', 'SSL', 'Security', 'Certificate', 'Pinning'] +sidebarTitle: "Manage Environments" +--- + +## Introduction + +It is possible with the Multi-Cloud and the Self-Managed version of Tyk to manage multiple environments across data centers. This can be very useful if you have QA, UAT and Production environments that are physically or geographically separate and you want to move API configurations between environments seamlessly. + +## What is API Sharding ? + +It is possible to use tags in various Tyk objects to change the behavior of a Tyk cluster or to modify the data that is sent to the analytics engine. Tags are free-form strings that can be embedded in Gateway configurations, API definitions, Policies and Individual Keys. + +Tags are used in two ways: To segment a cluster into various "zones" of API management, and secondly, to push more data into the analytics records to make reporting and tracking easier. + +### API Sharding + +API Sharding is what we are calling our approach to segmenting a Tyk cluster (or data centers) into different zones. An example of this in action would be to imagine you have separate VPCs that deal with different classes of services, lets say: Health, Banking and Pharma. + +You don't need the nodes that handle all the traffic for your Pharma APIs to load up the definitions for the other zones' services, this could allow someone to send unexpected traffic through (it may not go anywhere). + +Alternatively, you could use segmentation to have separate API definitions for multiple data centers. In this way you could shard your API definitions across those DC's and not worry about having to reconfigure them if there is a failover event. + +### Using Sharding to handle API life-cycle with multiple data centers + +You can use sharding to very quickly publish an API from a `development` system to `staging` or `live`, simply by changing the tags that are applied to an API definition. + +With Tyk Community Edition and Tyk Pro, these clusters must all share the same Redis DB. + +If you are an Enterprise user, then you can go a step further and use the [Tyk Multi Data Center Bridge](/api-management/mdcb#managing-geographically-distributed-gateways-to-minimize-latency-and-protect-data-sovereignty) to have full multi-DC, multi-zone cluster segmentation, and manage APIs in different segments across different database back-ends. + +### Analytics and Reporting + +In order to use tags in analytics, there are two places where you can add a `"tags":[]` section: a Policy Definition, and a Session object for a token. + +Policy tags completely replace key tags, these tags are then fed into the analytics system and can be filtered in the dashboard. + +### Node Tags + +If your API is segmented, node tags will be appended to the analytics data, this will allow you to filter out all traffic going through a specific node or node cluster. + + + +If you set `use_db_app_options.node_is_segmented` to `true` for multiple gateway nodes, you should ensure that `management_node` is set to `false`. This is to ensure visibility for the management node across all APIs. + + + + +`management_node` is available from v2.3.4 and onwards. + +See [Tyk Gateway Configuration Options](/tyk-oss-gateway/configuration) for more details on node tags. + +## Move APIs Between Environments + +It is possible to move APIs between Tyk environments in the following ways: + +### In Shared Dashboard Environments + +If the environments are both Self-Managed installations and are sharing a Tyk Dashboard (and optionally an MDCB instance) then you can use API and Gateway tagging to transparently and effortlessly move an API from one environment to another. + +See [API Tagging](/api-management/multiple-environments#api-tagging-with-on-premises) for more details. + +#### API Sharding + +You can also use [API Sharding](/api-management/multiple-environments#what-is-api-sharding-) to move APIs in a Shards (and or MDCB) Tyk Self-Managed installation. + +### In Separate Dashboard Environments + +If the API dashboards are separate and you wish to migrate API Definitions between two completely segregated environments (e.g. migrating to new hardware or a new DC), then you can use the Export functionality of the Dashboard to download the API definition as JSON and import it into your new installation. + +#### Steps for Configuration: + +1. **Select Your API** + + From the **API Designer**, select your API: + + API designer + +2. **Export the API** + + Click **EXPORT**: + + Export button location + +3. **Save the API** + + Save and rename the JSON file: + +4. **Import into your New Environment** + + In your new environment, click **IMPORT API**: + + Select import + +5. **Generate the new API** + + Select the **From Tyk Definition** tab and paste the contents of the JSON file into the code editor and click **GENERATE API**: + + Generate API + + This will now import the API Definition into your new environment, if you have kept the API ID in the JSON document as is, the ID will remain the same. + + + + + The ID you use in with any Dashboard API integrations will change as the documents physical ID will have changed with the import. + + + +### Use Tyk-Sync + +You can also use our new Tyk-Sync tool which allows you to sync your APIs (and Policies) with a Version Control System (VCS). You can then move your APIs between environments. See [Tyk-Sync](/api-management/automations/sync) for more details. + +## Move Keys Between Environments + +Tyk currently does not have a facility to export a group of keys from one environment and reissue them in another and still be able to manage those keys from within the Dashboard. + +However, it is possible to temporarily allow access to existing keys in a new environment, but it should be noted that these keys should eventually be expired and re-generated within the new environment. + +### Moving Keys Between Environments / Creating Custom Keys + +In order to use a legacy key in a new environment, simply extract the key from the old environment using the Tyk REST APIs and then create them in the new environment using the custom key creation API. + +To create a key with a custom identifier, ie Token, simply use the [Gateway (OSS)](/tyk-gateway-api) or [Dashboard (Pro)](/api-management/dashboard-configuration#create-a-custom-key) REST APIs to import a custom key. + +## Move Policies Between Environments + +Moving policies between two (Dashboard) environments is not as easy as moving API definitions and requires working with the Dashboard API to first retrieve the policies, and then modifying the document to reinsert them in your new environment: + +### Preparation + +First you must set up your new environment to respect explicit policy IDs. To do so, edit the `tyk.conf` and `tyk_analytics.conf` files in your new environment and set the `policies. allow_explicit_policy_id` setting to `true` (the setting is just `allow_explicit_policy_id` at the root level of the Dashboard configuration). In order to retain your `api_id` when moving between environments then set `enable_duplicate_slugs` to `true` in your target `tyk_analytics.conf`. + +### Steps for Configuration + +1. **Get your Policy** + + ```{.copyWrapper} expandable + curl -X GET -H "authorization: {YOUR TOKEN}" \ + -s \ + -H "Content-Type: application/json" \ + https://admin.cloud.tyk.io/api/portal/policies/{POLICY-ID} | python -mjson.tool > policy.json + ``` + +2. **Edit the file we just created** + + The original file will look something like this, notice the two ID fields: + + ```{.json} + { + "_id": "5777ecdb0a91ff0001000003", + "access_rights": { + "xxxxx": { + "allowed_urls": [], + "api_id": "xxxxx", + "api_name": "Test", + "versions": [ + "Default" + ] + } + }, + "active": true, + "date_created": "0001-01-01T00:00:00Z", + "hmac_enabled": false, + "id": "", + "is_inactive": false, + "key_expires_in": 0, + "name": "Test Policy", + "org_id": "xxxxx", + "partitions": { + "acl": false, + "quota": false, + "rate_limit": false + }, + "per": 60, + "quota_max": -1, + "quota_renewal_rate": 60, + "rate": 1000, + "tags": [] + } + ``` + +3. **Move the id field value** + + Remove the `_id` field and put the value of the `_id` field into the `id` field, so `policy.json` should look like this: + + ```{.json} + { + "access_rights": { + "xxxxx": { + "allowed_urls": [], + "api_id": "xxxxx", + "api_name": "Test", + "versions": [ + "Default" + ] + } + }, + "active": true, + "date_created": "0001-01-01T00:00:00Z", + "hmac_enabled": false, + "id": "5777ecdb0a91ff0001000003", <------ NEW ID FIELD + "is_inactive": false, + "key_expires_in": 0, + "name": "Test Policy", + "org_id": "xxxxx", + "partitions": { + "acl": false, + "quota": false, + "rate_limit": false + }, + "per": 60, + "quota_max": -1, + "quota_renewal_rate": 60, + "rate": 1000, + "tags": [] + } + ``` + +4. **Update the policy via the API** + + Save the new `policies.json` file and then let's POST it back to the new environment: + + ```{.copyWrapper} + curl -X POST -H "authorization: {API-TOKEN}" \ + -s \ + -H "Content-Type: application/json" \ + -d @policies.json \ + https://{YOUR-NEW-ENV}/api/portal/policies | python -mjson.tool + ``` + +That's it, Tyk will now load this policy, and you will be able to manage and edit it the same way in your new environment, if you are re-creating tokens in your new environment, then those tokens' ACL does not need to be changed to a new policy ID since the legacy one will always be used as the reference for the policy. + +#### Policy IDs in the Dashboard + +After migrating a Policy from one environment to another, it is important to note that the **displayed** Policy ID is not going to match. **That is okay**. It happens because Tyk Dashboard displays the [`Mongo ObjectId`](https://docs.mongodb.com/manual/reference/glossary/#term-id), which is the `_id` field, but the `id` is the important part. + +**For example:** + +Policies in source environment +Policy ID Before + +Policies in target environment after migration +Policy ID After + +Notice that the IDs appear to be different. These are the BSON IDs and are expected to be different. But if we look for the underlying GUID `id`, you can see it's been mapped properly in the target environment. + +``` +$ curl dash-host-source/api/portal/policies/ + + .... + "_id": "5eb1b133e7644400013e54ec", + "id": "", + "name": "credit score", + +$ curl dash-host-target/api/portal/policies/ + + .... + "_id": "5f03be2ce043fe000177b047", + "id": "5eb1b133e7644400013e54ec", + "name": "credit score", +``` expandable + +As you can see, under the hood, the policy has been migrated correctly with target Tyk Dashboard saving the proper ID inside `id`. That is the value that will be referred to inside Key Creation, etc. + +### Use Tyk-Sync + +You can also use our new Tyk-Sync tool which allows you to sync your Policies (and APIs) with a Version Control System (VCS). You can then move your Policies between environments. See [Tyk-Sync](/api-management/automations/sync) for more details. + +## Gateway Sharding + +With Tyk, it is easy to enable a sharded configuration, you can deploy Gateways which selectively load APIs. This unlocks abilities to run Gateways in multiple zones, all connected to the same Control Plane. This allows for GDPR deployments, development/test Gateways, or even DMZ/NON-DMZ Gateways. + +Couple this functionality with the Tyk [Multi Data Center Bridge](/api-management/mdcb#managing-geographically-distributed-gateways-to-minimize-latency-and-protect-data-sovereignty) to achieve a global, multi-cloud deployment. + +### Configure a Gateway as a shard + +Setting up a Gateway to be a shard, or a zone, is very easy. All you do is tell the node in the tyk.conf file what tags to respect and that it is segmented: + +```{.copyWrapper} +... +"db_app_conf_options": { + "node_is_segmented": true, + "tags": ["qa", "uat"] +}, + ... +``` expandable + +Tags are always treated as OR conditions, so this node will pick up all APIs that are marked as `qa` or `uat`. + +### Tag an API for a shard using the Dashboard + +From the API Designer, select the **Advanced Options** tab: + +Advanced options tab + +Scroll down to the **Segment Tags** options: + +Segment tags section + +Set the tag name you want to apply, and click **Add**. + +When you save the API, the tags will become immediately active. If any Gateways are configured to only load tagged API Definitions then this configuration will only be loaded by the relevant Gateway. + +### Tag an API for a shard using Tyk Operator + +Add the tag names to the tags mapping field within an API Definition as shown in the example below: + +```yaml {linenos=table,hl_lines=["8-9"],linenostart=1} +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin +spec: + name: httpbin + use_keyless: true + tags: + - edge + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true +``` expandable + +## Tyk Self Managed + +### Gateway & API Sharding +Tyk Gateway has a very powerful functionality that allows you to selectively choose which APIs are to be loaded on which Gateways. + +Imagine the case where you have two sets of APIs, Internal & External. You want to prevent your Internal APIs from being accessed or visible outside your protected network. Well, [sharding](/api-management/multiple-environments#what-is-api-sharding-) makes it extremely easy to configure your Tyk Gateways from the Dashboard. + +**Steps for Configuration:** + +1. **Configure a Gateway as a shard** + + Setting up a gateway to be a shard, or a zone, is very easy. All you do is tell the node in the tyk.conf file what tags to respect and that it is segmented: + + ```{.copyWrapper} + ... + "db_app_conf_options": { + "node_is_segmented": true, + "tags": ["private-gw", "edge"] + }, + ... + ``` + + Tags are always treated as OR conditions, so this node will pick up all APIs that are marked as `private-gw` or `edge`. + + + +In order to expose more details about the Gateway to the Dashboard, you can now configure the [edge_endpoints](/tyk-dashboard/configuration#edge_endpoints) section in the tyk-analytics.conf, and the Dashboard UI will pick that up and present you a list of Gateways you can chose from when creating an API. + + + +2. **Tag an API for a shard using the Dashboard** + + To add an API Tag to a an API configuration in the Dashboard, Select Edit from your API options, and select the *Advanced Options* tab: + + Advanced options tab location + + Then scroll down to the *Segment Tags* section: + + Segement tags section + + In this section, set the tag name you want to apply, and click *Add*. + + When you save the API, the tags will become immediately active, and if any Gateways are configured to only load tagged API Definitions then this configuration will only be loaded by the relevant Gateway. + +### Exposed Gateway tags to Dashboard UI + +From version 3.2.2 of the Tyk Dashboard, if [edge_endpoints](/tyk-dashboard/configuration#edge_endpoints) are being configured in tyk-analytics.conf, your Dashboard will automatically pick that list up for you, and display it in the UI when you create your API. + +List of available Gateways + +Once you select one or more Gateways, the *Segment Tags* section will be automatically prefilled with the tag values from the `edge_endpoints` configuration. + +List of segment tags + +Also, for every Gateway selected, there will be an API URL presented at the top of the page, within the *Core Settings* tab. + +List of API URLs + +### Target an API Definition via JSON + +In your API definition, add a tags section to the root of the API Definition: + +```{.copyWrapper} +"tags": ["private-gw"] +``` + +This will also set the tags for the API and when API requests are made through this Gateway, these tags will be transferred in to the analytics data set. + +### API Tagging with On-Premises + +API Sharding with Self-Managed is very flexible, but it behaves a little differently to sharding with Tyk Cloud Hybrid & Tyk Global Self-Managed deployments. The key difference is that with the latter, you can have federated Gateway deployments with **their own redis databases**. However with Tyk Self-Managed the zoning is limited to tags only, and must share a single Redis database. + +To isolate Self-Managed Gateway installations across data centers you will need to use Tyk Multi Data Center Bridge component. This system powers the functionality of Tyk Cloud & Tyk Cloud Hybrid in our cloud and is available to our enterprise customers as an add-on. diff --git a/api-management/performance-monitoring.mdx b/api-management/performance-monitoring.mdx new file mode 100644 index 00000000..38a844d4 --- /dev/null +++ b/api-management/performance-monitoring.mdx @@ -0,0 +1,111 @@ +--- +title: "Performance Monitoring" +'og:description': "How to analyze Tyk Performance" +tags: ['Performance', 'Monitoring', 'Observability'] +sidebarTitle: "Performance Monitoring" +--- + +## What is the performance impact of analytics + +Tyk Gateway allows analytics to be recorded and stored in a persistent data store (MongoDB/SQL) for all APIs by default, via [Tyk Pump](/api-management/tyk-pump#tyk-analytics-record-fields). + +Tyk Gateway generates transaction records for each API request and response, containing [analytics data](/api-management/tyk-pump#tyk-analytics-record-fields) relating to: the originating host (where the request is coming from), which Tyk API version was used, the HTTP method requested and request path etc. + +The transaction records are transmitted to Redis and subsequently transferred to a persistent [data store](/api-management/tyk-pump#external-data-stores) of your choice via Tyk Pump. Furthermore, Tyk Pump can also be configured to [aggregate](/api-management/logs-metrics#aggregated-analytics) the transaction records (using different data keys - API ID, access key, endpoint, response status code, location) and write to a persistent data store. Tyk Dashboard uses this data for: +- [Aggregated analytics](/api-management/dashboard-configuration#traffic-analytics) - Displaying analytics based on the aggregated data. +- [Log Browser](/api-management/dashboard-configuration#activity-logs) to display raw transaction records. + +### How Do Analytics Impact Performance? + +Analytics may introduce the problem of increased CPU load and a decrease in the number of requests per second (RPS). + +In the *Tyk Dashboard API* screen below, there are two APIs, *track* and *notrack*. The APIs were created to conduct a simple load test, to show the gateway's RPS (requests per second) for each API: + +- **track**: Traffic to this API is tracked, i.e. transaction records are generated for each request/response. +- **notrack**: Traffic to this API is not tracked, i.e. transaction records are not generated for each request/response. + +apis measured in Tyk Dashboard + +100,000 requests were sent to each API and the rate at which Tyk was able to handle those requests (number of requests per second) was measured. The results for the *tracked* API are displayed in the left pane terminal window; with the right pane showing the results for the *untracked* API. + +### Tracked API Performance + +measuring tracked API performance impact + +### Untracked API Performance + +measuring do_not_track API performance impact + +### Explaining the results + +We can see that **19,253.75** RPS was recorded for the *untracked* API; with **16,743.6011** RPS reported for the *tracked* API. The number of requests per second decreased by **~13%** when analytics was enabled. + +### What Can Be Done To Address This Performance Impact? + +Tyk is configurable, allowing fine grained control over which information should be recorded and which can be skipped, thus reducing CPU cycles, traffic and storage. + +Users can selectively prevent the generation of analytics for +[do_not_track](/api-management/traffic-transformation/do-not-track) middleware: +- **Per API**: Tyk Gateway will not create records for requests/responses for any endpoints of an API. +- **Per Endpoint**: Tyk Gateway will not create records for requests/responses for specific endpoints. + +When set, this prevents Tyk Gateway from generating the transaction records. Without transaction records, Tyk Pump will not transfer analytics to the chosen persistent data store. It's worth noting that the [track middleware](/api-management/dashboard-configuration#activity-by-endpoint) exclusively influences the generation of *endpoint popularity* aggregated data by Tyk Pump. + +### Conclusion + +[Disabling](/api-management/traffic-transformation/do-not-track) the creation of analytics (either per API or for specific endpoints) helps to reduce CPU cycles and network requests for systems that exhibit high load and traffic, e.g. social media platforms, streaming, financial services and trading platforms. + +Application decisions need to be made concerning which endpoints are non critical and can thus have analytics disabled. Furthermore, benchmarking and testing will be required to evaluate the actual benefits for the application specific use case. + +Subsequently, it is worthwhile monitoring traffic and system load and using this feature to improve performance. + +## How to reduce CPU usage in a Redis Cluster + +### What does high CPU usage in a Redis node within a Redis Cluster mean ? + +When a single Redis node within a Redis Cluster exhibits high CPU usage, it indicates that the CPU resources of that particular node are being heavily utilized compared to others in the cluster. + +The illustration below highlights the scenario where a single Redis node is exhibiting high CPU usage of 1.20% within a Redis Cluster. + +analytics keys stored in one Redis server + +### What could be causing this high CPU usage ? + +One possible reason for high CPU usage in a single Redis node within a Redis Cluster is that analytics features are enabled and keys are being stored within that specific Redis node. + +### How does storing keys within a single Redis server contribute to high CPU usage ? + +A high volume of analytics traffic can decrease performance, since all analytics keys are stored within one Redis server. Storing keys within a single Redis server can lead to increased CPU usage because all operations related to those keys, such as retrieval, updates and analytics processing, are concentrated on that server. This can result in heavier computational loads on that particular node. This leads to high CPU usage. + +### What can be done to address high CPU usage in this scenario ? + +Consider distributing the analytics keys across multiple Redis nodes within the cluster. This can help distribute the computational load more evenly, reducing the strain on any single node and potentially alleviating the high CPU usage. + +In Redis, *key sharding* is a term used to describe the practice of distributing data across multiple Redis instances or *shards* based on the keys. This feature is provided by [Redis Cluster](https://redis.io/docs/management/scaling/) and provides horizontal scalability and improved performance. + +Tyk supports configuring this behavior so that analytics keys are distributed across multiple servers within a Redis cluster. The image below illustrates that CPU usage is reduced across two Redis servers after making this configuration change. + +analytics keys distributed across Redis servers + +### How do I configure Tyk to distribute analytics keys to multiple Redis shards ? + +Follow these steps: + +1. **Check that your Redis Cluster is correctly configured** + + Confirm that the `enable_cluster` configuration option is set to true in the [Tyk Gateway](/tyk-oss-gateway/configuration#storageenable_cluster), [Tyk Dashboard](/tyk-dashboard/configuration#enable_cluster) and [Tyk Pump](/tyk-pump/tyk-pump-configuration/tyk-pump-environment-variables#analytics_storage_configenable_cluster) configuration files. This setting + informs Tyk that a Redis Cluster is in use for key storage. + + Ensure that the `addrs` array is populated in the [Tyk Gateway](/tyk-oss-gateway/configuration#storageaddrs) and [Tyk Pump](/tyk-pump/tyk-pump-configuration/tyk-pump-environment-variables#analytics_storage_configaddrs) configuration files (*tyk.conf* and *pump.conf*) with the addresses of all Redis Cluster nodes. If you are using Tyk Self Managed (the licensed product), also update [Tyk Dashboard](/tyk-dashboard/configuration#redis_addrs) configuration file (*tyk_analytics.conf*). This ensures that the Tyk components can interact with the entire Redis Cluster. Please refer to the [configure Redis Cluster](/tyk-configuration-reference/redis-cluster-sentinel#redis-cluster-and-tyk-gateway) guide for further details. + +2. **Configure Tyk to distribute analytics keys to multiple Redis shards** + + To distribute analytics keys across multiple Redis shards effectively you need to configure the Tyk components to leverage the Redis cluster's sharding capabilities: + + 1. **Optimize Analytics Configuration**: In the Tyk Gateway configuration (tyk.conf), set [analytics_config.enable_multiple_analytics_keys](/tyk-oss-gateway/configuration#analytics_configenable_multiple_analytics_keys) to true. This option allows Tyk to distribute analytics data across Redis nodes, using multiple keys for the analytics. There's a corresponding option for Self Managed MDCB, also named [enable_multiple_analytics_keys](/tyk-multi-data-centre/mdcb-configuration-options#enable_multiple_analytics_keys). Useful only if the gateways in the data plane are configured to send analytics to MDCB. + 2. **Optimize Connection Pool Settings**: Adjust the [optimization_max_idle](/tyk-oss-gateway/configuration#storageoptimisation_max_idle) and [optimization_max_active](/tyk-oss-gateway/configuration#storageoptimisation_max_active) settings in the configuration files to ensure that the connection pool can handle the analytics workload without overloading any Redis shard. + 3. **Use a Separate Analytics Store**: For high analytics traffic, you can opt to use a dedicated *Redis Cluster* for analytics by setting [enable_separate_analytics_store](/tyk-oss-gateway/configuration#enable_separate_analytics_store) to true in the Tyk Gateway configuration file (*tyk.conf*) and specifying the separate Redis cluster configuration in the `analytics_storage` section. Please consult the [separated analytics storage](/api-management/tyk-pump#separated-analytics-storage) guide for an example with *Tyk Pump* that can equally be applied to *Tyk Gateway*. + 4. **Review and Test**: After implementing these changes, thoroughly review your configurations and conduct load testing to verify that the analytics traffic is now evenly distributed across all Redis shards. + + By following these steps you can enhance the distribution of analytics traffic across the Redis shards. This should lead to improved scalability and performance of your Tyk deployment. + diff --git a/api-management/plugins/advance-config.mdx b/api-management/plugins/advance-config.mdx new file mode 100644 index 00000000..ee8066c8 --- /dev/null +++ b/api-management/plugins/advance-config.mdx @@ -0,0 +1,376 @@ +--- +title: "Custom Plugins Advance Configuration" +sidebarTitle: "Advance Configuration" +--- + +## CICD - Automating Your Plugin Builds + +It's very important to automate the deployment of your infrastructure. + +Ideally, you store your configurations and code in version control, and then through a trigger, have the ability to deploy everything automatically into higher environments. + +With custom plugins, this is no different. + +To illustrate this, we can look at the GitHub Actions of the [example repo][0]. + +We see that upon every pull request, a section of steps are taken to "Build, [Bundle](/api-management/plugins/overview#plugin-bundles), Release Go Plugin". + +Let's break down the [workflow file][1]: + + +### Compiling the Plugin + +We can see the first few steps replicate our first task, bootstrapping the environment and compiling the plugin into a binary format. + +```make expandable + steps: + - uses: actions/checkout@v3 + + - name: Copy Env Files + run: cp tyk/confs/tyk_analytics.env.example tyk/confs/tyk_analytics.env + + - name: Build Go Plugin + run: make go-build +``` + +We can look at the [Makefile][2] to further break down the last `go-build` command. + +### Bundle The Plugin + +The next step of the workflow is to "[bundle](/api-management/plugins/overview#plugin-bundles)" the plugin. + +``` +- name: Bundle Go Plugin + run: docker-compose run --rm --user=1000 --entrypoint "bundle/bundle-entrypoint.sh" tyk-gateway +``` + +This command generates a "bundle" from the sample Go plugin in the repo. + + + +For added security, please consider signing your [bundles](/api-management/plugins/overview#plugin-deployment-types), especially if the connection between the Gateways and the Bundler server traverses the internet. + + + + +Custom plugins can be "bundled", (zipped/compressed) into a standard format, and then uploaded to some server so that they can be downloaded by the Gateways in real time. + +This process allows us to decouple the building of our custom plugins from the runtime of the Gateways. + +In other words, Gateways can be scaled up and down, and pointed at different plugin repos very easily. This makes it easier to deploy Custom plugins especially in containerized environments such as Kubernetes, where we don't have to worry about persistent volumes. + +You can read more about plugin bundles [here][3]. + +### Deploy The Plugin + +Next step of the workflow is to publish our bundle to a server that's reachable by the Gateways. + +```make expandable +- name: Upload Bundle + uses: actions/upload-artifact@v3 + with: + name: customgoplugin.zip + path: tyk/bundle/bundle.zip + + - uses: jakejarvis/s3-sync-action@master + with: + args: --acl public-read --follow-symlinks + env: + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: 'us-east-1' + SOURCE_DIR: 'tyk/bundle' +``` + +This step uploads the Bundle to both GitHub and an AWS S3 bucket. Obviously, your workflow will look slightly different here. + + + +For seamless deployments, take a look at multi-version [plugin support](/tyk-cloud#configure-plugins) to enable zero downtime deployments of your Tyk Gateway installs + + + +### Configure the Gateway + +In order to instruct the Gateway to download the bundle, we need two things: + +1. The root server - The server which all bundles will be downloaded from. This is set globally in the Tyk conf file [here](/tyk-oss-gateway/configuration#enable_bundle_downloader). + +2. The name of the bundle - this is generated during your workflow usually. This is defined at the API level (this is where you declare Custom plugins, as evident in task 2) + +The field of the API Definition that needs to be set is `custom_middleware_bundle`. + +### Summary + +That's it! + +We've set up our dev environment, built, compiled, a Custom Go plugin, loaded onto a Tyk Gateway, and tested it by sending an API request. Finally, we've talked about deploying a Bundle in a production grade set up. + +Have a look at our [examples repo][4] for more inspiration. + +[0]: https://github.com/TykTechnologies/custom-go-plugin/actions +[1]: https://github.com/TykTechnologies/custom-go-plugin/blob/master/.github/workflows/makefile.yml +[2]: https://github.com/TykTechnologies/custom-go-plugin/blob/master/Makefile#L59 +[3]: https://github.com/TykTechnologies/custom-go-plugin#deploying-the-go-plugin +[4]: https://github.com/TykTechnologies/custom-plugin-examples + +## Instrumenting Plugins with OpenTelemetry + +By instrumenting your custom plugins with Tyk's *OpenTelemetry* library, you can gain additional insights into custom plugin behavior like time spent and exit status. Read on to see some examples of creating span and setting attributes for your custom plugins. + + + +**Note:** +Although this documentation is centered around Go plugins, the outlined principles are universally applicable to plugins written in other languages. Ensuring proper instrumentation and enabling detailed tracing will integrate the custom plugin span into the trace, regardless of the underlying programming language. + + + +### Prerequisites + +- Go v1.19 or higher +- Gateway instance with OpenTelemetry and DetailedTracing enabled: + +Add this field within your [Gateway config file](/tyk-oss-gateway/configuration): + +```json +{ + "opentelemetry": { + "enabled": true + } +} +``` + +And this field within your [API definition](/api-management/gateway-config-introduction): + +```json +{ + "detailed_tracing": true +} +``` + +You can find more information about enabling OpenTelemetry [here](/api-management/logs-metrics#opentelemetry). + + + +DetailedTracing must be enabled in the API definition to see the plugin spans in the traces. + + + +In order to instrument our plugins we will be using Tyk’s OpenTelemetry library implementation. +You can import it by running the following command: + +```console +$ go get github.com/TykTechnologies/opentelemetry +``` + +
+ + + +In this case, we are using our own OpenTelemetry library for convenience. You can also use the [Official OpenTelemetry Go SDK](https://github.com/open-telemetry/opentelemetry-go) + + + +### Create a new span from the request context + +`trace.NewSpanFromContext()` is a function that helps you create a new span from the current request context. When called, it returns two values: a fresh context with the newly created span embedded inside it, and the span itself. This method is particularly useful for tracing the execution of a piece of code within a web request, allowing you to measure and analyze its performance over time. + +The function takes three parameters: + +1. `Context`: This is usually the current request’s context. However, you can also derive a new context from it, complete with timeouts and cancelations, to suit your specific needs. +2. `TracerName`: This is the identifier of the tracer that will be used to create the span. If you do not provide a name, the function will default to using the `tyk` tracer. +3. `SpanName`: This parameter is used to set an initial name for the child span that is created. This name can be helpful for later identifying and referencing the span. + +Here's an example of how you can use this function to create a new span from the current request context: + +```go expandable +package main +import ( + "net/http" + "github.com/TykTechnologies/opentelemetry/trace" +) +// AddFooBarHeader adds a custom header to the request. +func AddFooBarHeader(rw http.ResponseWriter, r *http.Request) { + // We create a new span using the context from the incoming request. + _, newSpan := trace.NewSpanFromContext(r.Context(), "", "GoPlugin_first-span") + // Ensure that the span is properly ended when the function completes. + defer newSpan.End() + // Add a custom "Foo: Bar" header to the request. + r.Header.Add("Foo", "Bar") +} +func main() {} +``` + +In your exporter (in this case, Jaeger) you should see something like this: + +OTel Span from context + +As you can see, the name we set is present: `GoPlugin_first-span` and it’s the first child of the `GoPluginMiddleware` span. + +### Modifying span name and set status + +The span created using `trace.NewSpanFromContext()` can be further configured after its creation. You can modify its name and set its status: + +```go expandable +func AddFooBarHeader(rw http.ResponseWriter, r *http.Request) { + _, newSpan := trace.NewSpanFromContext(r.Context(), "", "GoPlugin_first-span") + defer newSpan.End() + + // Set a new name for the span. + newSpan.SetName("AddFooBarHeader Testing") + + // Set the status of the span. + newSpan.SetStatus(trace.SPAN_STATUS_OK, "") + + r.Header.Add("Foo", "Bar") +} +``` + +This updated span will then appear in the traces as `AddFooBarHeader Testing` with an **OK Status**. + +The second parameter of the `SetStatus` method can accept a description parameter that is valid for **ERROR** statuses. + +The available span statuses in ascending hierarchical order are: + +- `SPAN_STATUS_UNSET` + +- `SPAN_STATUS_ERROR` + +- `SPAN_STATUS_OK` + +This order is important: a span with an **OK** status cannot be overridden with an **ERROR** status. However, the reverse is possible - a span initially marked as **UNSET** or **ERROR** can later be updated to **OK**. + +OTel Span name and status + +Now we can see the new name and the `otel.status_code` tag with the **OK** status. + +### Setting attributes + +The `SetAttributes()` function allows you to set attributes on your spans, enriching each trace with additional, context-specific information. + +The following example illustrates this functionality using the OpenTelemetry library's implementation by Tyk + +```go expandable +func AddFooBarHeader(rw http.ResponseWriter, r *http.Request) { + _, newSpan := trace.NewSpanFromContext(r.Context(), "", "GoPlugin_first-span") + defer newSpan.End() + + // Set an attribute on the span. + newSpan.SetAttributes(trace.NewAttribute("go_plugin", "1")) + + r.Header.Add("Foo", "Bar") +} +``` + +In the above code snippet, we set an attribute `go_plugin` with a value of `1` on the span. This is just a demonstration; in practice, you might want to set attributes that carry meaningful data relevant to your tracing needs. + +Attributes are key-value pairs. The value isn't restricted to string data types; it can be any value, including numerical, boolean, or even complex data types, depending on your requirements. This provides flexibility and allows you to include rich, structured data within your spans. + +The illustration below, shows how the `go_plugin` attribute looks in Jaeger: + +OTel Span attributes + +### Multiple functions = Multiple spans + +To effectively trace the execution of your plugin, you can create additional spans for each function execution. By using context propagation, you can link these spans, creating a detailed trace that covers multiple function calls. This allows you to better understand the sequence of operations, pinpoint performance bottlenecks, and analyze application behavior. + +Here's how you can implement it: + +```go expandable +func AddFooBarHeader(rw http.ResponseWriter, r *http.Request) { + // Start a new span for this function. + ctx, newSpan := trace.NewSpanFromContext(r.Context(), "", "GoPlugin_first-span") + defer newSpan.End() + + // Set an attribute on this span. + newSpan.SetAttributes(trace.NewAttribute("go_plugin", "1")) + + // Call another function, passing in the context (which includes the new span). + NewFunc(ctx) + + // Add a custom "Foo: Bar" header to the request. + r.Header.Add("Foo", "Bar") +} + +func NewFunc(ctx context.Context) { + // Start a new span for this function, using the context passed from the calling function. + _, newSpan := trace.NewSpanFromContext(ctx, "", "GoPlugin_second-span") + defer newSpan.End() + + // Simulate some processing time. + time.Sleep(1 * time.Second) + + // Set an attribute on this span. + newSpan.SetAttributes(trace.NewAttribute("go_plugin", "2")) +} +``` + +In this example, the `AddFooBarHeader` function creates a span and then calls `NewFunc`, passing the updated context. The `NewFunc` function starts a new span of its own, linked to the original through the context. It also simulates some processing time by sleeping for 1 second, then sets a new attribute on the second span. In a real-world scenario, the `NewFunc` would contain actual code logic to be executed. + +The illustration below, shows how this new child looks in Jaeger: + +OTel Span attributes + +### Error handling + +In OpenTelemetry, it's essential to understand the distinction between recording an error and setting the span status to error. The `RecordError()` function records an error as an exception span event. However, this alone doesn't change the span's status to error. To mark the span as error, you need to make an additional call to the `SetStatus()` function. + +> RecordError will record err as an exception span event for this span. An additional call to SetStatus is required if the Status of the Span should be set to Error, as this method does not change the Span status. If this span is not being recorded or err is nil then this method does nothing. + +Here's an illustrative example with function calls generating a new span, setting attributes, setting an error status, and recording an error: + +```go expandable +func AddFooBarHeader(rw http.ResponseWriter, r *http.Request) { + // Create a new span for this function. + ctx, newSpan := trace.NewSpanFromContext(r.Context(), "", "GoPlugin_first-span") + defer newSpan.End() + + // Set an attribute on the new span. + newSpan.SetAttributes(trace.NewAttribute("go_plugin", "1")) + + // Call another function, passing in the updated context. + NewFunc(ctx) + + // Add a custom header "Foo: Bar" to the request. + r.Header.Add("Foo", "Bar") +} + +func NewFunc(ctx context.Context) { + // Create a new span using the context passed from the previous function. + ctx, newSpan := trace.NewSpanFromContext(ctx, "", "GoPlugin_second-span") + defer newSpan.End() + + // Simulate some processing time. + time.Sleep(1 * time.Second) + + // Set an attribute on the new span. + newSpan.SetAttributes(trace.NewAttribute("go_plugin", "2")) + + // Call a function that will record an error and set the span status to error. + NewFuncWithError(ctx) +} + +func NewFuncWithError(ctx context.Context) { + // Start a new span using the context passed from the previous function. + _, newSpan := trace.NewSpanFromContext(ctx, "", "GoPlugin_third-span") + defer newSpan.End() + + // Set status to error. + newSpan.SetStatus(trace.SPAN_STATUS_ERROR, "Error Description") + + // Set an attribute on the new span. + newSpan.SetAttributes(trace.NewAttribute("go_plugin", "3")) + + // Record an error in the span. + newSpan.RecordError(errors.New("this is an auto-generated error")) +} +``` + +In the above code, the `NewFuncWithError` function demonstrates error handling in OpenTelemetry. First, it creates a new span. Then it sets the status to error, and adds an attribute. Finally, it uses `RecordError()` to log an error event. This two-step process ensures that both the error event is recorded and the span status is set to reflect the error. + +OTel Span error handling + + + + diff --git a/api-management/plugins/golang.mdx b/api-management/plugins/golang.mdx new file mode 100644 index 00000000..ea441c5d --- /dev/null +++ b/api-management/plugins/golang.mdx @@ -0,0 +1,1010 @@ +--- +title: "Golang Plugins" +'og:description': "How to manage users, teams, permissions, rbac in Tyk Dashboard" +sidebarTitle: "Golang Plugins" +--- + +import { ButtonLeft } from '/snippets/ButtonLeft.mdx'; + +## Introduction + +Golang plugins are a very flexible and powerful way to extend the functionality of Tyk by attaching custom logic written in Go to [hooks](/api-management/plugins/plugin-types#plugin-types) in the Tyk [middleware chain](/api-management/traffic-transformation#request-middleware-chain). +The chain of middleware is specific to an API and gets created at API load time. When Tyk Gateway performs an API re-load it also loads any custom middleware and "injects" them into a chain to be called at different stages of the HTTP request life cycle. + +For a quick-start guide to working with Go plugins, start [here](/api-management/plugins/overview#getting-started). + +The [Go plugin writing guide](/api-management/plugins/golang#writing-custom-go-plugins) provides details of how to access dynamic data (such as the key session object) from your Go functions. Combining these resources provides you with a powerful set of tools for shaping and structuring inbound traffic to your API. + +## Supported plugin types + +All of Tyk's [custom middleware hooks](/api-management/plugins/plugin-types#plugin-types) support Go plugins. They represent different stages in the request and response [middleware chain](/api-management/traffic-transformation#request-middleware-chain) where custom functionality can be added. + +- **Pre** - supports an array of middleware that run before any others (i.e. before authentication) +- **Auth** - this middleware performs custom authentication and adds API key session info into the request context and can be used only if the API definition has both: + - `"use_keyless": false` + - `"use_go_plugin_auth": true` +- **Post-Auth** - supports an array of middleware to be run after authentication; at this point, we have authenticated the session API key for the given key (in the request context) so we can perform any extra checks. This can be used only if the API definition has both: + - `"use_keyless": false` + - an authentication method specified +- **Post** - supports an array of middleware that run at the very end of the middleware chain, just before Tyk makes a round-trip to the upstream target +- **Response** - run only at the point the response has returned from a service upstream of the API Gateway; note that the [method signature for Response Go plugins](/api-management/plugins/golang#creating-a-custom-response-plugin) is slightly different from the other hook types + + + + + The `use_keyless` and `use_go_plugin_auth` fields are populated automatically with the correct values if you add a plugin to the **Auth** or **Post-Auth** hooks when using the Tyk Dashboard. + + +## Custom Go plugin development flow + +Go Plugins need to be compiled to native shared object code, which can then be loaded by Tyk Gateway. + +We recommend that you familiarize yourself with the following official Go documentation to help you work effectively with Go plugins: + +- [The official plugin package documentation - Warnings](https://pkg.go.dev/plugin) +- [Tutorial: Getting started with multi-module workspaces](https://go.dev/doc/tutorial/workspaces) + + + + + Plugins are currently supported only on Linux, FreeBSD, and macOS, making them unsuitable for applications intended to be portable. + + + +### Tyk Plugin Compiler + +We provide the [Tyk Plugin Compiler](/api-management/plugins/golang#plugin-compiler) docker image, which we **strongly recommend** is used to build plugins compatible with the official Gateway releases. That tool provides the cross compilation toolchain, Go version used to build the release, ensures that compatible flags are used when compiling plugins (such as `-trimpath`, `CC`, `CGO_ENABLED`, `GOOS`, `GOARCH`) and also works around known Go issues such as: + +- https://github.com/golang/go/issues/19004 +- https://www.reddit.com/r/golang/comments/qxghjv/plugin_already_loaded_when_a_plugin_is_loaded/ + + +### Setting up your environment + +It's important to understand the need for plugins to be compiled using exactly the same environment and build flags as the Gateway. To simplify this and minimise the risk of compatibility problems, we recommend the use of [Go workspaces](https://go.dev/blog/get-familiar-with-workspaces), to provide a consistent environment. + +To develop plugins without using the Tyk Plugin Compiler, you'll need: + +- Go (matching the version used in the Gateway, which you can determine using `go.mod`). +- Git to check out Tyk Gateway source code. +- A folder with the code that you want to build into plugins. + +We recommend that you set up a *Go workspace*, which, at the end, is going to contain: + +- `/tyk-release-x.y.z` - the Tyk Gateway source code +- `/plugins` - the plugins +- `/go.work` - the *Go workspace* file +- `/go.work.sum` - *Go workspace* package checksums + +Using the *Go workspace* ensures build compatibility between the plugins and Gateway. + +### Steps for Configuration: + +1. **Checking out Tyk Gateway source code** + + ``` expandable + git clone --branch release-5.3.6 https://github.com/TykTechnologies/tyk.git tyk-release-5.3.6 || true + ``` + + This example uses a particular `release-5.3.6` branch, to match Tyk Gateway release 5.3.6. With newer `git` versions, you may pass `--branch v5.3.6` and it would use the tag. In case you want to use the tag it's also possible to navigate into the folder and issue `git checkout tags/v5.3.6`. + +2. **Preparing the Go workspace** + + Your Go workspace can be very simple: + + 1. Create a `.go` file containing the code for your plugin. + 2. Create a `go.mod` file for the plugin. + 3. Ensure the correct Go version is in use. + + As an example, we can use the [CustomGoPlugin.go](https://github.com/TykTechnologies/custom-go-plugin/blob/master/go/src/CustomGoPlugin.go) sample as the source for our plugin as shown: + + ``` + mkdir -p plugins + cd plugins + go mod init testplugin + go mod edit -go $(go mod edit -json go.mod | jq -r .Go) + wget -q https://raw.githubusercontent.com/TykTechnologies/custom-go-plugin/refs/heads/master/go/src/CustomGoPlugin.go + cd - + ``` + + The following snippet provides you with a way to get the exact Go version used by Gateway from it's [go.mod](https://github.com/TykTechnologies/tyk/blob/release-5.3.6/go.mod#L3) file: + + - `go mod edit -json go.mod | jq -r .Go` (e.g. `1.22.7`) + + This should be used to ensure the version matches between gateway and the plugin. + + To summarize what was done: + + 1. We created a plugins folder and initialzed a `go` project using `go mod` command. + 2. Set the Go version of `go.mod` to match the one set in the Gateway. + 3. Initialzied the project with sample plugin `go` code. + + At this point, we don't have a *Go workspace* but we will create one next so that we can effectively share the Gateway dependency across Go modules. + +3. **Creating the Go workspace** + + To set up the Go workspace, start in the directory that contains the Gateway and the Plugins folder. You'll first, create the `go.work` file to set up your Go workspace, and include the `tyk-release-5.3.6` and `plugins` folders. Then, navigate to the plugins folder to fetch the Gateway dependency at the exact commit hash and run `go mod tidy` to ensure dependencies are up to date. + + Follow these commands: + + ``` + go work init ./tyk-release-5.3.6 + go work use ./plugins + commit_hash=$(cd tyk-release-5.3.6 && git rev-parse HEAD) + cd plugins && go get github.com/TykTechnologies/tyk@${commit_hash} && go mod tidy && cd - + ``` + + The following snippet provides you to get the commit hash exactly, so it can be used with `go get`. + + - `git rev-parse HEAD` + + The Go workspace file (`go.work`) should look like this: + + ``` + go 1.22.7 + + use ( + ./plugins + ./tyk-release-5.3.6 + ) + ``` + +4. **Building and validating the plugin** + + Now that your *Go workspace* is ready, you can build your plugin as follows: + + ``` + cd tyk-release-5.3.6 && go build -tags=goplugin -trimpath . && cd - + cd plugins && go build -trimpath -buildmode=plugin . && cd - + ``` + + These steps build both the Gateway and the plugin. + + You can use the Gateway binary that you just built to test that your new plugin loads into the Gateway without having to configure and then make a request to an API using this command: + + ``` + ./tyk-release-5.3.6/tyk plugin load -f plugins/testplugin.so -s AuthCheck + ``` + + You should see an output similar to: + + ``` + time="Oct 14 13:39:55" level=info msg="--- Go custom plugin init success! ---- " + [file=plugins/testplugin.so, symbol=AuthCheck] loaded ok, got 0x76e1aeb52140 + ``` + + The log shows that the plugin has correctly loaded into the Gateway and that its `init` function has been successfully invoked. + +5. **Summary** + + In the preceding steps we have put together an end-to-end build environment for both the Gateway and the plugin. Bear in mind that runtime environments may have additional restrictions beyond Go version and build flags to which the plugin developer must pay attention. + + Compatibility in general is a big concern when working with Go plugins: as the plugins are tightly coupled to the Gateway, consideration must always be made for the build restrictions enforced by environment and configuration options. + + Continue with [Loading Go Plugins into Tyk](/api-management/plugins/golang#loading-custom-go-plugins-into-tyk). + +### Debugging Golang Plugins + +Plugins are native Go code compiled to a binary shared object file. The code may depend on `cgo` and require libraries like `libc` provided by the runtime environment. The following are some debugging steps for diagnosing issues arising from using plugins. + +#### Warnings + +The [Plugin package - Warnings](https://pkg.go.dev/plugin#hdr-Warnings) section in the Go documentation outlines several requirements which can't be ignored when working with plugins. The most important restriction is the following: + +> Runtime crashes are likely to occur unless all parts of the program (the application and all its plugins) are compiled using exactly the same version of the toolchain, the same build tags, and the same values of certain flags and environment variables. + +#### Using Incorrect Build Flags + +When working with Go plugins, it's easy to miss the restriction that the plugin at the very least must be built with the same Go version, and the same flags (notably `-trimpath`) as the Tyk Gateway on which it is to be used. + +If you miss an argument (for example `-trimpath`) when building the plugin, the Gateway will report an error when your API attempts to load the plugin, for example: + +``` +task: [test] cd tyk-release-5.3.6 && go build -tags=goplugin -trimpath . +task: [test] cd plugins && go build -buildmode=plugin . +task: [test] ./tyk-release-5.3.6/tyk plugin load -f plugins/testplugin.so -s AuthCheck +tyk: error: unexpected error: plugin.Open("plugins/testplugin"): plugin was built with a different version of package internal/goarch, try --help +``` + +Usually when the error hints at a standard library package, the build flags between the Gateway and plugin binaries don't match. + +Other error messages may be reported, depending on what triggered the issue. For example, if you omitted `-race` in the plugin but the gateway was built with `-race`, the following error will be reported: + +``` +plugin was built with a different version of package runtime/internal/sys, try --help +``` expandable + +Strictly speaking: + +- Build flags like `-trimpath`, `-race` need to match. +- Go toolchain / build env needs to be exactly the same. +- For cross compilation you must use the same `CC` value for the build (CGO). +- `CGO_ENABLED=1`, `GOOS`, `GOARCH` must match with runtime. + +When something is off, you can check what is different by using the `go version -m` command for the Gateway (`go version -m tyk`) and plugin (`go version -m plugin.so`). Inspecting and comparing the output of `build` tokens usually yields the difference that caused the compatibility issue. + +#### Plugin Compatibility Issues + +Below are some common situations where dependencies might cause issues: + +- The `Gateway` has a dependency without a `go.mod` file, but the plugin needs to use it. +- Both the `Gateway` and the plugin share a dependency. In this case, the plugin must use the exact same version as the `Gateway`. +- The plugin requires a different version of a shared dependency. + +Here’s how to handle each case: + +**Case 1: Gateway dependency lacks `go.mod`** + +- The plugin depends on the `Gateway`, which uses dependency *A*. +- *A* doesn’t have a `go.mod` file, so a pseudo version is generated during the build. +- Result: The build completes, but the plugin fails to load due to a version mismatch. + +**Solution:** Update the code to remove dependency *A*, or use a version of *A* that includes a `go.mod` file. + +**Case 2: Shared dependency with version matching** + +- The plugin and `Gateway` share a dependency, and this dependency includes a `go.mod` file. +- The version matches, and the dependency is promoted to *direct* in `go.mod`. +- Outcome: You’ll need to keep this dependency version in sync with the `Gateway`. + +**Case 3: Plugin requires a different version of a shared dependency** + +- The plugin and `Gateway` share a dependency, but the plugin needs a different version. +- If the other version is a major release (e.g., `/v4`), it’s treated as a separate package, allowing both versions to coexist. +- If it’s just a minor/patch difference, the plugin will likely fail to load due to a version conflict. + +**Recommendation:** For best results, use Go package versions that follow the Go module versioning (metaversion). However, keep in mind that many `Gateway` dependencies use basic `v1` semantic versioning, which doesn’t always enforce strict versioned import paths. + +#### List plugin symbols + +Sometimes it's useful to list symbols from a plugin. For example, we can list the symbols as they are compiled into our testplugin: + +``` +# nm -gD testplugin.so | grep testplugin +00000000014db4b0 R go:link.pkghashbytes.testplugin +000000000170f7d0 D go:link.pkghash.testplugin +000000000130f5e0 T testplugin.AddFooBarHeader +000000000130f900 T testplugin.AddFooBarHeader.deferwrap1 +000000000130f980 T testplugin.AuthCheck +0000000001310100 T testplugin.AuthCheck.deferwrap1 +000000000130f540 T testplugin.init +0000000001310ce0 T testplugin.init.0 +0000000001ce9580 D testplugin..inittask +0000000001310480 T testplugin.InjectConfigData +0000000001310180 T testplugin.InjectMetadata +0000000001d2a3e0 B testplugin.logger +0000000001310cc0 T testplugin.main +0000000001310820 T testplugin.MakeOutboundCall +0000000001310c40 T testplugin.MakeOutboundCall.deferwrap1 +``` + +This command prints other symbols that are part of the binary. In the worst case, a build compatibility issue may cause a crash in the Gateway due to an unrecoverable error and this can be used to further debug the binaries produced. + +A very basic check to ensure Gateway/plugin compatibility is using the built in `go version -m `: + +``` +[output truncated] + build -buildmode=exe + build -compiler=gc + build -race=true + build -tags=goplugin + build -trimpath=true + build CGO_ENABLED=1 + build GOARCH=amd64 + build GOOS=linux + build GOAMD64=v1 + build vcs=git + build vcs.revision=1db1935d899296c91a55ba528e7b653aec02883b + build vcs.time=2024-09-24T12:54:26Z + build vcs.modified=false +``` expandable + +These options should match between the Gateway binary and the plugin. You can use the command for both binaries and then compare the outputs. + + +## Writing Custom Go Plugins + +Tyk's custom Go plugin middleware is very powerful as it provides you with access to different data types and functionality as explained in this section. + +Golang plugins are a very flexible and powerful way to extend the functionality of Tyk and uses the native Golang plugins API (see [go pkg/plugin docs](https://golang.org/pkg/plugin) for more details). + +Custom Go plugins can access various data objects relating to the API request: + +- [session](/api-management/plugins/golang#accessing-the-session-object): the key session object provided by the client when making the API request +- [API definition](/api-management/plugins/golang#accessing-the-api-definition): the Tyk OAS or Tyk Classic API definition for the requested API + +Custom Go plugins can also [terminate the request](/api-management/plugins/golang#terminating-the-request) and stop further processing of the API request such that it is not sent to the upstream service. + +For more resources for writing plugins, please visit our [Plugin Hub](/api-management/plugins/overview#plugins-hub). +To see an example of a Go plugin, please visit our [Go plugin examples](/api-management/plugins/golang#example-custom-go-plugins) page. + +### Accessing the internal state of a custom plugin + +A Golang plugin can be treated as a normal Golang package but: + +- the package name is always `"main"` and this package cannot be imported +- this package loads at run-time by Tyk and loads after all other Golang packages +- this package has to have an empty `func main() {}`. + +A Go plugin can have a declared `func init()` and it gets called only once (when Tyk loads this plugin for the first time for an API). + +It is possible to create structures or open connections to 3d party services/storage and then share them within every call and export the function in your Golang plugin. + +For example, here is an example of a Tyk Golang plugin with a simple hit counter: + +```go {linenos=true, linenostart=1} +package main + +import ( + "encoding/json" + "net/http" + "sync" + + "github.com/TykTechnologies/tyk/ctx" + "github.com/TykTechnologies/tyk/log" + "github.com/TykTechnologies/tyk/user" +) + +var logger = log.Get() + +// plugin exported functionality +func MyProcessRequest(rw http.ResponseWriter, r *http.Request) { + endPoint := r.Method + " " + r.URL.Path + logger.Info("Custom middleware, new hit:", endPoint) + + hitCounter := recordHit(endPoint) + logger.Debug("New hit counter value:", hitCounter) + + if hitCounter > 100 { + logger.Warning("Hit counter to high") + } + + reply := myReply{ + Session: ctx.GetSession(r), + Endpoint: endPoint, + HitCounter: hitCounter, + } + + jsonData, err := json.Marshal(reply) + if err != nil { + logger.Error(err.Error()) + rw.WriteHeader(http.StatusInternalServerError) + return + } + + rw.Header().Set("Content-Type", "application/json") + rw.WriteHeader(http.StatusOK) + rw.Write(jsonData) +} + +// called once plugin is loaded, this is where we put all initialisation work for plugin +// i.e. setting exported functions, setting up connection pool to storage and etc. +func init() { + hitCounter = make(map[string]uint64) +} + +// plugin internal state and implementation +var ( + hitCounter map[string]uint64 + hitCounterMu sync.Mutex +) + +func recordHit(endpoint string) uint64 { + hitCounterMu.Lock() + defer hitCounterMu.Unlock() + hitCounter[endpoint]++ + return hitCounter[endpoint] +} + +type myReply struct { + Session *user.SessionState `json:"session"` + Endpoint string `json:"endpoint"` + HitCounter uint64 `json:"hit_counter"` +} + +func main() {} +``` expandable + +Here we see how the internal state of the Golang plugin is used by the exported function `MyProcessRequest` (the one we set in the API spec in the `"custom_middleware"` section). The map `hitCounter` is used to send internal state and count hits to different endpoints. Then our exported Golang plugin function sends an HTTP reply with endpoint hit statistics. + +### Accessing the API definition + +When Tyk passes a request to your plugin, the API definition is made available as part of the request context. + + + +The API definition is accessed differently for Tyk OAS APIs and Tyk Classic APIs, as indicated in the following sections. If you use the wrong call for your API type, it will return `nil`. + + + +#### Working with Tyk OAS APIs + +The API definition can be accessed as follows: + +```go +package main + +import ( + "fmt" + "net/http" + + "github.com/TykTechnologies/tyk/ctx" +) + +func MyPluginFunction(w http.ResponseWriter, r *http.Request) { + oas := ctx.GetOASDefinition(r) + fmt.Println("OAS doc title is", oas.Info.Title) +} + +func main() {} +``` + +The invocation of `ctx.GetOASDefinition(r)` returns an `OAS` object containing the Tyk OAS API definition. +The Go data structure can be found [here](https://github.com/TykTechnologies/tyk/blob/master/apidef/oas/oas.go#L28). + +#### Working with Tyk Classic APIs + +The API definition can be accessed as follows: + +```go +package main + +import ( + "fmt" + "net/http" + + "github.com/TykTechnologies/tyk/ctx" +) + +func MyPluginFunction(w http.ResponseWriter, r *http.Request) { + apidef := ctx.GetDefinition(r) + fmt.Println("API name is", apidef.Name) +} + +func main() {} +``` + +The invocation of `ctx.GetDefinition(r)` returns an APIDefinition object containing the Tyk Classic API Definition. +The Go data structure can be found [here](https://github.com/TykTechnologies/tyk/blob/master/apidef/api_definitions.go#L583). + +### Accessing the session object + +When Tyk passes a request to your plugin, the key session object is made available as part of the request context. This can be accessed as follows: + +```go +package main +import ( + "fmt" + "net/http" + "github.com/TykTechnologies/tyk/ctx" +) +func main() {} +func MyPluginFunction(w http.ResponseWriter, r *http.Request) { + session := ctx.GetSession(r) + fmt.Println("Developer ID:", session.MetaData["tyk_developer_id"] + fmt.Println("Developer Email:", session.MetaData["tyk_developer_email"] +} +``` expandable + +The invocation of `ctx.GetSession(r)` returns an SessionState object. +The Go data structure can be found [here](https://github.com/TykTechnologies/tyk/blob/master/user/session.go#L106). + +Here is an [example](https://github.com/TykTechnologies/custom-plugin-examples/blob/master/plugins/go-auth-multiple_hook_example/main.go#L135) custom Go plugin that makes use of the session object. + +### Terminating the request + +You can terminate the request within your custom Go plugin and provide an HTTP response to the originating client, such that the plugin behaves similarly to a [virtual endpoint](/api-management/traffic-transformation/virtual-endpoints). + +- the HTTP request processing is stopped and other middleware in the chain won't be used +- the HTTP request round-trip to the upstream target won't happen +- analytics records will still be created and sent to the analytics processing flow + +This [example](/api-management/plugins/golang#using-a-custom-go-plugin-as-a-virtual-endpoint) demonstrates a custom Go plugin configured as a virtual endpoint. + +### Logging from a custom plugin + +Your plugin can write log entries to Tyk's logging system. + +To do so you just need to import the package `"github.com/TykTechnologies/tyk/log"` and use the exported public method `Get()`: + +```go {linenos=true, linenostart=1} +package main + +import ( + "net/http" + + "github.com/TykTechnologies/tyk/log" +) + +var logger = log.Get() + +// AddFooBarHeader adds custom "Foo: Bar" header to the request +func AddFooBarHeader(rw http.ResponseWriter, r *http.Request) { + logger.Info("Processing HTTP request in Golang plugin!!") + r.Header.Add("Foo", "Bar") +} + +func main() {} +``` + +#### Monitoring instrumentation for custom plugins + +All custom middleware implemented as Golang plugins support Tyk's current built in instrumentation. + +The format for an event name with metadata is: `"GoPluginMiddleware:" + Path + ":" + SymbolName`, e.g., for our example, the event name will be: + +```text +"GoPluginMiddleware:/tmp/AddFooBarHeader.so:AddFooBarHeader" +``` + +The format for a metric with execution time (in nanoseconds) will have the same format but with the `.exec_time` suffix: + +```text +"GoPluginMiddleware:/tmp/AddFooBarHeader.so:AddFooBarHeader.exec_time" +``` + +### Creating a custom response plugin + +As explained [here](/api-management/plugins/plugin-types#response-plugins), you can register a custom Go plugin to be triggered in the response middleware chain. You must configure the `driver` field to `goplugin` in the API definition when registering the plugin. + +#### Response plugin method signature + +To write a response plugin in Go you need it to have a method signature as in the example below i.e. `func(http.ResponseWriter, *http.Response, *http.Request)`. +You can then access and modify any part of the request or response. User session and API definition data can be accessed as with other Go plugin hook types. + +```go +package main + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "net/http" + +) + +// MyPluginResponse intercepts response from upstream +func MyPluginResponse(rw http.ResponseWriter, res *http.Response, req *http.Request) { + // add a header to our response object + res.Header.Add("X-Response-Added", "resp-added") + + // overwrite our response body + var buf bytes.Buffer + buf.Write([]byte(`{"message":"Hi! I'm a response plugin"}`)) + res.Body = ioutil.NopCloser(&buf) + +} + +func main() {} +``` expandable + +## Plugin compiler + +Tyk provides a Plugin Compiler tool that will create a file that can be [loaded into Tyk](/api-management/plugins/golang#loading-custom-go-plugins-into-tyk) to implement your desired custom logic. + + + +The plugin compiler is not supported on Ubuntu 16.04 (Xenial Xerus) as it uses glibc 2.23 which is incompatible with our standard build environment. If you absolutely must have Go plugin support on Xenial, please contact Tyk support. + + + + + +### Compiler options + +Most of the following arguments are applied only to developer flows. These aid development and testing purposes, and support of these varies across releases, due to changes in the Go ecosystem. + +The latest plugin compiler implements the following options: + +- `plugin_name`: output root file name (for example `plugin.so`) +- `build_id`: [optional] provides build uniqueness +- `GOOS`: [optional] override of GOOS (add `-e GOOS=linux`) +- `GOARCH`: [optional] override of GOARCH (add `-e GOARCH=amd64`) + +By default, if `build_id` is not provided, the gateway will not allow the plugin to be loaded twice. This is a restriction of the Go plugins standard library implementation. As long as the builds are made with a unique `build_id`, the same plugin can be loaded multiple times. + +When you provide a unique `build_id` argument, that also enables hot-reload compatibility of your `.so` plugin build, so that you would not need to restart the gateway, only reload it. + +- before 5.1: the plugin would be built in a filesystem path based on `build_id` +- since 5.2.4: the plugin compiler adjusts the Go module in use for the plugin. + +As the plugins are built with `-trimpath`, to omit local filesystem path details and improve plugin compatibility, the plugin compiler relies on the Go module itself to ensure each plugin build is unique. It modifies the plugin build `go.mod` file and imports to ensure a unique build. + +- [plugin package: Warnings](https://pkg.go.dev/plugin#hdr-Warnings) +- [golang#29525 - plugin: cannot open the same plugin with different names](https://github.com/golang/go/issues/29525) + +### Output filename + +Since v4.1.0 the plugin compiler has automatically added the following suffixes to the root filename provided in the `plugin_name` argument: + +- `{Gw-version}`: the Tyk Gateway version, for example, `v5.3.0` +- `{OS}`: the target operating system, for example `linux` +- `{arch}`: the target CPU architecture, for example, `arm64` + +Thus, if `plugin_name` is set to `plugin.so` then given these example values the output file will be: `plugin_v5.3.0_linux_arm64.so`. + +This enables you to have one directory with multiple versions of the same plugin targeting different Gateway versions. + +#### Cross-compiling for different architectures and operating systems + +The Tyk Go Plugin Compiler can generate output for different architectures and operating systems from the one in which the compiler is run (cross-compiling). When you do this, the output filename will be suffixed with the target OS and architecture. + +You simply provide the target `GOOS` and `GOARCH` arguments to the plugin compiler, for example: + +```yaml +docker run --rm -v `pwd`:/plugin-source \ + --platform=linux/amd64 \ + tykio/tyk-plugin-compiler:v5.2.1 plugin.so $build_id linux arm64 +``` expandable + +This command will cross-compile your plugin for a `linux/arm64` architecture. It will produce an output file named `plugin_v5.2.1_linux_arm64.so`. + + + +If you are using the plugin compiler on MacOS, the docker run argument `--platform=linux/amd64` is necessary. The plugin compiler is a cross-build environment implemented with `linux/amd64`. + + + +### Experimental options + +The plugin compiler also supports a set of environment variables being passed: + +- `DEBUG=1`: enables debug output from the plugin compiler process. +- `GO_TIDY=1`: runs go mod tidy to resolve possible dependency issues. +- `GO_GET=1`: invokes go get to retrieve the exact Tyk gateway dependency. + +These environment options are only available in the latest gateway and plugin compiler versions. +They are unsupported and are provided to aid development and testing workflows. + +## Loading Custom Go Plugins into Tyk + +For development purposes, we are going to load the plugin from local file storage. For production, you can use [bundles](#loading-a-tyk-golang-plugin-from-a-bundle) to deploy plugins to multiple gateways. + +In this example we are using a Tyk Classic API. In the API definition find the `custom_middleware` section and make it look similar to the snippet below. Tyk Dashboard users should use RAW API Editor to access this section. + +```json +"custom_middleware": { + "pre": [], + "post_key_auth": [], + "auth_check": {}, + "post": [ + { + "name": "AddFooBarHeader", + "path": "/plugin.so" + } + ], + "driver": "goplugin" +} +``` expandable + +Here we have: + +- `driver` - Set this to `goplugin` (no value created for this plugin) which says to Tyk that this custom middleware is a Golang native plugin. +- `post` - This is the hook name. We use middleware with hook type `post` because we want this custom middleware to process the request right before it is passed to the upstream target (we will look at other types later). +- `post.name` - is your function name from the Go plugin project. +- `post.path` - is the full or relative (to the Tyk binary) path to the built plugin file (`.so`). Make sure Tyk has read access to this file. + +Also, let's set fields `"use_keyless": true` and `"target_url": "http://httpbin.org/"` - for testing purposes. We will test what request arrives to our upstream target and `httpbin.org` is a perfect fit for that. + +The API needs to be reloaded after that change (this happens automatically when you save the updated API in the Dashboard). + +Now your API with its Golang plugin is ready to process traffic: + +```bash +# curl http://localhost:8181/my_api_name/get +{ + "args": {}, + "headers": { + "Accept": "*/*", + "Accept-Encoding": "gzip", + "Foo": "Bar", + "Host": "httpbin.org", + "User-Agent": "curl/7.54.0" + }, + "url": "https://httpbin.org/get" +} +``` expandable + +We see that the upstream target has received the header `"Foo": "Bar"` which was added by our custom middleware implemented as a native Golang plugin in Tyk. + +### Updating the plugin + +Loading an updated version of your plugin requires one of the following actions: + +- An API reload with a NEW path or file name of your `.so` file with the plugin. You will need to update the API spec section `"custom_middleware"`, specifying a new value for the `"path"` field of the plugin you need to reload. +- Tyk main process reload. This will force a reload of all Golang plugins for all APIs. + +If a plugin is loaded as a bundle and you need to update it you will need to update your API spec with a new `.zip` file name in the `"custom_middleware_bundle"` field. Make sure the new `.zip` file is uploaded and available via the bundle HTTP endpoint before you update your API spec. + +### Loading a Tyk Golang plugin from a bundle + +Currently we have loaded Golang plugins only directly from the file system. However, when you have multiple gateway instances, you need a more dynamic way to load plugins. Tyk offer bundle instrumentation [Plugin Bundles](/api-management/plugins/overview#plugin-bundles). Using the bundle command creates an archive with your plugin, which you can deploy to the HTTP server (or AWS S3) and then your plugins will be fetched and loaded from that HTTP endpoint. + +You will need to set in `tyk.conf` these two fields: + +- `"enable_bundle_downloader": true` - enables the plugin bundles downloader +- `"bundle_base_url": "http://mybundles:8000/abc"` - specifies the base URL with the HTTP server where you place your bundles with Golang plugins (this endpoint must be reachable by the gateway) + +Also, you will need to specify the following field in your API spec: + +`"custom_middleware_bundle"` - here you place your filename with the bundle (`.zip` archive) to be fetched from the HTTP endpoint you specified in your `tyk.conf` parameter `"bundle_base_url"` + +To load a plugin, your API spec should set this field like so: + +```json +"custom_middleware_bundle": "FooBarBundle.zip" +``` + +Let's look at `FooBarBundle.zip` contents. It is just a ZIP archive with two files archived inside: + +- `AddFooBarHeader.so` - this is our Golang plugin +- `manifest.json` - this is a special file with meta information used by Tyk's bundle loader + +The contents of `manifest.json`: + +```yaml +{ + "file_list": [ + "AddFooBarHeader.so" + ], + "custom_middleware": { + "post": [ + { + "name": "AddFooBarHeader", + "path": "AddFooBarHeader.so" + } + ], + "driver": "goplugin" + }, + + ... +} +``` expandable + +Here we see: + +- field `"custom_middleware"` with exactly the same structure we used to specify `"custom_middleware"` in API spec without bundle +- field `"path"` in section `"post"` now contains just a file name without any path. This field specifies `.so` filename placed in a ZIP archive with the bundle (remember how we specified `"custom_middleware_bundle": "FooBarBundle.zip"`). + +## Using custom Go plugins with Tyk Cloud + +The following supporting resources are provided for developing plugins on Tyk Cloud: + +- [Enabling Plugins On The Control Plane](/tyk-cloud#configure-plugins) +- [Uploading Your Plugin Bundle To S3 Bucket](/tyk-cloud#uploading-your-bundle) + +## Example custom Go plugins + +This document provides a working example for providing specific functionality with a custom Go plugin. + +For more resources for writing plugins, please visit our [Plugin Hub](/api-management/plugins/overview#plugins-hub). + +### Using a custom Go plugin as a virtual endpoint + +It is possible to send a response from the Golang plugin custom middleware. In the case that the HTTP response was sent: + +- The HTTP request processing is stopped and other middleware in the chain won't be used. +- The HTTP request round-trip to the upstream target won't happen +- Analytics records will still be created and sent to the analytics processing flow. + +Let's look at an example of how to send an HTTP response from the Tyk Golang plugin. Imagine that we need middleware which would send JSON with the current time if the request contains the parameter `get_time=1` in the request query string: + +```go +package main + +import ( + "encoding/json" + "net/http" + "time" +) + +func SendCurrentTime(rw http.ResponseWriter, r *http.Request) { + // check if we don't need to send reply + if r.URL.Query().Get("get_time") != "1" { + // allow request to be processed and sent to upstream + return + } + + //Prepare data to send + replyData := map[string]interface{}{ + "current_time": time.Now(), + } + + jsonData, err := json.Marshal(replyData) + if err != nil { + rw.WriteHeader(http.StatusInternalServerError) + return + } + + //Send HTTP response from the Golang plugin + rw.Header().Set("Content-Type", "application/json") + rw.WriteHeader(http.StatusOK) + rw.Write(jsonData) +} + +func main() {} +``` + +Let's build the plugin by running this command in the plugin project folder: + +```bash +go build -trimpath -buildmode=plugin -o /tmp/SendCurrentTime.so +``` + +Then let's edit the API spec to use this custom middleware: + +```json +"custom_middleware": { + "pre": [ + { + "name": "SendCurrentTime", + "path": "/tmp/SendCurrentTime.so" + } + ], + "post_key_auth": [], + "auth_check": {}, + "post": [], + "driver": "goplugin" +} +``` + +Let's check that we still perform a round trip to the upstream target if the request query string parameter `get_time` is not set: + +```bash +# curl http://localhost:8181/my_api_name/get +{ + "args": {}, + "headers": { + "Accept": "*/*", + "Accept-Encoding": "gzip", + "Host": "httpbin.org", + "User-Agent": "curl/7.54.0" + }, + "url": "https://httpbin.org/get" +} +``` + +Now let's check if our Golang plugin sends an HTTP 200 response (with JSON containing current time) when we set `get_time=1` query string parameter: + +```bash +# curl http://localhost:8181/my_api_name/get?get_time=1 +{"current_time":"2019-09-11T23:44:10.040878-04:00"} +``` expandable + +Here we see that: + +- We've got an HTTP 200 response code. +- The response body has a JSON payload with the current time. +- The upstream target was not reached. Our Tyk Golang plugin served this request and stopped processing after the response was sent. + +### Performing custom authentication with a Golang plugin + +You can implement your own authentication method, using a Golang plugin and custom `"auth_check"` middleware. Ensure you set the two fields in Post Authentication Hook. + +Let's have a look at the code example. Imagine we need to implement a very trivial authentication method when only one key is supported (in the real world you would want to store your keys in some storage or have some more complex logic). + +```go +package main + +import ( + "net/http" + + "github.com/TykTechnologies/tyk/ctx" + "github.com/TykTechnologies/tyk/headers" + "github.com/TykTechnologies/tyk/user" +) + +func getSessionByKey(key string) *user.SessionState { + //Here goes our logic to check if the provided API key is valid and appropriate key session can be retrieved + + // perform auth (only one token "abc" is allowed) + if key != "abc" { + return nil + } + + // return session + return &user.SessionState{ + OrgID: "default", + Alias: "abc-session", + } +} + +func MyPluginAuthCheck(rw http.ResponseWriter, r *http.Request) { + //Try to get a session by API key + key := r.Header.Get(headers.Authorization) + session := getSessionByKey(key) + if session == nil { + // auth failed, reply with 403 + rw.WriteHeader(http.StatusForbidden) + return + } + + // auth was successful, add the session to the request's context so other middleware can use it + ctx.SetSession(r, session, true) + + // if compiling on a version older than 4.0.1, use this instead + // ctx.SetSession(r, session, key, true) +} + +func main() {} +``` expandable + +A couple of notes about this code: + +- the package `"github.com/TykTechnologies/tyk/ctx"` is used to set a session in the request context - this is something `"auth_check"`-type custom middleware is responsible for. +- the package `"github.com/TykTechnologies/tyk/user"` is used to operate with Tyk's key session structure. +- our Golang plugin sends a 403 HTTP response if authentication fails. +- our Golang plugin just adds a session to the request context and returns if authentication was successful. + +Let's build the plugin by running the following command in the folder containing your plugin project: + +```bash +go build -trimpath -buildmode=plugin -o /tmp/MyPluginAuthCheck.so +``` + +Now let's check if our custom authentication works as expected (only one key `"abc"` should work). + +Authentication will fail with the wrong API key: + +```bash +# curl -v -H "Authorization: xyz" http://localhost:8181/my_api_name/get +* Trying ::1... +* TCP_NODELAY set +* Connected to localhost (::1) port 8181 (#0) +> GET /my_api_name/get HTTP/1.1 +> Host: localhost:8181 +> User-Agent: curl/7.54.0 +> Accept: */* +> Authorization: xyz +> +< HTTP/1.1 403 Forbidden +< Date: Wed, 11 Sep 2019 04:31:34 GMT +< Content-Length: 0 +< +* Connection #0 to host localhost left intact +``` + +Here we see that our custom middleware replied with a 403 response and request processing was stopped at this point. + +Authentication successful with the right API key: + +```bash +# curl -v -H "Authorization: abc" http://localhost:8181/my_api_name/get +* Trying ::1... +* TCP_NODELAY set +* Connected to localhost (::1) port 8181 (#0) +> GET /my_api_name/get HTTP/1.1 +> Host: localhost:8181 +> User-Agent: curl/7.54.0 +> Accept: */* +> Authorization: abc +> +< HTTP/1.1 200 OK +< Content-Type: application/json +< Date: Wed, 11 Sep 2019 04:31:39 GMT +< Content-Length: 257 +< +{ + "args": {}, + "headers": { + "Accept": "*/*", + "Accept-Encoding": "gzip", + "Authorization": "abc", + "Host": "httpbin.org", + "User-Agent": "curl/7.54.0" + }, + "url": "https://httpbin.org/get" +} +* Connection #0 to host localhost left intact +``` + +Here we see that our custom middleware successfully authenticated the request and we received a reply from the upstream target. + +## Upgrading your Tyk Gateway + +When upgrading your Tyk Gateway deployment, you need to re-compile your plugin with the new version. At the moment of loading a plugin, the Gateway will try to find a plugin with the name provided in the API definition. If none is found then it will fall back to search the plugin file with the name: `{plugin-name}_{Gw-version}_{OS}_{arch}.so`. + +Since Tyk v4.1.0, the compiler [automatically](/api-management/plugins/golang#output-filename) creates plugin files following this convention so when you upgrade, say from Tyk v5.2.5 to v5.3.0 you only need to have the plugins compiled for v5.3.0 before performing the upgrade. + +This diagram shows how every Tyk Gateway will search and load the plugin binary that it is compatible with. +APIs Menu diff --git a/api-management/plugins/javascript.mdx b/api-management/plugins/javascript.mdx new file mode 100644 index 00000000..6a5f94d8 --- /dev/null +++ b/api-management/plugins/javascript.mdx @@ -0,0 +1,718 @@ +--- +title: "Javascript Plugins" +'og:description': "How to manage users, teams, permissions, rbac in Tyk Dashboard" +sidebarTitle: "Javscript Plugins" +--- + +## Introduction + +There are three middleware components that can be scripted with Tyk: + +1. **Custom JavaScript plugins**: These execute either *pre* or *post* validation. A *pre* middleware component will execute before any session validation or token validation has taken place, while a *post* middleware component will execute after the request has been passed through all checks and is ready to be proxied upstream. + +2. **Dynamic event handlers**: These components fire on certain API events (see the event handlers section), these are fired Async and do not have a cooldown timer. These are documented [here](/api-management/gateway-events#set-up-a-webhook-event-handler-in-the-tyk-oas-api-definition). + +3. **Virtual endpoints**: These are powerful programmable middleware invoked towards the end of the request processing chain. Unlike the custom JavaScript plugins, the virtual endpoint terminates the request. These are documented [here](/api-management/traffic-transformation/virtual-endpoints). + +The JavaScript (JS) [scripting guide](/api-management/plugins/javascript#using-javascript-with-tyk) provides details of how to access dynamic data (such as the key session object) from your JS functions. Combining these resources provides you with a powerful set of tools for shaping and structuring inbound traffic to your API. + +### Declared plugin functions + +JavaScript functions are available globally in the same namespace. So, if you include two or more JSVM plugins that call the same function, the last declared plugin implementation of the function will be returned. + +### Enabling the JavaScript Virtual Machine (JSVM) + +The JavaScript Virtual Machine (JSVM) provided in the Gateway is a traditional ECMAScript5 compatible environment. + +Before you can use JavaScript customization in any component you will need to enable the JSVM. + +You do this by setting `enable_jsvm` to `true` in your `tyk.conf` [file](/tyk-oss-gateway/configuration#enable_jsvm). + +### Installing JavaScript middleware + +Installing middleware is different for different Tyk deployments, for example, in Tyk OSS it is possible to directly specify a path to a file in the API Definition, while in Tyk Self-Managed, we recommend using a directory-based loader. + +We've provided the following guides: + +- [Tyk OSS](/api-management/plugins/javascript#installing-middleware-on-tyk-oss) +- [Tyk Self-Managed](/api-management/plugins/javascript#installing-middleware-on-tyk-self-managed) +- [Tyk Hybrid](/api-management/plugins/javascript#installing-middleware-on-tyk-hybrid) + + + + + Tyk Cloud Classic does not support custom middleware. + + +## Using JavaScript with Tyk + +Tyk's JavaScript Virtual Machine (JSVM) provides a serverless compute function that allows for the execution of custom logic directly within the gateway itself. This can be accessed from [multiple locations](/api-management/plugins/javascript#) in the API processing chain and allows significant customization and optimization of your request handling. + +In this guide we will cover the features and resources available to you when creating custom functions, highlighting where there are limitations for the different middleware stages. + +### Scripting basics + +Here we cover various facets that you need to be aware of when creating custom functions for Tyk. + +#### Accessing external and dynamic data + +JS functions can be given access to external data objects relating to the API request. These allow for the modification of both the request itself and the session: + +- `request`: an [object](/api-management/plugins/javascript#the-request-object) describing the API request that invoked the middleware +- `session`: the key session [object](/api-management/plugins/javascript#the-session-object) provided by the client when making the API request +- `config`: an [object](/api-management/plugins/javascript#the-config-object) containing fields from the API definition + + + + + There are other ways of accessing and editing a session object using the [Tyk JavaScript API functions](/api-management/plugins/javascript#working-with-the-key-session-object). + + + +#### Creating a middleware component + +Tyk injects a `TykJS` namespace into the JSVM, which can be used to initialise a new middleware component. The JS for each middleware component should be in its own `*.js` file. + +You create a middleware object by calling the `TykJS.TykMiddleware.NewMiddleware({})` constructor with an empty object and then initialising it with your function using the `NewProcessRequest()` closure syntax. This is where you expose the [external data objects](/api-management/plugins/javascript#accessing-external-and-dynamic-data) to your custom function. + + + +- For Custom JS plugins and Dynamic Event Handlers, the source code filename must match the function name +- Virtual Endpoints do not have this limitation + + + +#### Returning from the middleware + +When returning from the middleware, you provide specific return data depending upon the type of middleware. + +##### Returning from Custom JS plugin + +A custom JS plugin can modify fields in the API request and the session metadata, however this is not performed directly within the JSVM so the required updates must be passed out of the JSVM for Tyk to apply the changes. This is a requirement and omitting them can cause the middleware to fail. + +The JS function must provide the `request` and `session.meta_data` objects in the `ReturnData` as follows: + +```js +return sampleMiddleware.ReturnData(request, session.meta_data); +``` + +Custom JS plugins sit in the [middleware processing chain](/api-management/traffic-transformation#request-middleware-chain) and pass the request onto the next middleware before it is proxied to the upstream. If required, however, a custom JS plugin can terminate the request and provide a custom response to the client if you configure the `ReturnOverrides` in the `request` object, as described [here](/api-management/plugins/javascript#using-returnoverrides). + +##### Returning from Virtual Endpoint + +Unlike custom JS plugins, Virtual Endpoints always [terminate the request](/api-management/traffic-transformation/virtual-endpoints#working) so have a different method of returning from the JS function. + +The function must return a `responseObject`. This is crucial as it determines the HTTP response that will be sent back to the client. The structure of this object is defined to ensure that the virtual endpoint can communicate the necessary response details back to the Tyk Gateway, which then forwards it to the client. + +The `responseObject` has the following structure: + +- `code`: an integer representing the HTTP status code of the response +- `headers`: an object containing key-value pairs representing the HTTP headers of the response +- `body`: a string that represents the body of the response which can be plain text, JSON, or XML, depending on what your API client expects to receive + +You must provide the `responseObject` together with the `session.meta_data` as parameters in a call to `TykJsResponse` as follows: + +```js +return TykJsResponse(responseObject, session.meta_data); +``` + +You can find some examples of how this works [here](/api-management/traffic-transformation/virtual-endpoints#examples). + +### JavaScript resources + +JavaScript (JS) functions have access to a [system API](/api-management/plugins/javascript#javascript-api) and [library of functions](/api-management/plugins/javascript#underscorejs-library). They can also be given access to certain Tyk data objects relating to the API request. + +The system API provides access to resources outside of the JavaScript Virtual Machine sandbox, the ability to make outbound HTTP requests and access to the key management REST API functions. + +#### The `request` object + +The `request` object provides a set of arrays that describe the API request. These can be manipulated and, when changed, will affect the request as it passes through the middleware pipeline. For [virtual endpoints](/api-management/traffic-transformation/virtual-endpoints) the request object has a [different structure](#VirtualEndpoint-Request). + +The structure of the `request` object is: + +```typesecript expandable +class ReturnOverrides { + ResponseCode: number = 200; + ResponseBody: string = ""; + ResponseHeaders: string[] = []; +} + +class Request { + Headers: { [key: string]: string[] } = {}; + SetHeaders: { [key: string]: string } = {}; + DeleteHeaders: string[] = []; + Body: string = ""; + URL: string = ""; + AddParams: { [key: string]: string } = {}; + DeleteParams: string[] = []; + ReturnOverrides: ReturnOverrides = new ReturnOverrides(); + IgnoreBody: boolean = false; + Method: string = ""; + RequestURI: string = ""; + Scheme: string = ""; +} +``` + +{/* ```go expandable +struct { + Headers map[string][]string + SetHeaders map[string]string + DeleteHeaders []string + Body string + URL string + AddParams map[string]string + DeleteParams []string + ReturnOverrides { + ResponseCode: int + ResponseBody: string + ResponseHeaders []string + } + IgnoreBody bool + Method string + RequestURI string + Scheme string +} +``` */} + +- `Headers`: this is an object of string arrays, and represents the current state of the request header; this object cannot be modified directly, but can be used to read header data +- `SetHeaders`: this is a key-value map that will be set in the header when the middleware returns the object; existing headers will be overwritten and new headers will be added +- `DeleteHeaders`: any header name that is in this list will be deleted from the outgoing request; note that `DeleteHeaders` happens before `SetHeaders` +- `Body`: this represents the body of the request, if you modify this field it will overwrite the request +- `URL`: this represents the path portion of the outbound URL, you can modify this to redirect a URL to a different upstream path +- `AddParams`: you can add parameters to your request here, for example internal data headers that are only relevant to your network setup +- `DeleteParams`: these parameters will be removed from the request as they pass through the middleware; note `DeleteParams` happens before `AddParams` +- `ReturnOverrides`: values stored here are used to stop or halt middleware execution and return an error code +- `IgnoreBody`: if this parameter is set to `true`, the original request body will be used; if set to `false` the `Body` field will be used (`false` is the default behavior) +- `Method`: contains the HTTP method (`GET`, `POST`, etc.) +- `RequestURI`: contains the request URI, including the query string, e.g. `/path?key=value` +- `Scheme`: contains the URL scheme, e.g. `http`, `https` + + +##### Using `ReturnOverrides` + +If you configure values in `request.ReturnOverrides` then Tyk will terminate the request and provide a response to the client when the function completes. The request will not be proxied to the upstream. + +The response will use the parameters configured in `ReturnOverrides`: + +- `ResponseCode` +- `ResponseBody` +- `ResponseHeaders` + +In this example, if the condition is met, Tyk will return `HTTP 403 Access Denied` with the custom header `"X-Error":"the-condition"`: + +```js expandable +var testJSVMData = new TykJS.TykMiddleware.NewMiddleware({}); + +testJSVMData.NewProcessRequest(function(request, session, config) { + // Logic to determine if the request should be overridden + if (someCondition) { + request.ReturnOverrides.ResponseCode = 403; + request.ReturnOverrides.ResponseBody = "Access Denied"; + request.ReturnOverrides.headers = {"X-Error": "the-condition"}; + // This stops the request from proceeding to the upstream + } + return testJSVMData.ReturnData(request, session.meta_data); +}); +``` + +
The virtual endpoint `request` object
+For [virtual endpoint](/api-management/traffic-transformation/virtual-endpoints) functions the structure of a Javascript `request` object is: + +```typescript expandable +class VirtualEndpointRequest { + Body: string = ""; + Headers: { [key: string]: string[] } = {}; + Params: { [key: string]: string[] } = {}; + Scheme: string = ""; + URL: string = ""; +} +``` + +- `Body`: HTTP request body, e.g. `""` +- `Headers`: HTTP request headers, e.g. `"Accept": ["*/*"]` +- `Params`: Decoded query and form parameters, e.g. `{ "confirm": ["true"], "userId": ["123"] }` +- `Scheme`: The scheme of the URL ( e.g. `http` or `https`) +- `URL`: The full URL of the request, e.g `/vendpoint/anything?user_id=123\u0026confirm=true` + +
+ + + +Each query and form parameter within the request is stored as an array field in the `Params` field of the request object. + +Repeated parameter assignments are appended to the corresponding array. For example, a request against `/vendpoint/anything?user_id[]=123&user_id[]=234` would result in a Javascript request object similar to that shown below: + +```javascript expandable +const httpRequest = { + Headers: { + "Accept": ["*/*"], + "User-Agent": ["curl/8.1.2"] + }, + Body: "", + URL: "/vendpoint/anything?user_id[]=123\u0026user_id[]=234", + Params: { + "user_id[]": ["123", "234"] + }, + Scheme: "http" +}; +``` + + + +#### The `session` object + +Tyk uses an internal [session object](/api-management/policies#what-is-a-session-object) to handle the quota, rate limits, access allowances and auth data of a specific key. JS middleware can be granted access to the session object but there is also the option to disable it as deserialising it into the JSVM is computationally expensive and can add latency. Other than the `meta_data` field, the session object itself cannot be directly edited as it is crucial to the correct functioning of Tyk. + +##### Limitations + +- Custom JS plugins at the [pre-](/api-management/plugins/plugin-types#request-plugins) stage do not have access to the session object (as it has not been created yet) +- When scripting for Virtual Endpoints, the `session` data will only be available to the JS function if enabled in the middleware configuration. + +##### Sharing data between middleware using the `session` object + +For different middleware to be able to transfer data between each other, the session object makes available a `meta_data` key/value field that is written back to the session store (and can be retrieved by the middleware down the line) - this data is permanent, and can also be retrieved by the REST API from outside of Tyk using the `/tyk/keys/` method. + + + +A new JSVM instance is created for *each* API that is managed. Consequently, inter-API communication is not possible via shared methods, since they have different bounds. However, it *is* possible using the session object if a key is shared across APIs. + + + +#### The `config` object + +The third Tyk data object that is made available to the script running in the JSVM contains data from the API Definition. This is read-only and cannot be modified by the JS function. The structure of this object is: + +- `APIID`: the unique identifier for the API +- `OrgID`: the organization identifier +- `config_data`: custom attributes defined in the API description + +##### Adding custom attributes to the API Definition + +When working with Tyk OAS APIs, you can add custom attributes in the `data` object in the `x-tyk-api-gateway.middleware.global.pluginConfig` section of the API definition, for example: + +```json {linenos=true, linenostart=1} expandable +{ + "x-tyk-api-gateway": { + "middleware": { + "global": { + "pluginConfig": { + "data": { + "enabled": true, + "value": { + "foo": "bar" + } + } + } + } + } + } +} +``` + +When working with Tyk Classic APIs, you simply add the attributes in the `config_data` object in the root of the API definition: + +```json {linenos=true, linenostart=1} +{ + "config_data": { + "foo": "bar" + } +} +``` + +#### Underscore.js Library + +In addition to our Tyk JavaScript API functions, you also have access to all the functions from the [underscore](http://underscorejs.org) library. + +Underscore.js is a JavaScript library that provides a lot of useful functional programming helpers without extending any built-in objects. Underscore provides over 100 functions that support your favorite functional helpers: + +- map +- filter +- invoke + +There are also more specialized goodies, including: + +- function binding +- JavaScript templating +- creating quick indexes +- deep equality testing + +### Example + +In this basic example, we show the creation and initialisation of a middleware object. Note how the three Tyk data objects (`request`, `session`, `config`) are made available to the function and the two objects that are returned from the function (in case the external objects need to be updated). + +```js {linenos=true, linenostart=1} expandable +/* --- sampleMiddleware.js --- */ + +// Create new middleware object +var sampleMiddleware = new TykJS.TykMiddleware.NewMiddleware({}); + +// Initialise the object with your functionality by passing a closure that accepts +// two objects into the NewProcessRequest() function: +sampleMiddleware.NewProcessRequest(function(request, session, config) { + log("This middleware does nothing, but will print this to your terminal.") + + // You MUST return both the request and session metadata + return sampleMiddleware.ReturnData(request, session.meta_data); +}); +``` + +## JavaScript API + +This system API provides access to resources outside of the JavaScript Virtual Machine sandbox, the ability to make outbound HTTP requests and access to the key management REST API functions. + +Embedded JavaScript interpreters offer the bare bones of a scripting language, but not necessarily the functions that you would expect, especially with JavaScript, where objects such as `XMLHttpRequest()` are a given. However, those interfaces are actually provided by the browser / DOM that the script engine are executing in. In a similar vein, we have included a series of functions to the JSVM for convenience and give the interpreter more capability. + +This list is regularly revised and any new suggestions should be made in our [Github issue tracker](https://github.com/TykTechnologies/tyk/issues). + +Below is the list of functions currently provided by Tyk. + +- `log(string)`: Calling `log("this message")` will cause Tyk to log the string to Tyk's default logger output in the form `JSVM Log: this message` as an INFO statement. This function is especially useful for debugging your scripts. It is recommended to put a `log()` call at the end of your middleware and event handler module definitions to indicate on load that they have been loaded successfully - see the [example scripts](https://github.com/TykTechnologies/tyk/tree/master/middleware) in your Tyk installation `middleware` directory for more details. +- `rawlog(string)`: Calling `rawlog("this message")` will cause Tyk to log the string to Tyk's default logger output without any additional formatting, like adding prefix or date. This function can be used if you want to have own log format, and parse it later with custom tooling. +- `b64enc` - Encode string to base64 +- `b64dec` - Decode base64 string +- `TykBatchRequest` this function is similar to `TykMakeHttpRequest` but makes use of Tyk's [batch request feature](/api-management/batch-processing). +- `TykMakeHttpRequest(JSON.stringify(requestObject))`: This method is used to make an HTTP request, requests are encoded as JSON for deserialisation in the min binary and translation to a system HTTP call. The request object has the following structure: + +```js expandable +newRequest = { + "Method": "POST", + "Body": JSON.stringify(event), + "Headers": {}, + "Domain": "http://foo.com", + "Resource": "/event/quotas", + "FormData": {"field": "value"} +}; +``` + + + +If you want to include querystring values, add them as part of the `Domain` property. + + + +Tyk passes a simplified response back which looks like this: + +```go +type TykJSHttpResponse struct { + Code int + Body string + Headers map[string][]string +} +``` + +The response is JSON string encoded, and so will need to be decoded again before it is usable: + +```js +usableResponse = JSON.parse(response); +log("Response code: " + usableResponse.Code); +log("Response body: " + usableResponse.Body); +``` + +This method does not execute asynchronously, so execution will block until a response is received. + +### Working with the key session object + +To work with the key session object, two functions are provided: `TykGetKeyData` and `TykSetKeyData`: + +- `TykGetKeyData(api_key, api_id)`: Use this method to retrieve a [session object](/api-management/policies#what-is-a-session-object) for the key and the API provided: + + ```js expandable + // In an event handler, we can get the key idea from the event, and the API ID from the context variable. + var thisSession = JSON.parse(TykGetKeyData(event.EventMetaData.Key, context.APIID)) + log("Expires: " + thisSession.expires) + ``` + +- `TykSetKeyData(api_key, api_id)`: Use this method to write data back into the Tyk session store: + + ```js + // You can modify the object just like with the REST API + thisSession.expires = thisSession.expires + 1000; + + // Use TykSetKeyData to set the key data back in the session store + TykSetKeyData(event.EventMetaData.Key, JSON.stringify(thisSession)); + ``` + +All of these methods are described in functional examples in the Tyk `middleware/` and `event_handlers/` folders. + +## Installing Middleware on Tyk Self-Managed + +In some cases middleware references can't be directly embedded in API Definitions (for example, when using the Tyk Dashboard in an Self-Managed installation). However, there is an easy way to distribute and enable custom middleware for an API in a Tyk node by adding them as a directory structure. + +Tyk will load the middleware plugins dynamically on host-reload without needing a direct reference to them in the API Definition. + +The directory structure should look like this: + +```text +middleware + / {API Id} + / pre + / {middlewareObject1Name}.js + / {middlewareObject2Name}.js + / post + / {middlewareObject1Name}_with_session.js + / {middlewareObject2Name}.js +``` expandable + +Tyk will check for a folder that matches the `API Id` being loaded, and then load the `pre` and `post` middleware from the respective directories. + + + +The filename MUST match the object to be loaded exactly. + + + +If your middleware requires session injection, then append `_with_session` to the filename. + +### Enable the JSVM + +Before you can use Javascript Middleware you will need to enable the JSVM. + +You can do this by setting `enable_jsvm` to `true` in your `tyk.conf` file. + +## Installing Middleware on Tyk Hybrid + +In some cases middleware references can't be directly embedded in API Definitions (for example, when using the dashboard in a Hybrid install). However, there is an easy way to distribute and enable custom middleware for an API on a Tyk node. + +A second method of loading API Definitions in Tyk nodes is to add them as a directory structure in the Tyk node. Tyk will load the middleware plugins dynamically on host-reload without needing a direct reference to them in the API Definition. + +The directory structure looks as follows: + +```text +middleware + / {API Id} + / pre + / {middlewareObject1Name}.js + / {middlewareObject2Name}.js + / post + / {middlewareObject1Name}_with_session.js + / {middlewareObject2Name}.js +``` expandable + +Tyk will check for a folder that matches the `{API Id}` being loaded, and then load the `pre` and `post` middleware from the respective folders. + + + +The filename MUST match the object to be loaded exactly. + + + +If your middleware requires session injection, then append `_with_session` to the filename. + +### Enable the JSVM + +Before you can use Javascript Middleware you will need to enable the JSVM + +You can do this by setting `enable_jsvm` to `true` in your `tyk.conf` file. + +## Installing Middleware on Tyk OSS + +In order to activate middleware when using Tyk OSS or when using a file-based setup, the middleware needs to be registered as part of your API Definition. Registration of middleware components is relatively simple. + + + +It is important that your object names are unique. + + + + + +This functionality may change in subsequent releases. + + + +### Enable the JSVM + +Before you can use Javascript Middleware you will need to enable the JSVM + +You can do this by setting `enable_jsvm` to `true` in your `tyk.conf` file. + +Adding the middleware plugin is as simple as adding it to your definition file in the middleware sections: + +```json +... +"event_handlers": {}, +"custom_middleware": { + "driver": "otto", + "pre": [ + { + "name": "sampleMiddleware", + "path": "middleware/sample.js", + "require_session": false + } + ], + "post": [ + { + "name": "sampleMiddleware", + "path": "middleware/sample.js", + "require_session": false + } + ] +}, +"enable_batch_request_support": false, +... +``` + +As you can see, the parameters are all dynamic, so you will need to ensure that the path to your middleware is correct. The configuration sections are as follows: + +- `pre`: Defines a list of custom middleware objects to run *in order* from top to bottom. That will be executed *before* any authentication information is extracted from the header or parameter list of the request. Use middleware in this section to pre-process a request before feeding it through the Tyk middleware. + +- `pre[].name`: The name of the middleware object to call. This is case sensitive, and **must** match the name of the middleware object that was created, so in our example - we created `sampleMiddleware` by calling: + + `var sampleMiddleware = new TykJS.TykMiddleware.NewMiddleware({});` + +- `pre[].path`: The path to the middleware component, this will be loaded into the JSVM when the API is initialised. This means that if you reload an API configuration, the middleware will also be re-loaded. You can hot-swap middleware on reload with no service interruption. + +- `pre[].require_session`: Irrelevant for pre-processor middleware, since no auth data has been extracted by the authentication middleware, it cannot be made available to the middleware. + +- `post`: Defines a list of custom middleware objects to run *in order* from top to bottom. That will be executed *after* the authentication, validation, throttling, and quota-limiting middleware has been executed, just before the request is proxied upstream. Use middleware in this section to post-process a request before sending it to your upstream API. + +- `post[].name`: The name of the middleware object to call. This is case sensitive, and **must** match the name of the middleware object that was created, so in our example - we created `sampleMiddleware` by calling: + + `var sampleMiddleware = new TykJS.TykMiddleware.NewMiddleware({});` + +- `post[].path`: The path to the middleware component, this will be loaded into the JSVM when the API is initialised. This means that if you reload an API configuration, the middleware will also be re-loaded. You can hot-swap middleware on reload with no service interruption. + +- `post[].require_session`: Defaults to `false`, if you require access to the session object, it will be supplied as a `session` variable to your middleware processor function. + +## WAF (OSS) ModSecurity Plugin example + +Traditionally, a Web Application Firewall (WAF) would be the first layer requests would hit, before reaching the API gateway. This is not possible if the Gateway has to terminate SSL, for things such as mTLS. + +So what do you do if you still want to run your requests through a WAF to automatically scan for malicious action? We incorporate a WAF as part of the request lifecycle by using Tyk's plugin architecture. + +### Prerequisites + +* Already running Tyk - Community Edition or Pro +* Docker, to run the WAF + +### Disclaimer + +This is NOT a production ready plugin because + +* The JavaScript plugin creates a new connection with the WAF for every request +* The request is not sent over SSL +* The WAF is only sent the query params for inspection. + +For higher performance, the plugin could be written in Golang, and a connection pool would be opened and maintained over SSL + +### Steps for Configuration + +1. **Turn JSVM on your `tyk.conf` at the root level:** + + Turn on JSVM interpreter to allow Tyk to run JavaScript plugins. + + ``` + "enable_jsvm": true + ``` + +2. **Place the JavaScript plugin on Tyk file system** + + Copy the JS Plugin as a local .js file to the Gateway's file system. + + From the Gateway root, this will download the plugin called `waf.js` into the `middleware` directory: + ``` + curl https://raw.githubusercontent.com/TykTechnologies/custom-plugins/master/plugins/js-pre-post-waf/waf.js | cat > middleware/waf.js + ``` + + (Instructions) + If you are running Tyk in Docker, you can get into Tyk Gateway with `docker exec` + ``` + $ docker ps | grep gateway + 670039a3e0b8 tykio/tyk-gateway:latest "./entrypoint.sh" 14 minutes ago Up 14 minutes 0.0.0.0:8080->8080/tcp tyk-demo_tyk-gateway_1 + + ## copy container name or ID + $ docker exec -it 670039a3e0b8 bash + + ## Now SSH'd into Tyk Gateway container and can perform curl + root@670039a3e0b8:/opt/tyk-gateway# ls + + apps entrypoint.sh install middleware templates tyk-gateway.pid tyk.conf.example + coprocess event_handlers js policies tyk tyk.conf utils + + ## Download the plugin + root@670039a3e0b8:/opt/tyk-gateway# curl https://raw.githubusercontent.com/TykTechnologies/custom-plugins/master/plugins/js-pre-post-waf/waf.js | cat > middleware/waf.js + + % Total % Received % Xferd Average Speed Time Time Time Current + Dload Upload Total Spent Left Speed + 100 1125 100 1125 0 0 3906 0 --:--:-- --:--:-- --:--:-- 3975 + + ``` + + [waf.js source](https://raw.githubusercontent.com/TykTechnologies/custom-plugins/master/plugins/js-pre-post-waf/waf.js) + +3. **Import API definition into Tyk** + + Copy the following Tyk API definition and import it into your environment. + + [API Definition JSON](https://raw.githubusercontent.com/TykTechnologies/custom-plugins/master/plugins/js-pre-post-waf/apidef.json) + + Here's the important section which adds the plugin to the request lifecycle for this API: + ```{.json} + "custom_middleware": { + "pre": [ + { + "name": "Waf", + "path": "./middleware/waf.js" + } + ], + ``` + + ##### How to Import? + [Tyk Self-Managed](/api-management/gateway-config-managing-classic#import-an-api) + + [Tyk OSS](/api-management/gateway-config-managing-classic#create-an-api) + +4. **Run WAF ModSecurity Using Docker** + + First run ModSecurity with the popular [Core RuleSet](https://coreruleset.org/) in Docker + ``` + $ docker run -ti -p 80:80 -e PARANOIA=1 --rm owasp/modsecurity-crs:v3.0 + ``` + + Open a second terminal and curl it + ``` + $ curl localhost + + hello world + ``` + + We should see the request show in the WAF server: + + ``` + 172.17.0.1 - - [30/Jun/2020:00:56:42 +0000] "GET / HTTP/1.1" 200 12 + ``` + + Now try a dirty payload: + ``` + $ curl 'localhost/?param=">' + + + + 403 Forbidden + +

Forbidden

+

You don't have permission to access / + on this server.
+

+ + ``` + + Our WAF catches the response and returns a `403`. + + + Now we try through Tyk. + + ``` + ## Clean requests, should get response from upstream's IP endpoint + $ curl localhost:8080/waf/ip + + { + "origin": "172.30.0.1, 147.253.129.30" + } + + ## WAF will detect malicious payload and instruct Tyk to deny + $ curl 'localhost:8080/waf/ip?param="> + { + "error": "Bad request!" + } + ``` diff --git a/api-management/plugins/overview.mdx b/api-management/plugins/overview.mdx new file mode 100644 index 00000000..0ef6ace7 --- /dev/null +++ b/api-management/plugins/overview.mdx @@ -0,0 +1,1192 @@ +--- +title: "Custom Plugins" +sidebarTitle: "Overview" +--- + +import { ResponsiveGrid } from '/snippets/ResponsiveGrid.mdx'; + +## Introduction + +Plugins can be used to customize and enhance the capabilities of your APIs through integration with external services and databases to perform operations such as data transformation, custom authentication, logging and monitoring etc. + +When Tyk receives an API request, it works through a [chain](/api-management/traffic-transformation#request-middleware-chain) of processing *middleware* that is configured using the API definition. There are a large number of built-in middleware in the processing chain that are dedicated to performing [client authentication](/api-management/client-authentication), [request transformation](/api-management/traffic-transformation), [caching](/api-management/response-caching) and many other processes before proxying the request to the upstream. + +Tyk's custom plugin facility provides a powerful and flexible way to extend the middleware chain. It allows API developers to write custom middleware, in various programming languages, that can perform additional processing of requests and responses. + +For example, a custom authentication scheme can be implemented and executed on API requests, custom plugins can be used to provide integration with external services and databases, or additional processing can be performed on the response returned from the upstream. + +There are several different stages of the [API request lifecycle](/api-management/traffic-transformation#request-middleware-chain) where custom plugins can be attached (or *hooked*) into the middleware chain allowing significant customization to meet your specific requirements. + +Custom plugins are usually referred to by the location where they can be *hooked* into the middleware processing chain as follows: + +1. [Pre (Request)](/api-management/plugins/plugin-types#request-plugins) +2. [Authentication](/api-management/plugins/plugin-types#authentication-plugins) +3. [Post-Auth (Request)](/api-management/plugins/plugin-types#request-plugins) +4. [Post (Request)](/api-management/plugins/plugin-types#request-plugins) +5. [Response](/api-management/plugins/plugin-types#response-plugins) +6. [Analytics (Response)](/api-management/plugins/plugin-types#analytics-plugins) + +## How Plugin Works + +The diagram below illustrates a high level architectural overview for how Tyk Gateway interacts with plugins. + +plugins overview + +From the above illustration it can be seen that: + +1. The client sends a request to an API served by Tyk Gateway. +2. Tyk processes the request and forwards it to one or more plugins implemented and configured for that API. +3. A plugin performs operations (e.g., custom authentication, data transformation). +4. The processed request is then returned to Tyk Gateway, which forwards it upstream. +5. Finally, the upstream response is sent back to the client. + +## Types of Plugin + +Tyk supports four types of plugins: + +1. **[Request Plugin](/api-management/plugins/plugin-types#request-plugins)** +2. **[Authentication Plugin](/api-management/plugins/plugin-types#authentication-plugins)** +3. **[Response Plugin](/api-management/plugins/plugin-types#response-plugins)** +4. **[Analytics Plugin](/api-management/plugins/plugin-types#analytics-plugins)** + +To know more about plugin types and it's advanced configuration, refer the following [docs](/api-management/plugins/plugin-types). + +## Getting Started + +This section takes you through the process of running and building a quickstart **Go plugin**, included within Tyk's [getting started](https://github.com/TykTechnologies/custom-go-plugin) repository. Go plugins are the recommended plugin type and suitable for most use cases. + +##### Expected outcome + +At the end of this process you should have a Tyk Gateway or Tyk Self-Managed environment running locally, with a simple Go plugin executing on each API request. For each reponse to an API request the example plugin will inject a *Foo* header, with a value of *Bar*. + +##### Prerequisites + +- [Docker](https://docs.docker.com/get-docker/) +- [Docker-compose](https://docs.docker.com/compose/install/) +- [Tyk license](https://tyk.io/sign-up/#self) (if using Self-Managed Tyk, which will make the process easier via UI) +- [Make](https://www.gnu.org/software/make) +- OSX (Intel) -> Not a prerequisite, though these steps are tested on OSX Intel/ARM + +##### Before you begin + +Please clone the [getting started](https://github.com/TykTechnologies/custom-go-plugin) respository. + +```bash +git clone https://github.com/TykTechnologies/custom-go-plugin +cd custom-go-plugin +``` + +##### Choose your environment + + + + +**Read time: 15 mins** + +Dashboard Tutorial + + + +**Read time: 15 mins** + +Tyk OSS Gateway Tutorial + + + + + +### Dashboard Plugins + +This quick start explains how to run the [getting started](https://github.com/TykTechnologies/custom-go-plugin) Go plugin within Tyk Dashboard. + +**Estimated time**: 10-15 minutes + +In this tutorial you will learn how to: + +1. Add your Tyk license. +2. Bootstrap the Tyk Dashboard environment. +3. Login to Tyk Dashboard. +4. View the pre-configured API. +5. Test the plugin. +6. View the analytics. +7. Next steps. + +**Steps for Configuration:** + +1. **Add your Tyk license** + + Create and edit the file `.env` with your Tyk Dashboard license key + + ```console expandable + # Make a copy of the example .env file for the Tyk-Dashboard + cp .env.example .env + ``` + +2. **Bootstrap the getting started example** + + run the `make` command: + + ```bash + make + ``` + + This will take a few minutes to run as it compiles the plugin for the first time and downloads all the necessary Docker images. + +3. **Log in to Tyk Dashboard** + + Log on to the Tyk Dashboard on `http://localhost:3000` using the following Bootstrapped credentials: + ``` + demo@tyk.io + ``` + and password: + ``` + topsecretpassword + ``` + + Note: these are editable in `.env.example` + +4. **View the pre-configured API** + + Once you're logged on to the Tyk Dashboard, navigate to the *APIs* screen. + + You'll see a sample *Httpbin* API. Let's click into it for more details. + + Click on *VIEW RAW DEFINITION*. Note the *custom_middleware* block is filled out, injecting the compiled example Go plugin into the API. + +5. **Test the plugin** + + Let's send an API request to the API Gateway so it can reverse proxy to our API. + + ```terminal + curl localhost:8080/httpbin/get + ``` + + Yields the response: + ``` + { + "args": {}, + "headers": { + "Accept": "*/*", + "Accept-Encoding": "gzip", + "Foo": "Bar", + "Host": "httpbin.org", + "User-Agent": "curl/7.79.1", + "X-Amzn-Trace-Id": "Root=1-63f78c47-51e22c5b57b8576b1225984a" + }, + "origin": "172.26.0.1, 99.242.70.243", + "url": "http://httpbin.org/get" + } + ``` + + Note, we see a *Foo:Bar* HTTP Header was injected by our Go plugin and echoed back to us by the Httpbin mock server. + +6. **View the analytics** + + Navigate to the Dashboard's various *API Usage Data* to view analytics on the API request! + +### Open-Source Plugins + +This quick start guide will explain how to run the [getting started](https://github.com/TykTechnologies/custom-go-plugin) Go plugin using the Tyk OSS Gateway. + +**Steps for Configuration:** + +1. **Bootstrap the getting started example** + + Please run the following command from within your newly cloned directory to run the Tyk Stack and compile the sample plugin. This will take a few minutes as we have to download all the necessary dependencies and docker images. + + ```bash + make up-oss && make build + ``` + +2. **Test the plugin** + + Let's test the plugin by sending an API request to the pre-configured API definition: + + ``` + curl localhost:8080/httpbin/get + ``` + + Response: + ``` + { + "args": {}, + "headers": { + "Accept": "*/*", + "Accept-Encoding": "gzip", + "Foo": "Bar", + "Host": "httpbin.org", + "User-Agent": "curl/7.79.1" + }, + "origin": "172.28.0.1, 99.242.70.243", + "url": "http://httpbin.org/get" + } + ``` + + We've sent an API request to the Gateway. We can see that the sample custom plugin has injected an HTTP header with a value of *Foo:Bar*. This header was echoed back in the Response Body via the mock Httpbin server. + + The `./tyk/scripts/bootstrap-oss.sh` script creates an API definition that includes the custom plugin. + +3. **View the analytics** + + We can see that Tyk Pump is running in the background. Let's check the logs after sending the API request: + + ``` + docker logs custom-go-plugin_tyk-pump_1 + ``` + + Output: + ``` + time="Feb 23 16:29:27" level=info msg="Purged 1 records..." prefix=stdout-pump + {"level":"info","msg":"","time":"0001-01-01T00:00:00Z","tyk-analytics-record":{"method":"GET","host":"httpbin.org","path":"/get","raw_path":"/get","content_length":0,"user_agent":"curl/7.79.1","day":23,"month":2,"year":2023,"hour":16,"response_code":200,"api_key":"00000000","timestamp":"2023-02-23T16:29:27.53328605Z","api_version":"Non Versioned","api_name":"httpbin","api_id":"845b8ed1ae964ea5a6eccab6abf3f3de","org_id":"","oauth_id":"","request_time":1128,"raw_request":"...","raw_response":"...","ip_address":"192.168.0.1","geo":{"country":{"iso_code":""},"city":{"geoname_id":0,"names":null},"location":{"latitude":0,"longitude":0,"time_zone":""}},"network":{"open_connections":0,"closed_connections":0,"bytes_in":0,"bytes_out":0},"latency":{"total":1128,"upstream":1111},"tags":["key-00000000","api-845b8ed1ae964ea5a6eccab6abf3f3de"],"alias":"","track_path":false,"expireAt":"2023-03-02T16:29:27.54271855Z","api_schema":""}} + ``` + + As we can see, when we send API requests, the Tyk Pump will scrape them from Redis and then send them to a persistent store as configured in the Tyk Pump env file. + + In this example, we've configured a simple `STDOUT` Pump where the records will be printed to the Standard OUT (docker logs!) + +## API Configuration + +This page provides an overview on how to register one or more custom plugins to be executed at different stages or [hooks](/api-management/plugins/plugin-types#plugin-and-hook-types) in the API request/response lifecycle. If you wish to learn how to register custom plugins to be executed on the traffic logs generated by the Gateway please refer to the [analytics plugins](/api-management/plugins/plugin-types#analytics-plugins) page. + +If you need fine-grained control at the endpoint level then it is also possible to configure [per-endpoint plugins](/api-management/plugins/plugin-types#per-endpoint-custom-plugins). These are custom Golang plugins that are triggered at the end of the request processing chain before API-level *Post* plugins are executed. + +--- + +### Introduction + +There are three locations where Tyk Gateway can find plugin functions: + +1. **gRPC plugins**: Plugin functions are implemented by a gRPC server with the associated configuration specified with the API definition. For further details on how to configure gRPC plugins, please refer to our [gRPC](/api-management/plugins/rich-plugins#overview-1) documentation. +2. **Local plugins**: Plugins are implemented by functions within source code files located on the Gateway's file system. The API Definition allows the source code file path and function name to be configured for each plugin. For further details read on. +3. **Plugin bundles**: The plugin source code and configuration are bundled into a zip file that is served by a remote web server. For further details see the [plugin bundles](/api-management/plugins/overview#plugin-bundles) page. + +### Plugin configuration + +Each plugin for an API can be configured within the API Definition with the following details: + +| Property | Description | +| :------- | :------------- | +| `Enabled` | When true, the plugin is activated | +| `Name` | A name used to identify the plugin | +| `Path` | The path to the source code file on the Tyk Gateway file system | +| `Function name` | The name of the function that implements the plugin. The function should exist within the source code file referenced in `path` | +| `Raw body only` | When set to true, this flag indicates that only the raw request body should be processed | +| `Require session state`| When set to true, Tyk Gateway will serialize the request session state and pass it as an argument to the function that implements the plugin in the target language. This is applicable to Post, Response, and Authentication hooks only | + +--- + +### Language configuration + +For local and bundle plugins a [plugin driver](/api-management/plugins/overview#plugin-driver-names) is configured to specify the plugin implementation language. If using gRPC plugins a `grpc` plugin driver should be used to instruct Tyk to request execution of plugins from within a gRPC server that is external to the Tyk process. This offers additional language support since Tyk can integrate with a gRPC server that is implemented using any supported [gRPC language](https://grpc.io/docs/). + +For a given API it is not possible to mix the implementation language for the plugin types: Pre, Authentication, Post, Post Authentication and Response plugins. For example, it is not possible to implement a pre request plugin in *Go* and also implement a post request plugin in *Python* for the same API. + +### Tyk OAS APIs + +An API can be configured so that one or more of its associated plugins can execute at different phases of the request / response life cycle. Each plugin configuration serves to identify the plugin source file path and the name of the corresponding function, triggered at each request / response lifecycle stage. + +This guide explains how to configure plugins for Tyk OAS APIs within the [Tyk OAS API definition](#tyk-oas-apidef) or via the [API designer](#tyk-oas-dashboard) in Tyk Dashboard. + +If you’re using the legacy Tyk Classic APIs, then check out the [Tyk Classic](/api-management/plugins/overview#tyk-classic-apis) page. + +

Using API Definition

+The `x-tyk-api-gateway.middleware.global` section is used to configure plugins in a Tyk OAS API. It contains a `pluginConfig` section and a list of plugins for each phase of the API request / response lifecycle. + +The `pluginConfig` section contains the `driver` parameter that is used to configure the plugin implementation [language](/api-management/plugins/overview#plugin-driver-names): + +```yaml +"pluginConfig": { + "driver": "goplugin" +} +``` expandable + +Within the `x-tyk-api-gateway.middleware.global` section, keyed lists of plugins can be configured for each phase of the API request / response lifecycle described in the table below: + +| Phase | Description | Config Key | +| :----- | :--- | :---- | +| Pre | Executed at the start of the request processing chain | `prePlugins` | +| Post Auth | Executed after the requester has been authenticated | `postAuthenticationPlugins` | +| Post | Executed at the end of the request processing chain | `postPlugins` | +| Response | Occurs after the main request processing but before the response is sent. | `responsePlugins` | + +Each plugin configuration can have the following fields configured: + +- `enabled`: When true, enables the plugin. +- `functionName`: The name of the function that implements the plugin within the source file. +- `path`: The path to the plugin source file. +- `rawBodyOnly`: When true, indicates that only the raw body should be processed. +- `requireSession`: When true, indicates that session metadata will be available to the plugin. This is applicable only for post, post authentication and response plugins. + +For example a Post Authentication plugin would be configured within a `postAuthenticationPlugins` list as shown below: + +```yaml +"postAuthenticationPlugins": [ + { + "enabled": true, + "functionName": "post_authentication_func", + "path": "/path/to/plugin1.so", + "rawBodyOnly": true, + "requireSession": true + } +] +``` + +An full example is given below to illustrate how to set up plugins for different phases of the request / response lifecycle: + +```json {linenos=true, linenostart=1, hl_lines=["15-52"]} +{ + "x-tyk-api-gateway": { + "info": { + "dbId": "667962397f6de50001508ac4", + "id": "b4d8ac6e5a274d7c7959d069b47dc206", + "orgId": "6672f4377f6de50001508abf", + "name": "OAS APIs Plugins", + "state": { + "active": true, + "internal": false + } + }, + "middleware": { + "global": { + "pluginConfig": { + "driver": "goplugin" + }, + "postAuthenticationPlugins": [ + { + "enabled": true, + "functionName": "post_authentication_func", + "path": "/path/to/plugin1.so", + "rawBodyOnly": true, + "requireSession": true + } + ], + "postPlugins": [ + { + "enabled": true, + "functionName": "postplugin", + "path": "/path/to/plugin1.so", + "rawBodyOnly": true, + "requireSession": true + } + ], + "prePlugins": [ + { + "enabled": true, + "functionName": "pre-plugin", + "path": "/path/to/plugin1.so" + } + ], + "responsePlugins": [ + { + "enabled": true, + "functionName": "Response", + "path": "/path/to/plugin1.so", + "rawBodyOnly": true, + "requireSession": true + } + ] + } + } + } +} +``` expandable + +In this example we can see that the plugin driver has been configured by setting the `driver` field to `goplugin` within the `pluginConfig` object. This configuration instructs Tyk Gateway that our plugins are implemented using Golang. + +We can also see that the following type of plugins are configured: + +- **Pre**: A plugin is configured within the `prePlugins` list. The plugin is enabled and implemented by function `pre-plugin` within the source file located at path `/path/to/plugin1.so`. +- **Post Authentication**: A plugin is configured within the `postAuthenticationPlugins` list. The plugin is enabled and implemented by function `post_authentication_func` within the source file located at path `/path/to/plugin1.so`. The raw request body and session metadata is available to the plugin. +- **Post**: A plugin is configured within the `responsePlugins` list. The plugin is enabled and implemented by function `postplugin` within the source file located at path `/path/to/plugin1.so`. The raw request body and session metadata is available to the plugin. +- **Response**: A plugin is configured within the `postPlugins` list. The plugin is enabled and implemented by function `Response` within the source file located at path `/path/to/plugin1.so`. The raw request body and session metadata is available to the plugin. + +The configuration above is a complete and valid Tyk OAS API Definition that you can use as a basis for trying out custom plugins. You will need to update the [driver](/api-management/plugins/overview#plugin-driver-names) parameter to reflect the target language type of your plugins. You will also need to update the `path` and `functionName` parameters for each plugin to reflect the source code. + +

Using API Designer

+Select your API from the list of *Created APIs* to reach the API designer and then follow these steps: + +1. **Configure plugin type and custom data** + + In the *Plugins Configuration* section, select the *Plugin Driver*, which tells Tyk which type of plugin to expect: Go, gRPC, JavaScript (OTTO), Lua or Python. + + You can configure custom data that will be made available to your plugin function as a JSON formatted object in the *Config Data* option. + + OAS API Plugins Driver Config + +2. **Configure the custom plugins** + + For each plugin that you wish to register with the API, click on the **Add Plugin** button to display a plugin configuration section: + + OAS Plugins Config Section + + Complete the following fields: + + - `Function Name`: Enter the name of the function within your plugin code that Tyk should invoke. + - `Path`: Enter the path to the source file that contains the function that implements your plugin. + - `Raw Body Only`: Optionally, toggle the *Raw Body Only* switch to true when you do not wish to fill body in request or response object for your plugins. + +3. **Save the API** + + Select **Save API** to apply the changes to your API. + +### Tyk Classic APIs + +An API can be configured so that one or more of its associated plugins can execute at different phases of the request / response lifecycle. Each plugin configuration serves to identify the plugin source file path and the name of the corresponding function, triggered at each request / response lifecycle stage. + +This guide explains how to configure plugins for Tyk Classic APIs within the [Tyk Classic API definition](#tyk-classic-apidef) or via the [API designer](#tyk-classic-dashboard) in Tyk Dashboard. + +If you’re using the newer Tyk OAS APIs, then check out the [Tyk OAS](/api-management/plugins/overview#tyk-oas-apis) page. + +

Using API Definition

+In Tyk Classic APIs, the *custom_middleware* section of the Tyk Classic API Definition is where you configure plugins that will run at different points during the lifecycle of an API request. + +This table illustrates the different phases of the API request lifecycle where custom plugins can be executed: + +| Phase | Description | Config | +| :----- | :--- | :---- | +| Pre | Executed at the start of the request processing chain | `pre` | +| Auth | Executed during the authentication step | `auth_check` | +| Post Auth | Executed after the requester has been authenticated | `post_key_auth` | +| Post | Executed at the end of the request processing chain | `post` | +| Response | Executed on the response received from the upstream | `response` | + +This example configuration illustrates how to set up plugins for different phases of the request lifecycle: + +```json {linenos=true, linenostart=1} +{ + "custom_middleware": { + "pre": [ + { + "name": "PreHook1", + "path": "/path/to/plugin1.so", + "disabled": false, + "require_session": false, + "raw_body_only": false + } + ], + "auth_check": { + "name": "AuthCheck", + "path": "/path/to/plugin.so", + "disabled": false, + "require_session": false, + "raw_body_only": false + }, + "post_key_auth": [ + { + "name": "PostKeyAuth", + "path": "/path/to/plugin.so", + "disabled": false, + "require_session": false, + "raw_body_only": false + } + ], + "post": [ + { + "name": "PostHook1", + "path": "/path/to/plugin1.so", + "disabled": false, + "require_session": false, + "raw_body_only": false + }, + { + "name": "PostHook2", + "path": "/path/to/plugin2.so", + "disabled": false, + "require_session": false, + "raw_body_only": false + } + ], + "response": [ + { + "name": "ResponseHook", + "path": "/path/to/plugin.so", + "disabled": false, + "require_session": false, + "raw_body_only": false + } + ], + "driver": "goplugin" + } +} +``` expandable + +In this example we can see that there are Golang custom authentication (`auth_check`), post authentication (`post_key_auth`), post, pre and response plugins configured. + +It can be seen that each plugin is configured with the specific function name and associated source file path of the file that contains the function. Furthermore, each lifecycle phase (except `auth`) can have a list of plugins configured, allowing for complex processing workflows. For example, you might develop one plugin for logging and another for modifying the request in the pre request phase. When multiple plugins are configured for a phase they will be executed in the order that they appear in the API definition. + +The `driver` configuration parameter describes the plugin implementation language. Please refer to the [supported languages](/api-management/plugins/overview#plugin-driver-names) section for list of supported plugin driver names. + +Each plugin can have additional settings, such as: +- `disabled`: When true, disables the plugin. +- `raw_body_only`: When true, indicates that only the raw body should be processed. +- `require_session`: When true, indicates that session metadata will be available to the plugin. This is applicable only for post, post authentication and response plugins. + +

Using API Designer

+This section explains how to configure plugins for a Tyk Classic API using Tyk Dashboard. It specifically covers the use case where the source files of your plugins are deployed on the Tyk Gateway file system. + +Select your API from the list of *Created APIs* to reach the API designer and then follow these steps: + +Plugins Classic API screen + +1. **Display the Tyk Classic API Definition editor** + + Click on the **View Raw Definition** button to display an editor for updating the Tyk Classic API Definition. + + Plugins Classic API Definition editor screen + +2. **Edit the Tyk Classic API Definition to configure plugins** + + Use the editor to edit the `custom_middleware` section of the [Tyk Classic API Definition](/api-management/plugins/overview#tyk-classic-apis). + + Plugins Classic API Bundle Field + +3. **Save changes** + + Select the **Update** button to apply your changes to the Tyk Classic API Definition. + +## Plugin Deployment Types + +There are a variety of scenarios relating to the deployment of plugins for an API, concerning the location of the plugin source code and its associated configuration. + +### Local Plugins + +The plugin source code and associated configuration are co-located with Tyk Gateway in the same file system. The configuration is located within the API Definition. For further details please consult [API configuration](/api-management/plugins/overview#api-configuration). + +### Plugin Bundles (Remote) + +The plugin source code and associated configuration are bundled into a zip file and uploaded to a remote webserver. Multiple plugins can be stored in a single *plugin bundle*. Tyk Gateway will download the plugin bundle from the remote webserver and then extract, cache and execute plugins for each of the configured phases of the API request / response lifecycle. For further details on plugin bundles and how to configure them, please refer to the [plugin bundles](/api-management/plugins/overview#plugin-bundles) page. + +### gRPC Plugins (Remote) + +Custom plugins can be hosted on a remote server and executed from the Tyk Gateway middleware chain via gRPC. These plugins can be written in any language you prefer, as they are executed on the gRPC server. You'll configure your API definition so that Tyk Gateway will send requests to your gRPC server at the appropriate points in the API request / response lifecycle. For further details please consult our [gRPC](/api-management/plugins/rich-plugins#overview-1) documentation. + +## Plugin Bundles + +For Tyk Gateway to execute local custom plugins during the processing of API requests and responses, the plugin source code must be loaded into the Gateway. The source is usually stored in files and the API definition is used to point the Gateway at the correct file for each [plugin type](/api-management/plugins/plugin-types#plugin-types). To simplify the management of plugins, you can group (or *bundle*) multiple plugin files together in a ZIP file that is referred to as a *plugin bundle*. + +### When To Use Plugin Bundles + +Plugin bundles are intended to simplify the process of attaching and loading custom middleware. Multiple API definitions can refer to the same plugin bundle (containing the source code and configuration) if required. Having this common, shared resource avoids you from having to duplicate plugin configuration for each of your APIs definitions. + +### How Plugin Bundles Work + +The source code and a [manifest file](#manifest) are bundled into a zip file and uploaded to an external remote web server. The manifest file references the source code file path and the function name within the code that should be invoked for each [plugin type](/api-management/plugins/plugin-types#plugin-types). Within the API definition, custom plugins are configured simply using the name of the bundle (zip file). Tyk Gateway downloads, caches, extracts and executes plugins from the downloaded bundle according to the configuration in the manifest file. + +plugin bundles architectural overview + +#### Caching plugin bundles + +Tyk downloads a plugin bundle on startup based on the configuration in the API definition, e.g. `http://my-bundle-server.com/bundles/bundle-latest.zip`. The bundle contents will be cached so that, when a Tyk reload event occurs, the Gateway does not have to retrieve the bundle from the server again each time. If you want to use a different bundle then you must update your API to retrieve a different bundle filename and then trigger a reload. It is not sufficient simply to replace the bundle file on your server with an updated version with the same name - the caching ensures this will not be retrieved during a reload event. + +As a suggestion, you may organize your plugin bundle files using a Git commit reference or version number, e.g. `bundle-e5e6044.zip`, `bundle-48714c8.zip`, `bundle-1.0.0.zip`, `bundle-1.0.1.zip`, etc. + +Alternatively, you may delete the cached bundle from Tyk manually and then trigger a hot reload to tell Tyk to fetch a new one. By default, Tyk will store downloaded bundles in this path: +`{ TYK_ROOT } / { CONFIG_MIDDLEWARE_PATH } / bundles` + +#### Gateway configuration + +To configure Tyk Gateway to load plugin bundles the following parameters must be specified in your `tyk.conf`: + +```yaml +"enable_bundle_downloader": true, +"bundle_base_url": "http://my-bundle-server.com/bundles/", +"public_key_path": "/path/to/my/pubkey", +``` expandable + +- `enable_bundle_downloader`: Enables the bundle downloader. +- `bundle_base_url`: A base URL that will be used to download the bundle. For example if we have `bundle-latest.zip` specified in the API definition, Tyk will fetch the following file: `http://my-bundle-server.com/bundles/bundle-latest.zip` (see the next section for details). +- `public_key_path`: Sets a public key, used for verifying signed bundles. If unsigned bundles are used you may omit this. + + + + + Remember to set `"enable_coprocess": true` in your `tyk.conf` when using [rich plugins](/api-management/plugins/overview#plugin-bundles)! + + + +

The manifest file

+A plugin bundle must include a manifest file (called `manifest.json`). The manifest file contains important information like the configuration block and the list of source code files that will be included as part of the bundle file. If a file isn't specified in the list, it won't be included in the resulting file, even if it's present in the current directory. + +A sample manifest file looks like this: + +```json +{ + "file_list": [ + "middleware.py", + "mylib.py" + ], + "custom_middleware": { + "pre": [ + { + "name": "PreMiddleware" + } + ], + "post": [ + { + "name": "PostMiddleware" + } + ], + "driver": "python" + }, + "checksum": "", + "signature": "" +} +``` expandable + +You may leave the `checksum` and `signature` fields empty, the bundler tool will fill these during the build process. + +The `custom_middleware` block follows the standard syntax we use for Tyk plugins. In Tyk Community Edition, where file-based API configuration is used by default, a `custom_middleware` block is located/added to the API configuration file. + +#### Creating plugin bundles + +Tyk provides the Bundle CLI tool as part of the `tyk` binary. For further details please visit the [Bundle CLI tool](/api-management/plugins/overview#bundler-cli-tool) page. + +### Tyk OAS API Configuration + +For API plugins that are deployed as [plugin bundles](/api-management/plugins/overview#plugin-bundles), the API should be configured with the name of the plugin bundle file to download from your remote web server. Furthermore, the Gateway should be [configured](/api-management/plugins/overview#gateway-configuration) to enable downloading plugin bundles. + +You can configure your API with the name of the plugin bundle file to download within the Tyk OAS API definition or API Designer. + +If you’re using the legacy Tyk Classic APIs, then check out the [Tyk Classic](/api-management/plugins/overview#tyk-classic-apis) page. + +#### Using API Definition + +The configuration for a Tyk OAS API to fetch the download of a plugin bundle from a remote web server is encapsulated within the `pluginConfig` section within the `middleware.global` section of the `x-tyk-api-gateway` part of a Tyk OAS API Definition. + +The `pluginConfig` section is structured as follows: + +- `bundle`: A JSON entity that contains the following configuration parameters: + - `enabled`: When `true`, enables the plugin. + - `path`: The relative path of the zip file in relation to the base URL configured on the remote webserver that hosts plugin bundles. +- `driver`: Indicates the type of plugin, e.g. `golang`, `grpc`, `lua`, `otto` or `python`. + +An illustrative example is listed below: + +```json{hl_lines=["37-45"], linenos=true, linenostart=1} +{ + "components": {}, + "info": { + "title": "example-oas-plugin-configuration", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/anything": { + "put": { + "operationId": "anythingput", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-oas-plugin-configuration", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-oas-plugin-configuration/", + "strip": true + } + }, + "middleware": { + "global": { + "pluginConfig": { + "bundle": { + "enabled": true, + "path": "plugin.zip" + }, + "driver": "goplugin" + } + } + } + } +} +``` expandable + +In this example we can see that bundle plugin has been configured within the `middleware.global.pluginConfig.bundle` object. The plugin is enabled and bundled within file `plugin.zip`. The plugin bundle is a Go plugin, i.e. `middleware.global.pluginConfig.driver` has been configured with value `goplugin`. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out custom plugin bundles, assuming that you have provided a valid bundle file named `plugin.zip`. + +#### Using API Designer + +To configure plugin bundles for Tyk OAS APIs click on the APIs menu item in the *API Management* menu of Dashboard and select your API to display the editor screen. Subsequently, follow the steps below: + +1. **Access plugin options** + + Scroll down until the *Enable Plugin* section is displayed. + + Tyk OAS API Bundle section + +2. **Enable plugin bundle for you API** + + Enable a plugin bundle for your API by activating the toggle switch. + +3. **Enter relative path to plugin bundle file** + + Enter the relative path of the plugin bundle file in the *Plugin Bundle ID* field that Tyk Gateway should download from the web server that hosts your plugin bundles. + +4. **Save the API** + + Select **Save API** to apply the changes to your API. + +### Tyk Classic API Configuration + +For custom plugins that are deployed as [plugin bundles](/api-management/plugins/overview#plugin-bundles), the API should be configured with the name of the plugin bundle file to download from your remote web server. Furthermore, the Gateway should be [configured](/api-management/plugins/overview#gateway-configuration) to enable downloading plugin bundles. + +You can configure your API with the name of the plugin bundle file to download within the Tyk Classic API definition or API Designer. + +If you’re using the newer Tyk OAS APIs, then check out the [Tyk OAS](/api-management/plugins/overview#tyk-oas-api-configuration) page. + +#### Using API Definition + +The configuration for an API to fetch and download a plugin bundle from a remote server is encapsulated within the `custom_middleware_bundle` field of the Tyk Classic API Definition. An illustrative example is listed below: + +```json {hl_lines=["33"], linenos=true, linenostart=1} +{ + "name": "Tyk Classic Bundle API", + "api_id": "1", + "org_id": "default", + "definition": { + "location": "header", + "key": "version" + }, + "auth": { + "auth_header_name": "authorization" + }, + "use_keyless": true, + "version_data": { + "not_versioned": true, + "versions": { + "Default": { + "name": "Default", + "expires": "3000-01-02 15:04", + "use_extended_paths": true, + "extended_paths": { + "ignored": [], + "white_list": [], + "black_list": [] + } + } + } + }, + "proxy": { + "listen_path": "/quickstart/", + "target_url": "http://httpbin.org", + "strip_listen_path": true + }, + "custom_middleware_bundle": "bundle-latest.zip" +} +``` expandable + +With the configuration given in the example above, calls to the API will invoke the custom plugins defined in the `manifest.json` file contained within `bundle-latest.zip` uploaded to your remote webserver, e.g. `http://your-example-plugin-server.com/plugins`. + +Tyk Gateway should be configured for downloading plugin bundles from a secured web server. Please consult the [plugin bundles](/api-management/plugins/overview#plugin-bundles) documentation for further details. + +#### Using API Designer + +To configure plugin bundles for Tyk Classic APIs click on the APIs menu item in the *API Management* menu of Dashboard and select your API to display the API editor screen. Subsequently, follow the steps below: + +1. **Access plugin options** + + Click on the *Advanced Options* tab and scroll down until the *Plugin Options* section is displayed. + + Tyk Classic Plugin Options section + +2. **Enter relative path to bundle file** + + Enter the relative path of the plugin bundle file in the *Plugin Bundle ID* field that Tyk Gateway should download from the web server hosting plugin bundles. + +3. **Save the API** + + Select the **save** or **update** button to apply the changes to your API. + +### Bundler CLI Tool + +The bundler tool is a CLI service, provided by _Tyk Gateway_ as part of its binary since v2.8. This lets you generate +[plugin bundles](/api-management/plugins/overview#plugin-bundles). + + + +Generated plugin bundles must be served using your own web server. + + + +Issue the following command to see more details on the `bundle` command: + +```bash +/opt/tyk-gateway/bin/tyk bundle -h +``` expandable + +--- + +#### Prerequisites + +To create plugin bundles you will need the following: + +- **Manifest.json**: The [manifest.json](/api-management/plugins/overview#manifest) file + contains the paths to the plugin source files and the name of the function implementing each plugin. The + _manifest.json_ file is mandatory and must exist on the Tyk Gateway file system. By default the bundle CLI looks for + a file named _manifest.json_ in the current working directory where the bundle command is run from. The exact location + can be specified using the `--manifest` command option. +- **Plugin source code files**: The plugin source code files should be contained relative to the directory in which the + _manifest.json_ file is located. The _manifest.json_ should contain relative path references to source code files. + + + +Source code files are not required when creating a plugin bundle for gRPC plugins since the plugin + source code is located at the gRPC server. + + + +- **Certificate key**: Plugin bundles can optionally be signed with an RSA private key. The corresponding public key + should be located in the file configured in environmental variable `TYK_GW_PUBLICKEYPATH` or the `public_key_path` + parameter in `tyk.conf`: + +```json +{ + "enable_bundle_downloader": true, + "bundle_base_url": "http://my-bundle-server.com/bundles/", + "public_key_path": "/path/to/my/pubkey.pem" +} +``` expandable + +--- + +#### Directory Structure + +A suggested directory structure is shown below for Golang, Javascript and Python bundles in the tabs below. + + + +Sub-directories (folders) are not supported inside the `bundle-directory` location. + + + + + +```bash +/bundle-directory +β”œβ”€β”€ manifest.json # Manifest file with plugin references +└── plugin.so # Compiled Golang plugin +``` + + + +```bash +/bundle-directory +β”œβ”€β”€ manifest.json # Manifest file with plugin references +β”œβ”€β”€ plugin1.js # First JavaScript plugin source file +└── plugin2.js # Second JavaScript plugin source file +``` + + + + + +```bash +/bundle-directory +β”œβ”€β”€ manifest.json # Manifest file with plugin references +β”œβ”€β”€ plugin1.py # First Python plugin source file +└── plugin2.py # Second Python plugin source file +``` + + + + + +The `manifest.json` will reference the files located in the `bundle-directory`, ensure plugin source files are organized relative to the manifest. The Tyk Gateway will load and execute these plugins based on the paths defined in the `manifest.json` file. + +Sample `manifest.json` is shown below for Golang, Javascript and Python bundles in the tabs below. + + + +```json +{ + "file_list": [ + "plugin.so" + ], + "custom_middleware": { + "pre": [ + { + "name": "PreMiddleware", + "path": "./plugin.so" + } + ], + "post": [ + { + "name": "PostMiddleware", + "path": "./plugin.so" + } + ], + "driver": "goplugin" + }, + "checksum": "", + "signature": "" +} + +``` + + + +```json +{ + "file_list": [ + "plugin1.js", + "plugin2.js" + ], + "custom_middleware": { + "pre": [ + { + "name": "PreMiddleware", + "path": "./plugin1.js" + } + ], + "post": [ + { + "name": "PostMiddleware", + "path": "./plugin2.js" + } + ], + "driver": "otto" + }, + "checksum": "", + "signature": "" +} +``` + + + + + +```json +{ + "file_list": [ + "plugin1.py", + "plugin2.py" + ], + "custom_middleware": { + "pre": [ + { + "name": "PreMiddleware", + "path": "./plugin1.py" + } + ], + "post": [ + { + "name": "PostMiddleware", + "path": "./plugin2.py" + } + ], + "driver": "python" + }, + "checksum": "", + "signature": "" +} +``` + + + + + +--- + +#### Creating a plugin bundle + +Run the following command to create the bundle: + +```bash +$ tyk bundle build +``` expandable + +The resulting file will contain all your specified files and a modified `manifest.json` with the checksum and signature +(if required) applied, in ZIP format. + +By default, Tyk will attempt to sign plugin bundles for improved security. If no private key is specified, the program +will prompt for a confirmation. Use `-y` to override this (see options below). + +--- + +#### Command Options + +Instructions on how to create plugin bundles is displayed by issuing the following command: + +```bash +/opt/tyk-gateway/bin/tyk bundle build -h +``` expandable + +The following options are supported: + +- `--manifest`: Specifies the path to the manifest file. This defaults to `manifest.json` within the current working + directory. +- `--output`: Specifies the name of the bundle file e.g. `--output bundle-latest.zip`. If this flag is not specified, + `bundle.zip` will be used. +- `-y`: Force tool to create unsigned bundle without prompting e.g. `$ tyk bundle build --output bundle-latest.zip -y`. +- `--key`: Specifies the path to your private key which is used to generate signed bundle e.g. + `$ tyk bundle build --output bundle-latest.zip --key=mykey.pem`. + +--- + +#### Docker Example + +Since v5.5 Tyk Gateway uses distroless docker images. + +For Gateway version < v5.5 it is possible to use Docker to create plugin bundles as shown in the example below. + +```bash +docker run --rm -it \ + --name bundler \ + -v `pwd`:/plugin-source \ + -v `pwd`/../../../confs/keys:/keys \ + -w /plugin-source \ + --entrypoint /bin/bash \ + tykio/tyk-gateway:v5.4.0 \ + -c 'export PATH="/opt/tyk-gateway:$$PATH"; tyk bundle build -o bundle.zip -k /keys/key.pem' +``` + +This Docker command runs a container using the `tykio/tyk-gateway:v5.4.0` image to build a Tyk plugin bundle. It mounts +the current directory from the host as `/plugin-source` and a directory containing keys as `/keys` inside the container. +The working directory within the container is set to `/plugin-source`, and the default entrypoint is overridden to use +`/bin/bash`. The command executed in the container exports a modified `PATH` to include the Tyk Gateway binaries, then +runs `tyk bundle build` to generate a plugin bundle named `bundle.zip`, using the specified key for authentication. The +container is automatically removed after the command completes, and the operation is conducted interactively. + +## Supported Languages + +The following languages are supported for custom plugins: +* [Golang](/api-management/plugins/golang#): A plugin written in Golang is called a **Native Plugin**. Tyk recommends using Go plugins for performance, flexibility, and nativity reasons (all Tyk components are written in Go). +* [JavaScript](/api-management/plugins/javascript#): A plugin written in Javascript uses JavaScript Virtual Machine (JSVM) interpreter. +* [Rich Plugins](/api-management/plugins/rich-plugins#) includes Python, Lua, gRPC - With gRPC, you can write plugins in Java, .NET, C++ / C#, PHP, and all other [gRPC supported languages](https://grpc.io/docs/languages/). +Rich plugins give ultimate flexibility in the language of implementation, however, there are some performance and management overheads when compared with native GoLang plugin. + +**Common To All Plugin Languages:** + +* Make Layer 4 (TCP) or Layer 7 (HTTP/REST/SOAP) calls +* Open Persistent Connections +* Modify the request in-flight +* Used to stop the request and return a [custom response](/api-management/plugins/plugin-types#return-overrides--returnoverrides) +* Be served using [Bundles](/api-management/plugins/overview#plugin-deployment-types) or by files on the file system, except gRPC of course which by definition is served by some webserver in the language of your choosing + +### Plugin Hook Types + +Tyk provide 5 different phases, i.e. hooks to inject custom plugin throughout the [API execution lifecycle](/api-management/traffic-transformation#request-middleware-chain). + +Not all hooks are supported in every language. The following table shows you which plugin language support which phase/hook: + +| | Auth | Pre | Post-Auth | Post | Response +| :------------ | :-------- | :---------- | :----------- | :------ | :----------- | +| GoLang | βœ… |βœ… |βœ… |βœ… |βœ… +| JavaScript | ❌ |βœ… |❌ |βœ… |❌ +| gRPC | βœ… |βœ… |βœ… |βœ… |βœ… +| Python | βœ… |βœ… |βœ… |βœ… |βœ… +| Lua | βœ… |βœ… |βœ… |βœ… |❌ + +More reading on the [hook types](/api-management/plugins/rich-plugins#coprocess-dispatcher---hooks) in rich plugins and explanation with common use case for each [hook type](/api-management/plugins/plugin-types#plugin-types) + + +### Plugin Driver Names + +We use the following Plugin driver names: + +| Plugin | Name | +| :---------- | :--------- | +| GoLang | goplugin | +| JavaScript | otto | +| gRPC | grpc | +| Python | python | +| Lua | lua | + +### Limitations + +What are the limitations to using this programming Language? + +| | GoLang | JavaScript | gRPC | Python | Lua +| :--- | :-------- | :------------------ | :----------- | :----------- | :----------- | +| Runs in Gateway process | βœ…
Runs
natively | βœ…
Built-In JSVM Interpreter | ❌
Standalone server | βœ…
Tyk talks with Python interpreter |βœ… +| Built-in SDK | βœ…
All Gateway Functionality | βœ…
[Yes](/api-management/plugins/javascript#javascript-api) | ❌ | βœ…
[Yes](/api-management/plugins/rich-plugins#tyk-python-api-methods) | ❌ +| TCP Connections

(DBs, Redis, etc)

| βœ… | ❌
Very Limited | βœ… | βœ… | βœ… | + +### Custom Plugin Table + +We have put together a [GitHub repo with a table of custom plugins](https://github.com/TykTechnologies/custom-plugins#custom-gateway-plugins) in various languages that you can experiment with. If you would like to submit one that you have developed, feel free to open an issue in the repo. + +### Differences between Rich Plugins and JSVM middleware + +#### JavaScript +The JavaScript Virtual Machine provides pluggable middleware that can modify a request on the fly and are designed to augment a running Tyk process, are easy to implement and run inside the Tyk process in a sandboxed *ECMAScript* interpreter. This is good, but there are some drawbacks with the JSVM: + +* **Performance**: JSVM is performant, but is not easy to optimize and is dependent on the [otto interpreter](https://github.com/robertkrimen/otto) - this is not ideal. The JSVM also requires a copy of the interpreter object for each request to be made, which can increase memory footprint. +* **Extensibility**: JSVM is a limited interpreter, although it can use some NPM modules, it isn't NodeJS so writing interoperable code (especially with other DBs) is difficult. +* **TCP Access**: The JSVM has no socket access so working with DB drivers and directly with Redis is not possible. + +#### Rich Plugins +Rich Plugins can provide replacements for existing middleware functions (as opposed to augmentation) and are designed to be full-blown, optimized, highly capable services. They enable a full customized architecture to be built that integrates with a user's infrastructure. + +Rich Plugins bring about the following improvements: + +* **Performance**: Run on STDIN (unix pipes), which are extremely fast and run in their own memory space, and so can be optimized for performance way beyond what the JSVM could offer. +* **Extensibility**: By allowing any language to be used so long as GRPC is supported, the extensibility of a CPH is completely open. +* **TCP Access**: Because a plugin is a separate process, it can have it's own low-level TCP connections opens to databases and services. + +## Plugin Caveats + +- Tyk Gateway manages plugins for each API within the same process. +- For [gRPC plugins](/api-management/plugins/rich-plugins#overview-1), Tyk Gateway can only be configured to integrate with one gRPC server. +- Javascript plugins only allow Pre and Post Request hooks of the API Request Lifecycle. + + +## Plugins Hub + +{/* Want to try and get a design layout setup for this that uses stylesheets from home page to offer similar layout */} + +Welcome to the Tyk Plugins Hub, dedicated to providing you with a curated list of resources that showcase how to develop Tyk Plugins. + +[Tyk Plugins](/api-management/plugins/overview#) are a powerful tool that allows you to develop custom middleware that can intercept requests at different stages of the request lifecycle, modifying/transforming headers and body content. + +Tyk has extensive support for writing custom plugins using a wide range of languages, most notably: Go, Python, Javascript etc. In fact, plugins can be developed using most languages via *gRPC*. + +### Blogs + +Selected blogs for plugin development are included below. Further examples are available at the Tyk [website](https://tyk.io/?s=plugin). + +1. **[Decoupling micro-services using Message-based RPC](https://medium.com/@asoorm/decoupling-micro-services-using-message-based-rpc-fa1c12409d8f)** + + - **Summary**: Explains how to write a plugin that intercepts an API request and forwards it to a gRPC server. The gRPC server processes the request and dispatches work to an RabbitMQ message queue. The source code is available in the accompanying [GitHub repository](https://github.com/asoorm/tyk-rmq-middleware) + +2. **[How to configure a gRPC server using Tyk](https://tyk.io/blog/how-to-configure-a-grpc-server-using-tyk/)** + + - **Summary**: Explains how to configure a Python implementation of a gRPC server to add additional logic to API requests. During the request lifecycle, the Tyk-Gateway acts as a gRPC client that contacts the Python gRPC server, providing additional custom logic. + +3. **[How to deploy Python plugins in Tyk running On Kubernetes](https://tyk.io/blog/how-to-deploy-python-plugins-in-tyk-running-on-kubernetes/)** + + - **Summary**: Explains how to deploy a custom Python plugin into a Tyk installation running on a Kubernetes cluster. + +### GitHub Repositories + +Here are some carefully selected GitHub repositories that will help you learn how to integrate and utilize Tyk Plugins in your development projects: + +1. **[Tyk Awesome Plugins](https://github.com/TykTechnologies/tyk-awesome-plugins)** + + - **Description**: Index of plugins developed using a variety of languages. + - **Key Features Demonstrated**: A comprehensive index for a collection of plugins that can be used with the Tyk API Gateway in areas such as: rate limiting, authentication and request transformation. The examples are developed using a diverse array of languages, including but not limited to: Python, JavaScript and Go. This broad language support ensures that developers from different backgrounds and with various language preferences can seamlessly integrate these plugins with their Tyk API Gateway implementations. + +2. **[Custom Plugin Examples](https://github.com/TykTechnologies/custom-plugin-examples/tree/master)** + + - **Description**: Index of examples for a range of plugin hooks (Pre, Post, Post-Auth and Response) developed using a variety of languages. + - **Key Features Demonstrated**: Specific examples include invoking an AWS lambda, inserting a new claim into a JWT, inject a signed JWT into authorization header, request header modification. A range of examples are available including Python, Java, Ruby, Javascript, NodeJS and Go. + +3. **[Environment For Plugin Development](https://github.com/TykTechnologies/custom-go-plugin)** + + - **Description**: Provides a docker-compose environment for developing your own custom Go plugins. + - **Key Features Demonstrated**: Showcases support for bundling plugins, uploading plugins to AWS S3 storage, test coverage etc. + + diff --git a/api-management/plugins/plugin-types.mdx b/api-management/plugins/plugin-types.mdx new file mode 100644 index 00000000..1cba8472 --- /dev/null +++ b/api-management/plugins/plugin-types.mdx @@ -0,0 +1,724 @@ +--- +title: "Plugin Types" +'og:description': "How to manage users, teams, permissions, rbac in Tyk Dashboard" +tags: ['Dashboard', 'User Management', 'RBAC', 'Role Based Access Control', 'User Groups', 'Teams', 'Permissions', 'API Ownership', 'SSO', 'Single Sing On', 'Multi Tenancy'] +sidebarTitle: "Plugin Types" +--- + +## Introduction + +Custom Plugins enable users to execute custom code to complete tasks specific to their use case, allowing users to complete tasks that would not otherwise be possible using Tyk’s standard middleware options. + +Tyk has a [pre-defined execution order](/api-management/traffic-transformation#request-middleware-chain) for the middleware which also includes **seven hooks** for the custom plugins. As such, users can execute, or `hook`, their plugin in these phases of the API request/response lifecycle based on their specific use case. + +## Plugin and Hook Types +This table includes all the plugin types with the relevant hooks, their place in the execution chain, description and examples: + +| Hook Type (in their execution order) | Plugin Type | HTTP Request/Response phase | Executed before/after reverse proxy to the upstream API | Details | Common Use Cases | +|--------------------------|----|---|--------------|--------------------|--------- +| Pre (Request) | Request Plugin | HTTP request | Before | The first thing to be executed, before any middleware | IP Rate Limit plugins, API Request enrichment | +| Authentication| Authentication Plugin | HTTP request | Before | Replaces Tyk's authentication & authorization middleware with your own business logic | When you need your a custom flow, for example, interfacing with legacy Auth database | +| Post-Auth (Request)| Authentication Plugin | HTTP request | Before | Executed immediately after authentication middleware | Additional special custom authentication is needed | +| Post (Request)| Request Plugin | HTTP request| Before | The final middleware to be executed during the *HTTP request* phase (see **Note** below) | Update the request before it gets to the upstream, for example, adding a header that might override another header, so we add it at the end to ensure it doesn't get overridden | +| Response Plugin| Response Plugin | HTTP Response | After | Executed after the reverse proxy to the upstream API | Executed straight after the reverse proxy returns from the upstream API to Tyk | Change the response before the user gets it, for example, change `Location` header from internal to an external URL | +| Analytics Plugin (Request+Response)| Analytics Plugin | HTTP request | After | The final middleware to be executed during the *HTTP response* phase | Change analytics records, for example, obfuscating sensitive data such as the `Authorization` header | + + + +There are two different options for the Post Plugin that is executed at the end of the request processing chain. The API-level Post Plugin is applied to all requests, whilst the [endpoint-level](/api-management/plugins/plugin-types#per-endpoint-custom-plugins) custom Golang plugin is only applied to requests made to specific endpoints. If both are configured, the endpoint-level plugin will be executed first. + + + +## Plugin Types + +Tyk supports four types of plugins: + +1. **[Request Plugin](#request-plugins)** +2. **[Authentication Plugin](#authentication-plugins)** +3. **[Response Plugin](#response-plugins)** +4. **[Analytics Plugin](#analytics-plugins)** + +## Request Plugins + +There are 4 different phases in the [request lifecycle](/api-management/traffic-transformation#request-middleware-chain) you can inject custom plugins, including [Authentication plugins](/api-management/plugins/plugin-types#authentication-plugins). There are performance advantages to picking the correct phase, and of course that depends on your use case and what functionality you need. + +### Hook Capabilities +| Functionality | Pre | Auth | Post-Auth | Post | +| :------------------------- | :---------- | :------------- | :----------- | :----------- | +| Can modify the Header | βœ… | βœ… | βœ… | βœ… +| Can modify the Body | βœ… | βœ… | βœ… |βœ… +| Can modify Query Params | βœ… | βœ… | βœ… |βœ… +| Can view Session1 Details (metadata, quota, context-vars, tags, etc) | ❌ | βœ… |βœ… |βœ… +| Can modify Session1 2 | ❌ | βœ… | ❌ |❌ +| Can Add More Than One3 | βœ… | ❌ |βœ… | βœ… + +1. A [Session object](/api-management/policies#what-is-a-session-object) contains allowances and identity information that is unique to each requestor + +2. You can modify the session by using your programming language's SDK for Redis. Here is an [example](https://github.com/TykTechnologies/custom-plugins/blob/master/plugins/go-auth-multiple_hook_example/main.go#L135) of doing that in Golang. + +3. For select hook locations, you can add more than one plugin. For example, in the same API request, you can have 3 Pre, 1 auth, 5 post-auth, and 2 post plugins. + +### Return Overrides / ReturnOverrides +You can have your plugin finish the request lifecycle and return a response with custom payload & headers to the requestor. + +[Read more here](/api-management/plugins/rich-plugins#returnoverrides) + +##### Python Example + +```python expandable +from tyk.decorators import * + +@Hook +def MyCustomMiddleware(request, session, spec): + print("my_middleware: MyCustomMiddleware") + request.object.return_overrides.headers['content-type'] = 'application/json' + request.object.return_overrides.response_code = 200 + request.object.return_overrides.response_error = "{\"key\": \"value\"}\n" + return request, session +``` + +##### JavaScript Example +```javascript expandable +var testJSVMData = new TykJS.TykMiddleware.NewMiddleware({}); + +testJSVMData.NewProcessRequest(function(request, session, config) { + request.ReturnOverrides.ResponseError = "Foobarbaz" + request.ReturnOverrides.ResponseBody = "Foobar" + request.ReturnOverrides.ResponseCode = 200 + request.ReturnOverrides.ResponseHeaders = { + "X-Foo": "Bar", + "X-Baz": "Qux" + } + return testJSVMData.ReturnData(request, {}); +}); +``` + + +## Authentication Plugins + +If you have unique authentication requirements, you can write a custom authentication plugin. + +### Session Authentication and Authorization + +A very important thing to understand when using custom authentication plugins is that Tyk will continue to perform session authentication and authorization using the information returned by your plugin. Tyk will cache this Session information. **This is necessary in order to do things like rate limiting, access control, quotas, throttling, etc.** + +Tyk will try to be clever about what to cache, but we need to help it. There are two ways to do that, with and without the `ID Extractor`. + +#### The ID Extractor + +The ID Extractor is a caching mechanism that's used in combination with Tyk Plugins. It can be used specifically with plugins that implement custom authentication mechanisms. The ID Extractor works for all rich plugins: gRPC-based plugins, Python and Lua. + +See [ID Extractor](/api-management/plugins/plugin-types#plugin-caching-mechanism) for more details. + +#### Token Metadata + +Tyk creates an in-memory object to track the rate limit, quotas, and more for each session. + +This is why we set the `token` metadata when using custom authentication middleware, in order to give Tyk a unique ID with which to track each session. + +For backwards compatibility, even when using an ID Extractor, we need to continue to set the `token` metadata. For example, when building a session object in GoLang custom middleware: + +```{.copyWrapper} expandable +object.Session = &coprocess.SessionState{ + LastUpdated: time.Now().String(), + Rate: 5, + Per: 10, + QuotaMax: int64(0), + QuotaRenews: time.Now().Unix(), + IdExtractorDeadline: extractorDeadline, + Metadata: map[string]string{ + "token": "my-unique-token", + }, + ApplyPolicies: ["5d8929d8f56e1a138f628269"], + } +``` +[source](https://github.com/TykTechnologies/tyk-grpc-go-basicauth-jwt/blob/master/main.go#L102) + +#### Without ID Extractor + +When not using ID Extractor, Tyk will continue to cache authenticated sessions returned by custom auth plugins. We must set a unique `token` field in the Metadata (see above) that Tyk will use to cache. + +### Supported Languages + +The following languages are supported for custom authentication plugins: + +- All Rich Plugins (gRPC, Python, Lua) +- GoLang + +See the [supported languages](/api-management/plugins/overview#supported-languages) section for custom authentication plugin examples in a language of your choosing. There's also a [blog that walks you through setting up gRPC custom auth in Java](https://tyk.io/how-to-setup-custom-authentication-middleware-using-grpc-and-java/). + +### Tyk Operator + +Please consult the Tyk Operator supporting documentation for examples of how to configure a Tyk Operator API to use: + +- [Go custom authentication plugin](/api-management/automations/operator#custom-plugin-auth-go) +- [gRPC custom authentication plugin](/api-management/automations/operator#custom-plugin-auth-grpc) + +## Response Plugins + +Since Tyk 3.0 we have incorporated response hooks, this type of hook allows you to modify the response object returned by the upstream. The flow is follows: + +- Tyk receives the request. +- Tyk runs the full middleware chain, including any other plugins hooks like Pre, Post, Custom Authentication, etc. +- Tyk sends the request to your upstream API. +- The request is received by Tyk and the response hook is triggered. +- Your plugin modifies the response and sends it back to Tyk. +- Tyk takes the modified response and is received by the client. + +This snippet illustrates the hook function signature: + +```python expandable +@Hook +def ResponseHook(request, response, session, metadata, spec): + tyk.log("ResponseHook is called", "info") + # In this hook we have access to the response object, to inspect it, uncomment the following line: + # print(response) + tyk.log("ResponseHook: upstream returned {0}".format(response.status_code), "info") + # Attach a new response header: + response.headers["injectedkey"] = "injectedvalue" + return response +``` + +If working with a Tyk Classic API, you would add this configuration to the API definition: + +``` expandable +{ + "custom_middleware": { + "response": [ + { + "name": "ResponseHook", + "path": "middleware/middleware.py" + } + ], + "driver": "python" + } +} +``` + + - `driver`: set this to the appropriate value for the plugin type (e.g. `python`, `goplugin`) + - `response`: this is the hook name. You use middleware with the `response` hook type because you want this custom middleware to process the request on its return leg of a round trip. + - `response.name`: is your function name from the plugin file. + - `response.path`: is the full or relative (to the Tyk binary) path to the plugin source file. Ensure Tyk has read access to this file. + +Starting from versions 5.0.4 and 5.1.1+ for our Go, Python and Ruby users we have introduced the `multivalue_headers` field to facilitate more flexible and efficient management of headers, particularly for scenarios involving a single header key associated with multiple values. The `multivalue_headers` field, similar to its predecessor, the `headers` field, is a key-value store. However, it can accommodate an array or list of string values for each key, instead of a single string value. This feature empowers you to represent multiple values for a single header key. Here's an example of how you might use `multivalue_headers`, using the Set-Cookie header which often has multiple values: + +``` +multivalue_headers = { + "Set-Cookie": ["sessionToken=abc123; HttpOnly; Secure", "language=en-US; Secure"], +} +``` + +In this example, Set-Cookie header has two associated values: `"sessionToken=abc123; HttpOnly; Secure"` and `"language=en-US; Secure"`. To help you understand this further, let's see how `multivalue_headers` can be used in a Tyk response plugin written in Python: + +```python expandable +from tyk.decorators import * +from gateway import TykGateway as tyk + +@Hook +def Del_ResponseHeader_Middleware(request, response, session, metadata, spec): + # inject a new header with 2 values + new_header = response.multivalue_headers.add() + new_header.key = "Set-Cookie" + new_header.values.extend("sessionToken=abc123; HttpOnly; Secure") + new_header.values.extend("language=en-US; Secure") + + tyk.log(f"Headers content :\n {response.headers}\n----------", "info") + tyk.log(f"Multivalue Headers updated :\n {response.multivalue_headers}\n----------", "info") + + return response +``` + +In this script, we add 2 values for the `Set-Cookie` header and then log both: the traditional `headers` and the new `multivalue_headers`. This is a great way to monitor your transition to `multivalue_headers` and ensure that everything is functioning as expected. + +Please note, while the `headers` field will continue to be available and maintained for backward compatibility, we highly encourage the adoption of `multivalue_headers` for the added flexibility in handling multiple header values. + +### Go response plugins + +[Go response plugins](/api-management/plugins/golang#creating-a-custom-response-plugin) have been available since Tyk v3.2. + +### Supported Response Plugin Languages + +See [Supported Plugins](/api-management/plugins/overview#supported-languages) for details on which languages the response plugin is supported in. + +## Analytics Plugins + +Since Tyk 4.1.0 we have incorporated analytic plugins which enables editing or removal of all parts of analytics records and raw request and responses recorded by Tyk at the gateway level. This feature leverages existing Go plugin infrastructure. + +- Tyk receives the request. +- Tyk runs the full middleware chain, including any other plugins hooks like Pre, Post, Custom Authentication, etc. +- Tyk sends the request to your upstream API. +- The response is received and analytics plugin function is triggered before recording the hit to redis. +- Your plugin modifies the analytics record and sends it back to Tyk. +- Tyk takes the modified analytics record and record the hit in redis. + +Example analytics Go plugins can be found [here](https://github.com/TykTechnologies/tyk/blob/master/test/goplugins/test_goplugin.go#L149) + +An analytics plugin is configured using the `analytics_plugin` configuration block within an API Definition. This contains the following configuration parameters: + +- `enable`: Set to `true` to enable the plugin +- `func_name`: The name of the function representing the plugin +- `plugin_path`: The path to the source code file containing the function that implements the plugin + + + + + +To enable the analytics rewriting functionality, adjust the following in API definition: + +```json expandable +{ + "analytics_plugin": { + "enable": true, + "func_name": "", + "plugin_path": "/analytics_plugin.so" + } +} +``` + + + + + +The example API Definition resource listed below listens on path */httpbin* and forwards requests upstream to *http://httpbin.org*. A Go Analytics Plugin is enabled for function *MaskAnalyticsData*, located within the */opt/tyk-gateway/plugins/example-plugin.so* shared object file. + +```yaml {linenos=table,hl_lines=["15-18"],linenostart=1} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: analytics-plugin +spec: + name: httpbin-analytics-plugin + active: true + protocol: http + proxy: + listen_path: /httpbin + strip_listen_path: true + target_url: http://httpbin.org + use_keyless: true + enable_detailed_recording: true + analytics_plugin: + enable: true + func_name: MaskAnalyticsData # Replace it with function name of your plugin + plugin_path: /opt/tyk-gateway/plugins/example-plugin.so # Replace it with path of your plugin file +``` + + + + + +
+ +## Advance Configuration + +There are two advance configuratin with plugin types: + +1. **[Per Endpoint Custom Plugin](#per-endpoint-custom-plugins)** +2. **[Plugin Caching Mechanism for Authentication Plugin](#plugin-caching-mechanism)** + +## Per-Endpoint Custom Plugins + +Tyk's custom plugin architecture allows you to deploy custom logic that will be invoked at certain points in the [middleware chain](/api-management/traffic-transformation#request-middleware-chain) as Tyk processes requests to your APIs. + +At the API-level, there are several points in the processing flow where custom plugins can be "hooked", as explained [here](/api-management/plugins/plugin-types#plugin-types). Each of these will be invoked for calls to any endpoint on an API. If you want to perform custom logic only for specific endpoints, you must include selective processing logic within the plugin. + +At the endpoint-level, Tyk provides the facility to attach a custom Golang plugin at the end of the request processing chain (immediately before the API-level post-plugin is executed). + +### When to use the per-endpoint custom plugin + +##### Aggregating data from multiple services + +From a custom plugin, you can make calls out to other internal and upstream APIs. You can then aggregate and process the responses, returning a single response object to the originating client. This allows you to configure a single externally facing API to simplify interaction with multiple internal services, leaving the heavy lifting to Tyk rather than standing up an aggregation service within your stack. + +##### Enforcing custom policies + +Tyk provides a very flexible middleware chain where you can combine functions to implement the access controls you require to protect your upstream services. Of course, not all scenarios can be covered by Tyk’s standard middleware functions, but you can use a custom plugin to apply whatever custom logic you require to optimize your API experience. + +##### Dynamic Routing + +With a custom plugin you can implement complex dynamic routing of requests made to a single external endpoint on to different upstream services. The flexibility of the virtual endpoint gives access to data within the request (including the key session) and also the ability to make calls to other APIs to make decisions on the routing of the request. It can operate as a super-powered URL rewrite middleware. + +### How the per-endpoint custom plugin works + +Tyk Gateway is written using Golang. This has a flexible plugin architecture which allows for custom code to be compiled separately from the gateway and then invoked natively by the gateway. When registering a custom Go plugin in the API definition, you must provide the location of the compiled plugin and also the name of the function to be invoked within that package. + +Go plugins must therefore be [compiled](/api-management/plugins/golang#plugin-compiler) and [loaded](/api-management/plugins/golang#loading-custom-go-plugins-into-tyk) into the Gateway in order that the function named in the plugin configuration in the API definition can be located and executed at the appropriate stage in the request middleware processing chain. + +The custom code within the plugin has access to contextual data such as the session object and API definition. If required, it can [terminate the request](/api-management/plugins/golang#terminating-the-request) and hence can provide a [Virtual Endpoint](/api-management/traffic-transformation/virtual-endpoints) style capability using the Go language, rather than JavaScript (as supported by the virtual endpoint middleware). This can then act as a high-performance replacement for the JavaScript virtual endpoints or for cases when you want to make use of external libraries. + +{/* proposed "summary box" to be shown graphically on each middleware page + ## Ignore Authentication middleware summary + - The Per-Endpoint Custom Plugin is an optional stage in Tyk's API Request processing chain, sitting between the [TBC]() and [TBC]() middleware. + - The Per-Endpoint Custom Plugin can be configured at the per-endpoint level within the API Definition and is supported by the API Designer within the Tyk Dashboard. */} + +### Using the Per-Endpoint Plugin with Tyk OAS APIs + +The [per-endpoint custom plugin](/api-management/plugins/plugin-types#per-endpoint-custom-plugins) provides the facility to attach a custom Go plugin at the end of the request processing chain. +This plugin allows you to add custom logic to the processing flow for the specific endpoint without adding to the processing complexity of other endpoints. +It can [terminate the request](/api-management/plugins/golang#terminating-the-request) if required, +and provides a [Virtual Endpoint](/api-management/traffic-transformation/virtual-endpoints) style capability using the Go language, rather than JavaScript (as supported by the virtual endpoint middleware). + +The middleware is configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation). You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the legacy Tyk Classic APIs, then check out the [Tyk Classic](/api-management/plugins/plugin-types#using-the-per-endpoint-plugin-with-tyk-classic-apis) page. + +#### Using Tyk OAS API Definition + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. The `path` can contain wildcards in the form of any string bracketed by curly braces, for example `{user_id}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +The endpoint plugin middleware (`postPlugins`) can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + +The `postPlugins` object has the following configuration: + +- `enabled`: enable the middleware for the endpoint +- `functionName`: this is the name of the Go function that will be executed when the middleware is triggered +- `path`: the relative path to the source file containing the compiled Go code + +You can chain multiple plugin functions in an array. Tyk will process them in the order they appear in the API definition. + +For example: + +```json {hl_lines=["39-45"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-endpoint-plugin", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/anything": { + "get": { + "operationId": "anythingget", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-endpoint-plugin", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-endpoint-plugin/", + "strip": true + } + }, + "middleware": { + "operations": { + "anythingget": { + "postPlugins": [ + { + "enabled": true, + "functionName": "myUniqueFunctionName", + "path": "/middleware/myPlugin.so" + } + ] + } + } + } + } +} +``` + +In this example the per-endpoint custom plugin middleware has been configured for HTTP `GET` requests to the `/anything` endpoint. For any call made to this endpoint, Tyk will invoke the function `myUniqueFunctionName` in the file located at `/middleware/myPlugin.so`. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the per-endpoint custom plugin middleware. + +#### Using API Designer + +Adding a per-endpoint custom plugin to your API endpoints is easy when using the API Designer in the Tyk Dashboard, simply follow these steps: + +1. **Add an endpoint** + + From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + + Tyk OAS API Designer showing no endpoints created + + Adding an endpoint to an API using the Tyk OAS API Designer + + Tyk OAS API Designer showing no middleware enabled on endpoint + +2. **Select the Go Post-Plugin middleware** + + Select **ADD MIDDLEWARE** and choose **Go Post-Plugin** from the *Add Middleware* screen. + + Adding the Go Post-Plugin middleware + +3. **Configure the middleware** + + You must provide the path to the compiled plugin and the name of the Go function that should be invoked by Tyk Gateway when the middleware is triggered. + + Configuring the per-endpoint custom plugin + +4. **Save the API** + + Select **ADD MIDDLEWARE** to save the middleware configuration. Remember to select **SAVE API** to apply the changes. + + + + + You are only able to add one custom plugin to each endpoint when using the API Designer, however you can add more by editing the API definition directly in the Raw Definition editor. + + + +### Using the Per-Endpoint Plugin with Tyk Classic APIs + +The [per-endpoint custom plugin](/api-management/plugins/plugin-types#per-endpoint-custom-plugins) provides the facility to attach a custom Golang plugin at the end of the request processing chain. +This plugin allows you to add custom logic to the processing flow for the specific endpoint without adding to the processing complexity of other endpoints. +It can [terminate the request](/api-management/plugins/golang#terminating-the-request), if required, +and hence can provide a [Virtual Endpoint](/api-management/traffic-transformation/virtual-endpoints) style capability using the Go language, rather than JavaScript (as supported by the virtual endpoint middleware). + +This middleware is configured in the Tyk Classic API Definition. You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the newer Tyk OAS APIs, then check out the [Tyk OAS](/api-management/plugins/plugin-types#using-the-per-endpoint-plugin-with-tyk-oas-apis) page. + +#### Using Tyk Classic API Definition + +To enable the middleware you must add a new `go_plugin` object to the `extended_paths` section of your API definition. + +The `go_plugin` object has the following configuration: + +- `path`: the endpoint path +- `method`: the endpoint HTTP method +- `func_name`: this is the "symbol" or function name you are calling in your Go plugin once loaded - a function can be called by one or more APIs +- `plugin_path`: the relative path of the shared object containing the function you wish to call, one or many `.so` files can be called + +You can register multiple plugin functions for a single endpoint. Tyk will process them in the order they appear in the API definition. + +For example: +```json {linenos=true, linenostart=1} expandable +{ + "extended_paths": { + "go_plugin": [ + { + "disabled": false, + "path": "/anything", + "method": "GET", + "plugin_path": "/middleware/myPlugin.so", + "func_name": "myUniqueFunctionName" + } + ] + } +} +``` + +In this example the per-endpoint custom plugin middleware has been configured for HTTP `GET` requests to the `/anything` endpoint. For any call made to this endpoint, Tyk will invoke the function `myUniqueFunctionName` in the file located at `/middleware/myPlugin.so`. + +#### Using API Designer + +You can use the API Designer in the Tyk Dashboard to add the per-endpoint custom plugin middleware for your Tyk Classic API by following these steps. + +1. **Add an endpoint for the path and select the plugin** + + From the **Endpoint Designer** add an endpoint that matches the path for which you want to trigger the custom plugin function. Select the **Go Plugin** plugin. + + Selecting the middleware + +2. **Locate the middleware in the raw API definition** + + Once you have selected the middleware for the endpoint, you need to switch to the *Raw Definition* view and then locate the `go_plugin` section (you can search within the text editor window). + + Locating the middleware configuration + +3. **Configure the middleware** + + Now you can directly edit the `plugin_path` and `func_name` to locate your compiled plugin function. + + Configuring the middleware + +4. **Save the API** + + Use the *save* or *create* buttons to save the changes and activate the middleware. + +## Plugin Caching Mechanism + +The **ID extractor** is a caching mechanism that's used in combination with Tyk Plugins. It is used specifically with plugins that implement **custom authentication mechanisms**. + +We use the term `ID` to describe any key that's used for authentication purposes. + +When a custom authentication mechanism is used, every API call triggers a call to the associated middleware function, if you're using a gRPC-based plugin this translates into a gRPC call. If you're using a native plugin -like a Python plugin-, this involves a Python interpreter call. + +The ID extractor works the following rich plugins: gRPC-based plugins, Python and Lua. + +### When to use the ID Extractor? + +The main idea of the ID extractor is to reduce the number of calls made to your plugin and cache the API keys that have been already authorized by your authentication mechanism. This means that after a successful authentication event, subsequent calls will be handled by the Tyk Gateway and its Redis cache, resulting in a performance similar to the built-in authentication mechanisms that Tyk provides. + +### When does the ID Extractor Run? + +When enabled, the ID extractor runs right before the authentication step, allowing it to take control of the flow and decide whether to call your authentication mechanism or not. + +If my ID is cached by this mechanism and my plugin isn't longer called, how do I expire it? +When you implement your own authentication mechanism using plugins, you initialise the session object from your own code. The session object has a field that's used to configure the lifetime of a cached ID, this field is called `id_extractor_deadline`. See [Plugin Data Structures](/api-management/plugins/rich-plugins#rich-plugins-data-structures) for more details. +The value of this field should be a UNIX timestamp on which the cached ID will expire, like `1507268142958`. It's an integer. + +For example, this snippet is used in a NodeJS plugin, inside a custom authentication function: + +``` expandable +// Initialize a session state object + var session = new tyk.SessionState() + // Get the current UNIX timestamp + var timestamp = Math.floor( new Date() / 1000 ) + // Based on the current timestamp, add 60 seconds: + session.id_extractor_deadline = timestamp + 60 + // Finally inject our session object into the request object: + Obj.session = session +``` + +If you already have a plugin that implements a custom authentication mechanism, appending the `id_extractor_deadline` and setting its value is enough to activate this feature. +In the above sample, Tyk will cache the key for 60 seconds. During that time any requests that use the cached ID won't call your plugin. + +### How to enable the ID Extractor + +The ID extractor is configured on a per API basis. +The API should be a protected one and have the `enable_coprocess_auth` flag set to true, like the following definition: + +```json expandable +{ + "name": "Test API", + "api_id": "my-api", + "org_id": "my-org", + "use_keyless": false, + "auth": { + "auth_header_name": "Authorization" + }, + "proxy": { + "listen_path": "/test-api/", + "target_url": "http://httpbin.org/", + "strip_listen_path": true + }, + "enable_coprocess_auth": true, + "custom_middleware_bundle": "bundle.zip" +} +``` + +If you're not using the Community Edition, check the API settings in the dashboard and make sure that "Custom Auth" is selected. + +The second requirement is to append an additional configuration block to your plugin manifest file, using the `id_extractor` key: + +```json expandable +{ + "custom_middleware": { + "auth_check": { "name": "MyAuthCheck" }, + "id_extractor": { + "extract_from": "header", + "extract_with": "value", + "extractor_config": { + "header_name": "Authorization" + } + }, + "driver": "grpc" + } +} +``` + +* `extract_from` specifies the source of the ID to extract. +* `extract_with` specifies how to extract and parse the extracted ID. +* `extractor_config` specifies additional parameters like the header name or the regular expression to use, this is different for every choice, see below for more details. + + +### Available ID Extractor Sources + +#### Header Source + +Use this source to extract the key from a HTTP header. Only the name of the header is required: + +```json expandable +{ + "id_extractor": { + "extract_from": "header", + "extract_with": "value", + "extractor_config": { + "header_name": "Authorization" + } + } +} +``` + +#### Form source + +Use this source to extract the key from a submitted form, where `param_name` represents the key of the submitted parameter: + + +```json expandable +{ + "id_extractor": { + "extract_from": "form", + "extract_with": "value", + "extractor_config": { + "param_name": "my_param" + } + } +} +``` + + +### Available ID Extractor Modes + +#### Value Extractor + +Use this to take the value as its present. This is commonly used in combination with the header source: + +```json expandable +{ + "id_extractor": { + "extract_from": "header", + "extract_with": "value", + "extractor_config": { + "header_name": "Authorization" + } + } +} +``` + +#### Regular Expression Extractor + +Use this to match the ID with a regular expression. This requires additional parameters like `regex_expression`, which represents the regular expression itself and `regex_match_index` which is the item index: + +```json expandable +{ + "id_extractor": { + "extract_from": "header", + "extract_with": "regex", + "extractor_config": { + "header_name": "Authorization", + "regex_expression": "[^-]+$", + "regex_match_index": 0 + } + } +} +``` + +Using the example above, if we send a header like `prefix-d28e17f7`, given the regular expression we're using, the extracted ID value will be `d28e17f7`. + +### Example Session +Here's an example of a Session being built in GoLang custom middleware: +```{.copyWrapper} expandable +extractorDeadline := time.Now().Add(time.Second * 5).Unix() +object.Session = &coprocess.SessionState{ + + LastUpdated: time.Now().String(), + Rate: 5, + Per: 10, + QuotaMax: int64(0), + QuotaRenews: time.Now().Unix(), + Metadata: map[string]string{ + "token": "my-unique-token", + }, + ApplyPolicies: ["5d8929d8f56e1a138f628269"], + } +``` +[source](https://github.com/TykTechnologies/tyk-grpc-go-basicauth-jwt/blob/master/main.go#L102) + +Note: When using an ID Extractor, you must set a `LastUpdated` or else token updates will not be applied. If you don't set an ID Extractor, Tyk will store session information in the cache based off the `token` field that is set in the metadata. + diff --git a/api-management/plugins/rich-plugins.mdx b/api-management/plugins/rich-plugins.mdx new file mode 100644 index 00000000..8da19fcd --- /dev/null +++ b/api-management/plugins/rich-plugins.mdx @@ -0,0 +1,3519 @@ +--- +title: "Rich Plugins" +'og:description': "How to manage users, teams, permissions, rbac in Tyk Dashboard" +sidebarTitle: "Rich Plugins" +--- + +import GrpcInclude from '/snippets/grpc-include.mdx'; +import { ButtonLeft } from '/snippets/ButtonLeft.mdx'; + +## Introduction + +Rich plugins make it possible to write powerful middleware for Tyk. Tyk supports: + +* [Python](/api-management/plugins/rich-plugins#overview) +* [gRPC](/api-management/plugins/rich-plugins#overview-1) +* [Lua](/api-management/plugins/rich-plugins#using-lua) + +gRPC provides the ability to write plugins using many languages including C++, Java, Ruby and C#. + +The dynamically built Tyk binaries can expose and call Foreign Function Interfaces in guest languages that extend the functionality of a gateway process. + +The plugins are able to directly call some Tyk API functions from within their guest language. They can also be configured so that they hook into various points along the standard middleware chain. + + + +When using Python plugins, the middleware function names are set globally. So, if you include two or more plugins that implement the same function, the last declared plugin implementation of the function will be returned. We plan to add namespaces in the future. + + + +## How do rich plugins work ? + +

ID Extractor & Auth Plugins

+The ID Extractor is a caching mechanism that's used in combination with Tyk Plugins. It can be used specifically with plugins that implement custom authentication mechanisms. The ID Extractor works for all rich plugins: gRPC-based plugins, Python and Lua. + +See [ID Extractor](/api-management/plugins/plugin-types#plugin-caching-mechanism) for more details. + +### Interoperability + +This feature implements an in-process message passing mechanism, based on [Protocol Buffers](https://developers.google.com/protocol-buffers/), any supported languages should provide a function to receive, unmarshal and process this kind of messages. + +The main interoperability task is achieved by using [cgo](https://golang.org/cmd/cgo/) as a bridge between a supported language -like Python- and the Go codebase. + +Your C bridge function must accept and return a `CoProcessMessage` data structure like the one described in [`api.h`](https://github.com/TykTechnologies/tyk/blob/master/coprocess/api.h), where `p_data` is a pointer to the serialised data and `length` indicates the length of it. + +```{.copyWrapper} +struct CoProcessMessage { + void* p_data; + int length; +}; +``` + +The unpacked data will hold the actual `CoProcessObject` data structure. + +- `HookType` - the hook type (see below) +- `Request` - the HTTP request +- `Session` - the [Tyk session object](/api-management/policies#session-object). +- `Metadata` - the metadata from the session data above (key/value string map). +- `Spec` - the API specification data. Currently organization ID, API ID and config_data. + +```{.copyWrapper} expandable +type CoProcessObject struct { + HookType string + Request CoProcessMiniRequestObject + Session SessionState + Metadata map[string]string + Spec map[string]string +} +``` + +### Coprocess Dispatcher + +`Coprocess.Dispatcher` describes a very simple interface for implementing the dispatcher logic, the required methods are: `Dispatch`, `DispatchEvent` and `Reload`. + +`Dispatch` accepts a pointer to a struct `CoProcessObject` (as described above) and must return an object of the same type. This method is called for every configured hook on every request. +Typically, it performs a single function call in the target language (such as `Python_DispatchHook` in `coprocess_python`), where the corresponding logic is handledβ€”mainly because different languages have different ways of loading, referencing, or calling middleware. + +`DispatchEvent` provides a way of dispatching Tyk events to a target language. This method doesn't return any variables but does receive a JSON-encoded object containing the event data. For extensibility purposes, this method doesn't use Protocol Buffers, the input is a `[]byte`, the target language will take this (as a `char`) and perform the JSON decoding operation. + +`Reload` is called when triggering a hot reload, this method could be useful for reloading scripts or modules in the target language. + +### Coprocess Dispatcher - Hooks + +This component is in charge of dispatching your HTTP requests to the custom middleware. The list, from top to bottom, shows the order of execution. The dispatcher follows the standard middleware chain logic and provides a simple mechanism for "hooking" your custom middleware behavior, the supported hooks are: + +* **Pre**: gets executed before the request is sent to your upstream target and before any authentication information is extracted from the header or parameter list of the request. When enabled, this applies to both keyless and protected APIs. +* **AuthCheck**: gets executed as a custom authentication middleware, instead of the standard ones provided by Tyk. Use this to provide your own authentication mechanism. +* **PostKeyAuth**: gets executed right after the authentication process. +* **Post**: gets executed after the authentication, validation, throttling, and quota-limiting middleware has been executed, just before the request is proxied upstream. Use this to post-process a request before sending it to your upstream API. This is only called when using protected APIs. If you want to call a hook after the authentication but before the validation, throttling and other middleware, see **PostKeyAuth**. +* **Response**: gets executed after the upstream API replies. The arguments passed to this hook include both the request and response data. Use this to modify the HTTP response before it's sent to the client. This hook also receives the request object, the session object, the metadata and API definition associated with the request. + + + + + Response hooks are not available for native Go plugins. Python and gRPC plugins are supported. + + + + +### Coprocess Gateway API + +[`coprocess_api.go`](https://github.com/TykTechnologies/tyk/tree/master/coprocess) provides a bridge between the Gateway API and C. Any function that needs to be exported should have the `export` keyword: + +```{.copyWrapper} expandable +//export TykTriggerEvent +func TykTriggerEvent( CEventName *C.char, CPayload *C.char ) { + eventName := C.GoString(CEventName) + payload := C.GoString(CPayload) + + FireSystemEvent(tykcommon.TykEvent(eventName), EventMetaDefault{ + Message: payload, + }) +} +``` + +You should also expect a header file declaration of this function in [`api.h`](https://github.com/TykTechnologies/tyk/blob/master/coprocess/api.h), like this: + +```{.copyWrapper} +#ifndef TYK_COPROCESS_API +#define TYK_COPROCESS_API +extern void TykTriggerEvent(char* event_name, char* payload); +#endif +``` + +The language binding will include this header file (or declare the function inline) and perform the necessary steps to call it with the appropriate arguments (like an FFI mechanism could do). As a reference, this is how this could be achieved if you're building a [Cython](http://cython.org/) module: + +```{.copyWrapper} expandable +cdef extern: + void TykTriggerEvent(char* event_name, char* payload); + +def call(): + event_name = 'my event'.encode('utf-8') + payload = 'my payload'.encode('utf-8') + TykTriggerEvent( event_name, payload ) +``` + +### Basic usage + +The intended way of using a Coprocess middleware is to specify it as part of an API Definition: + +```{.json} expandable +"custom_middleware": { + "pre": [ + { + "name": "MyPreMiddleware", + "require_session": false + }, + { + "name": "AnotherPreMiddleware", + "require_session": false + } + ], + "post": [ + { + "name": "MyPostMiddleware", + "require_session": false + } + ], + "post_key_auth": [ + { + "name": "MyPostKeyAuthMiddleware", + "require_session": true + } + ], + "auth_check": { + "name": "MyAuthCheck" + }, + "driver": "python" +} +``` + + +All hook types support chaining except the custom auth check (`auth_check`). + + + +## Rich Plugins Data Structures + +This page describes the data structures used by Tyk rich plugins, for the following plugin drivers: + +- Python (built-in) +- Lua (built-in) +- gRPC (external, compatible with any supported [gRPC language](https://grpc.io/docs/)) + +The Tyk [Protocol Buffer definitions](https://github.com/TykTechnologies/tyk/tree/master/coprocess/proto) are intended for users to generate their own bindings using the appropriate gRPC tools for the required target language. +The remainder of this document illustrates a class diagram and explins the attributes of the protobuf messages. + +--- + +### Class Diagram + +The class diagram below illustrates the structure of the [Object](#object) message, dispatched by Tyk to a gRPC server that handles custom plugins. + + + +--- + +### Object + +The `Coprocess.Object` data structure wraps a `Coprocess.MiniRequestObject` and `Coprocess.ResponseObject` It contains additional fields that are useful for users that implement their own request dispatchers, like the middleware hook type and name. +It also includes the session state object (`SessionState`), which holds information about the current key/user that's used for authentication. + +```protobuf expandable +message Object { + HookType hook_type = 1; + string hook_name = 2; + MiniRequestObject request = 3; + SessionState session = 4; + map metadata = 5; + map spec = 6; + ResponseObject response = 7; +} +``` + +#### Field Descriptions + +`hook_type` +Contains the middleware hook type: pre, post, custom auth. + +`hook_name` +Contains the hook name. + +`request` +Contains the request object, see `MiniRequestObject` for more details. + +`session` +Contains the session object, see `SessionState` for more details. + +`metadata` +Contains the metadata. This is a dynamic field. + +`spec` +Contains information about API definition, including `APIID`, `OrgID` and `config_data`. + +`response` +Contains information populated from the upstream HTTP response data, for response hooks. See [ResponseObject](#responseobject) for more details. All the field contents can be modified. + +--- + +### MiniRequestObject + +The `Coprocess.MiniRequestObject` is the main request data structure used by rich plugins. It's used for middleware calls and contains important fields like headers, parameters, body and URL. A `MiniRequestObject` is part of a `Coprocess.Object`. + +```protobuf expandable +message MiniRequestObject { + map headers = 1; + map set_headers = 2; + repeated string delete_headers = 3; + string body = 4; + string url = 5; + map params = 6; + map add_params = 7; + map extended_params = 8; + repeated string delete_params = 9; + ReturnOverrides return_overrides = 10; + string method = 11; + string request_uri = 12; + string scheme = 13; + bytes raw_body = 14; +} +``` + +#### Field Descriptions + +`headers` +A read-only field for reading headers injected by previous middleware. Modifying this field won't alter the request headers See `set_headers` and `delete_headers` for this. + +`set_headers` +This field appends the given headers (keys and values) to the request. + +`delete_headers` +This field contains an array of header names to be removed from the request. + +`body` +Contains the request body. See `ReturnOverrides` for response body modifications. + +`raw_body` +Contains the raw request body (bytes). + +`url` +The request URL. + +`params` +A read-only field that contains the request params. Modifying this value won't affect the request params. + +`add_params` +Add paramaters to the request. + +`delete_params` +This field contains an array of parameter keys to be removed from the request. + +`return_overrides` +See `ReturnOverrides` for more information. + +`method` +The request method, e.g. GET, POST, etc. + +`request_uri` +Raw unprocessed URL which includes query string and fragments. + +`scheme` +Contains the URL scheme, e.g. `http`, `https`. + +--- + +### ResponseObject + +The `ResponseObject` exists within an [object](#object) for response hooks. The fields are populated with the upstream HTTP response data. All the field contents can be modified. + +```protobuf expandable +syntax = "proto3"; + +package coprocess; + +message ResponseObject { + int32 status_code = 1; + bytes raw_body = 2; + string body = 3; + map headers = 4; + repeated Header multivalue_headers = 5; +} + +message Header { + string key = 1; + repeated string values = 2; +} +``` + +#### Field Descriptions + +`status_code` +This field indicates the HTTP status code that was sent by the upstream. + +`raw_body` +This field contains the HTTP response body (bytes). It's always populated. + +`body` +This field contains the HTTP response body in string format. It's not populated if the `raw_body` contains invalid UTF-8 characters. + +`headers` +A map that contains the headers sent by the upstream. + +`multivalue_headers` +A list of headers, each header in this list is a structure that consists of two parts: a key and its corresponding values. +The key is a string that denotes the name of the header, the values are a list of strings that hold the content of the header, this is useful when the header has multiple associated values. +This field is available for Go, Python and Ruby since tyk v5.0.4 and 5.1.1+. + +--- + +### ReturnOverrides + +The `ReturnOverrides` object, when returned as part of a `Coprocess.Object`, overrides the response of a given HTTP request. It also stops the request flow and the HTTP request isn't passed upstream. The fields specified in the `ReturnOverrides` object are used as the HTTP response. +A sample usage for `ReturnOverrides` is when a rich plugin needs to return a custom error to the user. + +```protobuf expandable +syntax = "proto3"; + +package coprocess; + +message ReturnOverrides { + int32 response_code = 1; + string response_error = 2; + map headers = 3; + bool override_error = 4; + string response_body = 5; +} +``` + +#### Field Descriptions + +`response_code` +This field overrides the HTTP response code and can be used for error codes (403, 500, etc.) or for overriding the response. + +`response_error` +This field overrides the HTTP response body. + +`headers` +This field overrides response HTTP headers. + +`override_error` +This setting provides enhanced customization for returning custom errors. It should be utilized alongside `response_body` for optimal effect. + +`response_body` +This field serves as an alias for `response_erro`r and holds the HTTP response body. + +--- + +

SessionState

+A `SessionState` data structure is created for every authenticated request and stored in Redis. It's used to track the activity of a given key in different ways, mainly by the built-in Tyk middleware like the quota middleware or the rate limiter. +A rich plugin can create a `SessionState` object and store it in the same way built-in authentication mechanisms do. This is what a custom authentication middleware does. This is also part of a `Coprocess.Object`. +Returning a null session object from a custom authentication middleware is considered a failed authentication and the appropriate HTTP 403 error is returned by the gateway (this is the default behavior) and can be overridden by using `ReturnOverrides`. + +#### Field Descriptions + +`last_check` +No longer used. + +`allowance` +No longer in use, should be the same as `rate`. + +`rate` +The number of requests that are allowed in the specified rate limiting window. + +`per` +The number of seconds that the rate window should encompass. + +`expires` +An epoch that defines when the key should expire. + +`quota_max` +The maximum number of requests allowed during the quota period. + +`quota_renews` +An epoch that defines when the quota renews. + +`quota_remaining` +Indicates the remaining number of requests within the user's quota, which is independent of the rate limit. + +`quota_renewal_rate` +The time in seconds during which the quota is valid. So for 1000 requests per hour, this value would be 3600 while `quota_max` and `quota_remaining` would be 1000. + +`access_rights` +Defined as a `map` instance, that maps the session's API ID to an [AccessDefinition](#access-definition). The AccessDefinition defines the [access rights](/api-management/policies#setting-granular-paths-on-a-per-key-basis) for the API in terms of allowed: versions and URLs(endpoints). Each URL (endpoint) has a list of allowed methods. For further details consult the tutorials for how to create a [security policy](/api-management/gateway-config-managing-classic#secure-an-api) for Tyk Cloud, Tyk Self Managed and Tyk OSS platforms. + +`org_id` +The organization this user belongs to. This can be used in conjunction with the org_id setting in the API Definition object to have tokens "owned" by organizations. + +`oauth_client_id` +This is set by Tyk if the token is generated by an OAuth client during an OAuth authorization flow. + +`basic_auth_data` +This section contains a hashed representation of the basic auth password and the hashing method used. +For further details see [BasicAuthData](#basicauthdata). + +`jwt_data` +Added to sessions where a Tyk key (embedding a shared secret) is used as the public key for signing the JWT. The JWT token's KID header value references the ID of a Tyk key. See [JWTData](#jwtdata) for an example. + +`hmac_enabled` +When set to `true` this indicates generation of a [HMAC signature](/basic-config-and-security/security/authentication-authorization/hmac-signatures) using the secret provided in `hmac_secret`. If the generated signature matches the signature provided in the *Authorization* header then authentication of the request has passed. + +`hmac_secret` +The value of the HMAC shared secret. + +`is_inactive` +Set this value to true to deny access. + +`apply_policy_id` +The policy ID that is bound to this token. + + + +Although `apply_policy_id` is still supported, it is now deprecated. `apply_policies` is now used to list your policy IDs as an array. This supports the **[Multiple Policy](/api-management/policies#partitioned-policy-functionality)** feature introduced in the **v2.4 - 1.4** release. + + + +`data_expires` +A value, in seconds, that defines when data generated by this token expires in the analytics DB (must be using Pro edition and MongoDB). + +`monitor` +Defines a [quota monitor](/api-management/gateway-events#monitoring-quota-consumption) containing a list of percentage threshold limits in descending order. These limits determine when webhook notifications are triggered for API users or an organization. Each threshold represents a percentage of the quota that, when reached, triggers a notification. See [Monitor](#monitor) for further details and an example. + +`enable_detailed_recording` +Set this value to true to have Tyk store the inbound request and outbound response data in HTTP Wire format as part of the analytics data. + +`metadata` +Metadata to be included as part of the session. This is a key/value string map that can be used in other middleware such as transforms and header injection to embed user-specific data into a request, or alternatively to query the providence of a key. + +`tags` +Tags are embedded into analytics data when the request completes. If a policy has tags, those tags will supersede the ones carried by the token (they will be overwritten). + +`alias` +As of v2.1, an Alias offers a way to identify a token in a more human-readable manner, add an Alias to a token in order to have the data transferred into Analytics later on so you can track both hashed and un-hashed tokens to a meaningful identifier that doesn't expose the security of the underlying token. + +`last_updated` +A UNIX timestamp that represents the time the session was last updated. Applicable to *Post*, *PostAuth* and *Response* plugins. When developing *CustomAuth* plugins developers should add this to the SessionState instance. + +`id_extractor_deadline` +This is a UNIX timestamp that signifies when a cached key or ID will expire. This relates to custom authentication, where authenticated keys can be cached to save repeated requests to the gRPC server. See [id_extractor](/api-management/plugins/plugin-types#plugin-caching-mechanism) and [Auth Plugins](/api-management/plugins/plugin-types#authentication-plugins) for additional information. + +`session_lifetime` +UNIX timestamp that denotes when the key will automatically expire. AnyΒ·subsequent API request made using the key will be rejected. Overrides the global session lifetime. See [Key Expiry and Deletion](/api-management/policies#set-physical-key-expiry-and-deletion) for more information. + +--- + +

AccessDefinition

+```protobuf expandable +message AccessDefinition { + string api_name = 1; + string api_id = 2; + repeated string versions = 3; + repeated AccessSpec allowed_urls = 4; +} +``` + +Defined as an attribute within a [SessionState](#session-state) instance. Contains the allowed versions and URLs (endpoints) for the API that the session request relates to. Each URL (endpoint) specifies an associated list of allowed methods. See also [AccessSpec](#access-spec). + +#### Field Descriptions + +`api_name` +The name of the API that the session request relates to. + +`api_id` +The ID of the API that the session request relates to. + +`versions` +List of allowed API versions, e.g. `"versions": [ "Default" ]`. + +`allowed_urls` List of [AccessSpec](#access-spec) instances. Each instance defines a URL (endpoint) with an associated allowed list of methods. If all URLs (endpoints) are allowed then the attribute is not set. + +--- + +

AccessSpec

+Defines an API's URL (endpoint) and associated list of allowed methods + +```protobuf +message AccessSpec { + string url = 1; + repeated string methods = 2; +} +``` + +#### Field Descriptions + +`url` +A URL (endpoint) belonging to the API associated with the request session. + +`methods` +List of allowed methods for the URL (endpoint), e.g. `"methods": [ "GET". "POST", "PUT", "PATCH" ]`. + +--- + +### BasicAuthData + +The `BasicAuthData` contains a hashed password and the name of the hashing algorithm used. This is represented by the `basic_auth_data` attribute in [SessionState](#session-state) message. + +```yaml +"basicAuthData": { + "password": , + "hash": +} +``` + +#### Field Descriptions + +`password` +A hashed password. + +`hash` +Name of the [hashing algorithm](/api-management/policies#access-key-hashing) used to hash the password. + +--- + +### JWTData + +Added to [sessions](#session-state) where a Tyk key (embedding a shared secret) is used as the public key for signing the JWT. This message contains the shared secret. + +```yaml +"jwtData": { + "secret": "the_secret" +} +``` + +#### Field Descriptions + +`secret` +The shared secret. + +--- + +

Monitor

+Added to a [session](#session-state) when [monitor quota thresholds](/api-management/gateway-events#monitoring-quota-consumption) are defined within the Tyk key. This message contains the quota percentage threshold limits, defined in descending order, that trigger webhook notification. + +```yaml +message Monitor { + repeated double trigger_limits = 1; +} +``` + +#### Field Descriptions + +`trigger_limits` +List of trigger limits defined in descending order. Each limit represents the percentage of the quota that must be reached in order for the webhook notification to be triggered. + +```yaml +"monitor": { + "trigger_limits": [80.0, 60.0, 50.0] +} +``` + +--- + +
+ + + +## Using Python + +### Overview + +#### Requirements + +Since v2.9, Tyk supports any currently stable [Python 3.x version](https://www.python.org/downloads/). The main requirement is to have the Python shared libraries installed. These are available as `libpython3.x` in most Linux distributions. + +- Python3-dev +- [Protobuf](https://pypi.org/project/protobuf/): provides [Protocol Buffers](https://developers.google.com/protocol-buffers/) support +- [gRPC](https://pypi.org/project/grpcio/): provides [gRPC](http://www.grpc.io/) support + +#### Important Note Regarding Performance + +Python plugins are [embedded](https://docs.python.org/3/extending/embedding.html) within the Tyk Gateway process. Tyk Gateway integrates with Python custom plugins via a [cgo](https://golang.org/cmd/cgo) bridge. + +`Tyk Gateway` <-> CGO <-> `Python Custom Plugin` + +In order to integrate with Python custom plugins, the *libpython3.x.so* shared object library is used to embed a Python interpreter directly in the Tyk Gateway. Further details can be found [here](/api-management/plugins/rich-plugins#coprocess-gateway-api) + +This allows combining the strengths of both Python and Go in a single application. However, it's essential to be aware of the potential complexities and performance implications of mixing languages, as well as the need for careful memory management when working with Python objects from Go. + +The Tyk Gateway process initialises the Python interpreter using [Py_initialize](https://docs.python.org/3/c-api/init.html#c.Py_Initialize). The Python [Global Interpreter Lock (GIL)](https://docs.python.org/3/glossary.html/#term-global-interpreter-lock) allows only one thread to execute Python bytecode at a time, ensuring thread safety and simplifying memory management. While the GIL simplifies these aspects, it can limit the scalability of multi-threaded applications, particularly those with CPU-bound tasks, as it restricts parallel execution of Python code. + +In the context of custom Python plugins, API calls are queued and the Python interpreter handles requests sequentially, processing them one at a time. Subsequently, this would consume large amounts of memory, and network sockets would remain open and blocked until the API request is processed. + +#### Install the Python development packages + + + + + + +Starting from Tyk Gateway version `v5.3.0`, Python is no longer bundled with the official Tyk Gateway Docker image by default, to address security vulnerabilities in the Python libraries highlighted by [Docker Scout](https://docs.docker.com/scout/). +
+Whilst Python plugins are still supported by Tyk Gateway, if you want to use them you must extend the image to add support for Python. For further details, please refer to the [release notes](/developer-support/release-notes/gateway) for Tyk Gateway `v5.3.0`. + + + +If you wish to use Python plugins using Docker, you can extend the official Tyk Gateway Docker image by adding Python to it. + +This example Dockerfile extends the official Tyk Gateway image to support Python plugins by installing python and the required modules: + +```dockerfile expandable +ARG BASE_IMAGE +FROM ${BASE_IMAGE} AS base + +FROM python:3.11-bookworm +COPY --from=base /opt/tyk-gateway/ /opt/tyk-gateway/ +RUN pip install setuptools && pip install google && pip install 'protobuf==4.24.4' + +EXPOSE 8080 80 443 + +ENV PYTHON_VERSION=3.11 +ENV PORT=8080 + +WORKDIR /opt/tyk-gateway/ + +ENTRYPOINT ["/opt/tyk-gateway/tyk" ] +CMD [ "--conf=/opt/tyk-gateway/tyk.conf" ] +``` + +To use this, you simply run `docker build` with this Dockerfile, providing the Tyk Gateway image that you would like to extend as build argument `BASE_IMAGE`. +As an example, this command will extend Tyk Gateway `v5.3.0` to support Python plugins, generating the image `tyk-gateway-python:v5.3.0`: + +```bash +docker build --build-arg BASE_IMAGE=tykio/tyk-gateway:v5.3.0 -t tyk-gateway-python:v5.3.0 . +``` + + + + + +```apt +apt install python3 python3-dev python3-pip build-essential +``` + +#### Install the Required Python Modules + +Make sure that "pip" is available in your system, it should be typically available as "pip", "pip3" or "pipX.X" (where X.X represents the Python version): + +```pip3 +pip3 install protobuf grpcio +``` + + + + + +```yum +yum install python3-devel python3-setuptools +python3 -m ensurepip +``` + +#### Install the Required Python Modules + +Make sure that "pip" is now available in your system, it should be typically available as "pip", "pip3" or "pipX.X" (where X.X represents the Python version): + +```pip3 +pip3 install protobuf grpcio +``` + + + + + +#### Python versions + +Newer Tyk versions provide more flexibility when using Python plugins, allowing the users to set which Python version to use. By default, Tyk will try to use the latest version available. + +To see the Python initialisation log, run the Tyk gateway in debug mode. + +To use a specific Python version, set the `python_version` flag under `coprocess_options` in the Tyk Gateway configuration file (tyk.conf). + + + +Tyk doesn't support Python 2.x. + + + +#### Troubleshooting + +To verify that the required Python Protocol Buffers module is available: + +```python3 +python3 -c 'from google import protobuf' +``` + +No output is expected from this command on successful setups. + +#### How do I write Python Plugins? + +We have created [a demo Python plugin repository](https://github.com/TykTechnologies/tyk-plugin-demo-python). + +The project implements a simple middleware for header injection, using a Pre hook (see [Tyk custom middleware hooks](/api-management/plugins/rich-plugins#coprocess-dispatcher---hooks). A single Python script contains the code for it, see [middleware.py](https://github.com/TykTechnologies/tyk-plugin-demo-python/blob/master/middleware.py). + + +### Custom Authentication Plugin Tutorial + +#### Introduction +This tutorial will guide you through the creation of a custom authentication plugin, written in Python. +A custom authentication plugin allows you to implement your own authentication logic and override the default Tyk authentication mechanism. The sample code implements a very simple key check; currently it supports a single, hard-coded key. It could serve as a starting point for your own authentication logic. We have tested this plugin with Ubuntu 14. + +The code used in this tutorial is also available in [this GitHub repository](https://github.com/TykTechnologies/tyk-plugin-demo-python). + +#### Requirements + +* Tyk API Gateway: This can be installed using standard package management tools like Yum or APT, or from source code. See [here](/tyk-self-managed/install) for more installation options. + +##### Dependencies + +* The Tyk CLI utility, which is bundled with our RPM and DEB packages, and can be installed separately from [https://github.com/TykTechnologies/tyk-cli](https://github.com/TykTechnologies/tyk-cli) +* In Tyk 2.8 the Tyk CLI is part of the gateway binary, you can find more information by running "tyk help bundle". +* Python 3.4 + +#### Create the Plugin +The first step is to create a new directory for your plugin file: + +```bash +mkdir ~/my-tyk-plugin +cd ~/my-tyk-plugin +``` + +Next you need to create a manifest file. This file contains information about our plugin file structure and how you expect it to interact with the API that will load it. +This file should be named `manifest.json` and needs to contain the following content: + +```json expandable +{ + "file_list": [ + "middleware.py" + ], + "custom_middleware": { + "driver": "python", + "auth_check": { + "name": "MyAuthMiddleware" + } + } +} +``` + +* The `file_list` block contains the list of files to be included in the bundle, the CLI tool expects to find these files in the current working directory. +* The `custom_middleware` block contains the middleware settings like the plugin driver we want to use (`driver`) and the hooks that our plugin will expose. You use the `auth_check` for this tutorial. For other hooks see [here](/api-management/plugins/rich-plugins#coprocess-dispatcher---hooks). +* The `name` field references the name of the function that you implement in your plugin code: `MyAuthMiddleware`. +* You add an additional file called `middleware.py`, this will contain the main implementation of our middleware. + + + + + Your bundle should always contain a file named `middleware.py` as this is the entry point file. + + + +##### Contents of middleware.py + +You import decorators from the Tyk module as this gives you the `Hook` decorator, and you import [Tyk Python API helpers](/api-management/plugins/rich-plugins#tyk-python-api-methods) + +You implement a middleware function and register it as a hook, the input includes the request object, the session object, the API meta data and its specification: + +```python expandable +from tyk.decorators import * +from gateway import TykGateway as tyk + +@Hook +def MyAuthMiddleware(request, session, metadata, spec): + auth_header = request.get_header('Authorization') + if auth_header == '47a0c79c427728b3df4af62b9228c8ae': + tyk.log("I'm logged!", "info") + tyk.log("Request body" + request.object.body, "info") + tyk.log("API config_data" + spec['config_data'], "info") + session.rate = 1000.0 + session.per = 1.0 + metadata["token"] = "47a0c79c427728b3df4af62b9228c8ae" + return request, session, metadata +``` + + +You can modify the `manifest.json` to add as many files as you want. Files that aren't listed in the `manifest.json` file will be ignored when building the plugin bundle. + +#### Building the Plugin + +A plugin bundle is a packaged version of the plugin, it may also contain a cryptographic signature of its contents. The `-y` flag tells the Tyk CLI tool to skip the signing process in order to simplify the flow of this tutorial. For more information on the Tyk CLI tool, see [here](/api-management/plugins/overview#plugin-bundles). + +You will use the Dockerised version of the Tyk CLI tool to bundle our package. + +First, export your Tyk Gateway version to a variable. +```bash +##### THIS MUST MATCH YOUR TYK GATEWAY VERSION +$ IMAGETAG=v3.1.2 +``` + +Then run the following commands to generate a `bundle.zip` in your current directory: +```docker +$ docker run \ + --rm -w "/tmp" -v $(pwd):/tmp \ + --entrypoint "/bin/sh" -it \ + tykio/tyk-gateway:$IMAGETAG \ + -c '/opt/tyk-gateway/tyk bundle build -y' +``` + +**Success!** + +You should now have a `bundle.zip` file in the plugin directory. + +#### Publishing the Plugin + +To allow Tyk access to the plugin bundle, you need to serve this file using a web server. For this tutorial we'll use the Python built-in HTTP server (check the official docs for additional information). This server listens on port 8000 by default. To start it use: + +`python3 -m http.server` + +When the server is started our current working directory is used as the web root path, this means that our `bundle.zip` file should be accessible from the following URL: + +`http://:8000/bundle.zip` + +The Tyk Gateway fetches and loads a plugin bundle during startup time and subsequent reloads. For updating plugins using the hot reload feature, you should use different plugin bundle names as you expect them to be used for versioning purposes, e.g. bundle-1, bundle-2, etc. +If a bundle already exists, Tyk will skip the download process and load the version that's already present. + +#### Configure Tyk + +You will need to modify the Tyk global configuration file (`tyk.conf`) to use Python plugins. The following block should be present in this file: + +```json expandable +"coprocess_options": { + "enable_coprocess": true, + "python_path_prefix": "/opt/tyk-gateway" +}, +"enable_bundle_downloader": true, +"bundle_base_url": "http://dummy-bundle-server.com/bundles/", +"public_key_path": "/path/to/my/pubkey" +``` + +##### Options + +* `enable_coprocess`: This enables the plugin +* `python_path_prefix`: Sets the path to built-in Tyk modules, this will be part of the Python module lookup path. The value used here is the default one for most installations. +* `enable_bundle_downloader`: This enables the bundle downloader +* `bundle_base_url`: This is a base URL that will be used to download the bundle. You should replace the `bundle_base_url` with the appropriate URL of the web server that's serving your plugin bundles. For now HTTP and HTTPS are supported but we plan to add more options in the future (like pulling directly from S3 buckets). You use the URL that's exposed by the test HTTP server in the previous step. +* `public_key_path`: Modify `public_key_path` in case you want to enforce the cryptographic check of the plugin bundle signatures. If the `public_key_path` isn't set, the verification process will be skipped and unsigned plugin bundles will be loaded normally. + +#### Configure an API Definition + +There are two important parameters that you need to add or modify in the API definition. +The first one is `custom_middleware_bundle` which must match the name of the plugin bundle file. If we keep this with the default name that the Tyk CLI tool uses, it will be `bundle.zip`. + +`"custom_middleware_bundle": "bundle.zip"` + +The second parameter is specific to this tutorial, and should be used in combination with `use_keyless` to allow an API to authenticate against our plugin: + +`"use_keyless": false` +`"enable_coprocess_auth": true` + +`"enable_coprocess_auth"` will instruct the Tyk gateway to authenticate this API using the associated custom authentication function that's implemented by the plugin. + +#### Configuration via the Tyk Dashboard + +To attach the plugin to an API, From the **Advanced Options** tab in the **API Designer** enter **bundle.zip** in the **Plugin Bundle ID** field. + +Plugin Options + +You also need to modify the authentication mechanism that's used by the API. +From the **Core Settings** tab in the **API Designer** select **Use Custom Authentication (Python, CoProcess, and JSVM plugins)** from the **Authentication - Authentication Mode** drop-down list. + +Advanced Options + +#### Testing the Plugin + +Now you can simply make an API call against the API for which we've loaded the Python plugin. + + +##### If Running Tyk Gateway from Source + +At this point you have your test HTTP server ready to serve the plugin bundle and the configuration with all the required parameters. +The final step is to start or restart the **Tyk Gateway** (this may vary depending on how you setup Tyk). +A separate service is used to load the Tyk version that supports Python (`tyk-gateway-python`), so we need to stop the standard one first (`tyk-gateway`): + +```service +service tyk-gateway stop +service tyk-gateway-python start +``` + +From now on you should use the following command to restart the service: + +```service +service tyk-gateway-python restart +``` + +A cURL request will be enough for testing our custom authentication middleware. + +This request will trigger a bad authentication: + +```curl +curl http://:8080/my-api/my-path -H 'Authorization: badtoken' +``` + +This request will trigger a successful authentication. You are using the token that's set by your Python plugin: + +```curl +curl http://:8080/my-api/my-path -H 'Authorization: 47a0c79c427728b3df4af62b9228c8ae' +``` + +#### What's Next? + +In this tutorial you learned how Tyk plugins work. For a production-level setup we suggest the following steps: + +* Configure Tyk to use your own key so that you can enforce cryptographic signature checks when loading plugin bundles, and sign your plugin bundles! +* Configure an appropriate web server and path to serve your plugin bundles. + + +### Add Python Plugin To Your Gateway + +#### API settings + +To add a Python plugin to your API, you must specify the bundle name using the `custom_middleware_bundle` field: + +```{.json} expandable +{ + "name": "Tyk Test API", + "api_id": "1", + "org_id": "default", + "definition": { + "location": "header", + "key": "version" + }, + "auth": { + "auth_header_name": "authorization" + }, + "use_keyless": true, + "version_data": { + "not_versioned": true, + "versions": { + "Default": { + "name": "Default", + "expires": "3000-01-02 15:04", + "use_extended_paths": true, + "extended_paths": { + "ignored": [], + "white_list": [], + "black_list": [] + } + } + } + }, + "proxy": { + "listen_path": "/quickstart/", + "target_url": "http://httpbin.org", + "strip_listen_path": true + }, + "custom_middleware_bundle": "test-bundle" +} +``` + +#### Global settings + +To enable Python plugins you need to add the following block to `tyk.conf`: + +```{.copyWrapper} expandable +"coprocess_options": { + "enable_coprocess": true, + "python_path_prefix": "/opt/tyk-gateway" +}, +"enable_bundle_downloader": true, +"bundle_base_url": "http://dummy-bundle-server.com/bundles/", +"public_key_path": "/path/to/my/pubkey", +``` + +`enable_coprocess`: enables the rich plugins feature. + +`python_path_prefix`: Sets the path to built-in Tyk modules, this will be part of the Python module lookup path. The value used here is the default one for most installations. + +`enable_bundle_downloader`: enables the bundle downloader. + +`bundle_base_url`: is a base URL that will be used to download the bundle, in this example we have `test-bundle` specified in the API settings, Tyk will fetch the URL for your specified bundle server (in the above example): `dummy-bundle-server.com/bundles/test-bundle`. You need to create and then specify your own bundle server URL. + +`public_key_path`: sets a public key, this is used for verifying signed bundles, you may omit this if unsigned bundles are used. + +### Tyk Python API methods + +Python plugins may call these Tyk API methods: + +#### store_data(key, value, ttl) + +`store_data` sets a Redis `key` with the specified `value` and `ttl`. + +#### get_data(key) + +`get_data` retrieves a Redis `key`. + +#### trigger_event(event_name, payload) + +`trigger_event` triggers an internal Tyk event, the `payload` must be a JSON object. + +#### log(msg, level) + +`log` will log a message (`msg`) using the specified `level`. + +#### log_error(*args) + +`log_error` is a shortcut for `log`, it uses the error log level. + +### Python Performance + +These are some benchmarks performed on Python plugins. Python plugins run in a standard Python interpreter, embedded inside Tyk. + +Python Performance + +Python Performance + +## Using gRPC + +### Overview + +gRPC is a very powerful framework for RPC communication across different [languages](https://www.grpc.io/docs). It was created by Google and makes heavy use of HTTP2 capabilities and the [Protocol Buffers](https://developers.google.com/protocol-buffers/) serialisation mechanism to dispatch and exchange requests between Tyk and your gRPC plugins. + +When it comes to built-in plugins, we have been able to integrate several languages like Python, Javascript & Lua in a native way: this means the middleware you write using any of these languages runs in the same process. At the time of writing, the following languages are supported: C++, Java, Objective-C, Python, Ruby, Go, C# and Node.JS. + +For supporting additional languages we have decided to integrate gRPC connections and perform the middleware operations within a gRPC server that is external to the Tyk process. Please contact us to learn more: + +
+ + + +Tyk has built-in support for gRPC backends, enabling you to build rich plugins using any of the gRPC supported languages. See [gRPC by language](http://www.grpc.io/docs/) for further details. + +--- + +#### Architectural overview + +An example architecture is illustrated below. + +Using gRPC for plugins + +Here we can see that Tyk Gateway sends requests to an external Java gRPC server to handle authentication, via a CustomAuth plugin. The flow is as follows: + +- Tyk receives a HTTP request. +- Tyk serialises the request and session into a protobuf message that is dispatched to your gRPC server. +- The gRPC server performs custom middleware operations (for example, any modification of the request object). Each plugin (Pre, PostAuthKey, Post, Response etc.) is handled as separate gRPC request. +- The gRPC server sends the request back to Tyk. +- Tyk proxies the request to your upstream API. + +--- + +#### Use cases + +Deploying an external gRPC server to handle plugins provides numerous technical advantages: + +- Allows for independent scalability of the service from the Tyk Gateway. +- Utilizes a custom-designed server tailored to address specific security concerns, effectively mitigating various security risks associated with native plugins. + +--- + +#### Limitations + +At the time of writing the following features are currently unsupported and unavailable in the serialised request: +- Client certificiates +- OAuth keys +- For graphQL APIs details concerning the *max_query_depth* is unavailable +- A request query parameter cannot be associated with multiple values + +--- + +#### Developer Resources + +The [Protocol Buffers](https://github.com/TykTechnologies/tyk/tree/master/coprocess/proto ) and [bindings](https://github.com/TykTechnologies/tyk/tree/master/coprocess/bindings) provided by Tyk should be used in order for successful ommunication between Tyk Gateway and your gRPC plugin server. Documentation for the protobuf messages is available in the [Rich Plugins Data Structures](/api-management/plugins/rich-plugins#rich-plugins-data-structures) page. + +You can generate supporting HTML documentation using the *docs* task in the [Taskfile](https://github.com/TykTechnologies/tyk/blob/master/coprocess/proto/Taskfile.yml) file of the [Tyk repository](https://github.com/TykTechnologies/tyk). This documentation explains the protobuf messages and services that allow gRPC plugins to handle a request made to the Gateway. Please refer to the README file within the proto folder of the tyk repository for further details. + +You may re-use the bindings that were generated for our samples or generate the bindings youself for Go, Python and Ruby, as implemented by the *generate* task in the [Taskfile](https://github.com/TykTechnologies/tyk/blob/master/coprocess/proto/Taskfile.yml) file of the [Tyk repository](https://github.com/TykTechnologies/tyk). + +If you wish to generate bindings for another target language you may generate the bindings yourself. The [Protocol Buffers](https://developers.google.com/protocol-buffers/) and [gRPC documentation](http://www.grpc.io/docs) provide specific requirements and instructions for each language. + +--- + +#### What's next? + +See our [getting started](/api-management/plugins/rich-plugins#key-concepts) guide for an explanation of how to write and configure gRPC plugins. + +--- + +### Key Concepts + +This document serves as a developer's guide for understanding the key concepts and practical steps for writing and configuring gRPC plugins for Tyk Gateway. It provides technical insights and practical guidance to seamlessly integrate Tyk plugins into your infrastructure through gRPC. The goal is to equip developers with the knowledge and tools needed to effectively utilize gRPC for enhancing Tyk Gateway functionalities. + +This comprehensive guide covers essential tasks, including: + +1. **Developing a gRPC Server:** Learn how to develop a gRPC server using [Tyk protocol buffers](https://github.com/TykTechnologies/tyk/tree/master/coprocess/proto). The gRPC server facilitates the execution of Tyk plugins, which offer custom middleware for various phases of the API request lifecycle. By integrating these plugins, developers can enable Tyk Gateway with enhanced control and flexibility in managing API requests, allowing for fine-grained customization and tailored processing at each stage of the request lifecycle. + +2. **Configuring Tyk Gateway:** Set up Tyk Gateway to communicate with your gRPC Server and, optionally, an external secured web server hosting the gRPC plugin bundle for API configurations. Configure Tyk Gateway to fetch the bundle configured for an API from the web server, enabling seamless integration with gRPC plugins. Specify connection settings for streamlined integration. + +3. **API Configuration:** Customize API settings within Tyk Gateway to configure gRPC plugin utilization. Define plugin hooks directly within the API Definition or remotely via an external web server for seamless request orchestration. Tyk plugins provide custom middleware for different phases of the API request lifecycle, enhancing control and flexibility. + +4. **API Testing:** Test that Tyk Gateway integrates with your gRPC server for the plugins configured for your API. + +--- + +#### Develop gRPC server + +Develop your gRPC server, using your preferred language, to handle requests from Tyk Gateway for each of the required plugin hooks. These hooks allow Tyk Gateway to communicate with your gRPC server to execute custom middleware at various stages of the API request lifecycle. + +##### Prerequisites + +The following prerequisites are necessary for developing a gRPC server that integrates with Tyk Gateway. + +####### Tyk gRPC Protocol Buffers + +A collection of [Protocol Buffer](https://github.com/TykTechnologies/tyk/tree/master/coprocess/proto) messages are available in the Tyk Gateway repository to allow Tyk Gateway to integrate with your gRPC server, requesting execution of plugin code. These messages establish a standard set of data structures that are serialised between Tyk Gateway and your gRPC Server. Developers should consult the [Rich Plugins Data Structures](/api-management/plugins/rich-plugins#rich-plugins-data-structures) page for further details. + +####### Protocol Buffer Compiler + +The protocol buffer compiler, `protoc`, should be installed to generate the service and data structures in your preferred language(s) from the [Tyk gRPC Protocol Buffer](https://github.com/TykTechnologies/tyk/tree/master/coprocess/proto) files. Developers should consult the [installation](https://grpc.io/docs/protoc-installation/) documentation at [grpc.io](https://grpc.io/) for an explanation of how to install `protoc`. + +##### Generate Bindings + +Generate the bindings (service and data structures) for your target language using the `protoc` compiler. Tutorials are available at [protobuf.dev](https://protobuf.dev/getting-started/) for your target language. + +##### Implement service + +Your gRPC server should implement the *Dispatcher* service to enable Tyk Gateway to integrate with your gRPC server. The Protocol Buffer definition for the *Dispatcher* service is listed below: + +```protobuf +service Dispatcher { + rpc Dispatch (Object) returns (Object) {} + rpc DispatchEvent (Event) returns (EventReply) {} +} +``` + +The *Dispatcher* service contains two RPC methods, *Dispatch* and *DispatchEvent*. Dispatch handles a requests made by Tyk Gateway for each plugin configured in your API. DispatchEvent receives notification of an event. + +Your *Dispatch* RPC should handle the request made by Tyk Gateway, implementing custom middleware for the intended plugin hooks. Each plugin hook allows Tyk Gateway to communicate with your gRPC server to execute custom middleware at various stages of the API request lifecycle, such as Pre, PostAuth, Post, Response etc. The Tyk Protocol Buffers define the [HookType](https://github.com/TykTechnologies/tyk/blob/master/coprocess/proto/coprocess_common.proto) enumeration to inspect the type of the intended gRPC plugin associated with the request. This is accessible as an attribute on the *Object* message, e.g. *object_message_instance.hook_type*. + +##### Developer resources + +Consult the [Tyk protocol buffers](https://github.com/TykTechnologies/tyk/tree/master/coprocess/proto) for the definition of the service and data structures that enable integration of Tyk gateway with your gRPC server. Tyk provides pre-generated [bindings](https://github.com/TykTechnologies/tyk/tree/master/coprocess/bindings) for C++, Java, Python and Ruby. + +Example tutorials are available that explain how to generate the protobuf bindings and implement a server for [Java](/api-management/plugins/rich-plugins#create-a-request-transformation-plugin-with-java), [.NET](/api-management/plugins/rich-plugins#create-custom-auth-plugin-with-dotnet) and [NodeJS](/api-management/plugins/rich-plugins#create-custom-auth-plugin-with-dotnet). + +Tyk Github repositories are also available with examples for [Ruby](https://github.com/TykTechnologies/tyk-plugin-demo-ruby) and [C#/.NET](https://github.com/TykTechnologies/tyk-plugin-demo-dotnet) + +--- + +#### Configure Tyk Gateway + +Configure Tyk Gateway to issue requests to your gRPC server and optionally, specify the URL of the web server that will serve plugin bundles. + +##### Configure gRPC server + +Modify the root of your `tyk.conf` file to include the *coprocess_options* section, similar to that listed below: + +```yaml expandable +"coprocess_options": { + "enable_coprocess": true, + "coprocess_grpc_server": "tcp://127.0.0.1:5555", + "grpc_authority": "localhost", + "grpc_recv_max_size": 100000000, + "grpc_send_max_size": 100000000 +}, +``` + +A gRPC server can configured under the `coprocess_options` section as follows: + +- `enable_coprocess`: Enables the rich plugins feature. +- `coprocess_grpc_server`: Specifies the gRPC server URL, in this example we're using TCP. Tyk will attempt a connection on startup and keep reconnecting in case of failure. +- `grpc_recv_max_size`: Specifies the message size supported by the gateway gRPC client, for receiving gRPC responses. +- `grpc_send_max_size`: Specifies the message size supported by the gateway gRPC client for sending gRPC requests. +- `grpc_authority`: The `authority` header value, defaults to `localhost` if omitted. Allows configuration according to [RFC 7540](https://datatracker.ietf.org/doc/html/rfc7540#section-8.1.2.3). + +When using gRPC plugins, Tyk acts as a gRPC client and dispatches requests to your gRPC server. gRPC libraries usually set a default maximum size, for example, the official gRPC Java library establishes a 4 +MB message size [https://jbrandhorst.com/post/grpc-binary-blob-stream/](https://jbrandhorst.com/post/grpc-binary-blob-stream/). + +Configuration parameters are available for establishing a message size in both directions (send and receive). For most use cases and especially if you're dealing with multiple hooks, where the same request object is dispatched, it is recommended to set both values to the same size. + +##### Configure Web server (optional) + +Tyk Gateway can be configured to download the gRPC plugin configuration for an API from a web server. For further details related to the concept of bundling plugins please refer to [plugin bundles](/api-management/plugins/overview#plugin-bundles). + +```yaml +"enable_bundle_downloader": true, +"bundle_base_url": "https://my-bundle-server.com/bundles/", +"public_key_path": "/path/to/my/pubkey", +``` + +The following parameters can be configured: +- `enable_bundle_downloader`: Enables the bundle downloader to download bundles from a webserver. +- `bundle_base_url`: Base URL from which to serve bundled plugins. +- `public_key_path`: Public key for bundle verification (optional) + +The `public_key_path` value is used for verifying signed bundles, you may omit this if unsigned bundles are used. + +--- + +#### Configure API + +Plugin hooks for your APIs in Tyk can be configured either by directly specifying them in a configuration file on the Gateway server or by hosting the configuration externally on a web server. This section explains how to configure gRPC plugins for your API endpoints on the local Gateway or remotely from an external secured web server. + +##### Local + +This section provides examples for how to configure gRPC plugin hooks, locally within an API Definition. Examples are provided for Tyk Gateway and Tyk Operator. + +###### Tyk Gateway + +For configurations directly embedded within the Tyk Gateway, plugin hooks can be defined within your API Definition. An example snippet from a Tyk Classic API Definition is provided below: + +```yaml expandable +"custom_middleware": { + "pre": [ + {"name": "MyPreMiddleware"} + ], + "post": [ + {"name": "MyPostMiddleware"} + ], + "auth_check": { + "name": "MyAuthCheck" + }, + "driver": "grpc" +} +``` + +For example, a Post request plugin hook has been configured with name `MyPostMiddleware`. Before the request is sent upstream Tyk Gateway will serialize the request into a [Object protobuf message](/api-management/plugins/rich-plugins#object) with the `hook_name` property set to `MyPostMiddleware` and the `hook_type` property set to `Post`. This message will then then be dispatched to the gRPC server for processing before the request is sent upstream. + +
+ + +Ensure the plugin driver is configured as type *grpc*. Tyk will issue a request to your gRPC server for each plugin hook that you have configured. + + + +###### Tyk Operator + +The examples below illustrate how to configure plugin hooks for an API Definition within Tyk Operator. + +Setting the `driver` configuring parameter to `gRPC` instructs Tyk Gateway to issue a request to your gRPC server for each plugin hook that you have configured. + +**Pre plugin hook example** + +In this example we can see that a `custom_middleware` configuration block has been used to configure a gRPC Pre request plugin hook with name `HelloFromPre`. Before any middleware is executed Tyk Gateway will serialize the request into a [Object protobuf message](/api-management/plugins/rich-plugins#object) with the `hook_name` property set to `HelloFromPre` and the `hook_type` property set to `Pre`. This message will then then be dispatched to the gRPC server. + +```yaml {linenos=table,hl_lines=["14-18"],linenostart=1} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-grpc-pre +spec: + name: httpbin-grpc-pre + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.default.svc:8000 + listen_path: /httpbin-grpc-pre + strip_listen_path: true + custom_middleware: + driver: grpc + pre: + - name: HelloFromPre + path: "" +``` + +**Post plugin hook example** + +In the example we can see that a `custom_middleware` configuration block has been used to configure a gRPC Post plugin with name `HelloFromPost`. + +Before the request is sent upstream Tyk Gateway will serialize the request and session details into a [Object protobuf message](/api-management/plugins/rich-plugins#object) with the `hook_name` property set to `HelloFromPost` and the `hook_type` property set to `Post`. This message will then then be dispatched to the gRPC server for processing before the request is sent upstream. + +```yaml {linenos=table,hl_lines=["14-18"],linenostart=1} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-grpc-post +spec: + name: httpbin-grpc-post + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.default.svc:8000 + listen_path: /httpbin-grpc-post + strip_listen_path: true + custom_middleware: + driver: grpc + post: + - name: HelloFromPost + path: "" +``` + +##### Remote + +It is possible to configure your API so that it downloads a bundled configuration of your plugins from an external webserver. The bundled plugin configuration is contained within a zip file. + +A gRPC plugin bundle is similar to the [standard bundling mechanism](/api-management/plugins/overview#plugin-bundles). The standard bundling mechanism zips the configuration and plugin source code, which will be executed by Tyk. Conversely, a gRPC plugin bundle contains only the configuration (`manifest.json`), with plugin code execution being handled independently by the gRPC server. + +Bundling a gRPC plugin requires the following steps: +- Create a `manifest.json` that contains the configuration of your plugins +- Build a zip file that bundles your plugin +- Upload the zip file to an external secured webserver +- Configure your API to download your plugin bundle + +###### Create the manifest file + +The `manifest.json` file specifies the configuration for your gRPC plugins. An example `manifest.json` is listed below: + +```yaml expandable +{ + "file_list": [], + "custom_middleware": { + "pre": [{"name": "MyPreMiddleware"}], + "post": [{"name": "MyPostMiddleware"}], + "auth_check": {"name": "MyAuthCheck"}, + "driver": "grpc" + }, + "checksum": "", + "signature": "" +} +``` + + + +The source code files, *file_list*, are empty for gRPC plugins. Your gRPC server contains the source code for handling plugins. + + + +###### Build plugin bundle + +A plugin bundle can be built using the Tyk Gateway binary and should only contain the `manifest.json` file: + +```bash +tyk bundle build -output mybundle.zip -key mykey.pem +``` + +The example above generates a zip file, name `mybundle.zip`. The zip file is signed with key `mykey.pem`. + +The resulting bundle file should then be uploaded to the webserver that hosts your plugin bundles. + +###### Configure API + +####### Tyk Gateway + +To add a gRPC plugin to your API definition, you must specify the bundle file name within the `custom_middleware_bundle` field: + +```yaml +{ + "name": "Tyk Test API", + ... ++ "custom_middleware_bundle": "mybundle.zip" +} +``` + +The value of the `custom_middleware_bundle` field will be used in combination with the gateway settings to construct a bundle URL. For example, if Tyk Gateway is configured with a webserver base URL of `https://my-bundle-server.com/bundles/` then an attempt would be made to download the bundle from `https://my-bundle-server.com/bundles/mybundle.zip`. + +####### Tyk Operator + + Currently this feature is not yet documented with a Tyk Operator example for configuring an API to use plugin bundles. For further details please reach out and contact us on the [community support forum](https://community.tyk.io). + +--- + +#### Test your API Endpoint + +It is crucial to ensure the security and reliability of your gRPC server. As the developer, it is your responsibility to verify that your gRPC server is secured and thoroughly tested with appropriate test coverage. Consider implementing unit tests, integration tests and other testing methodologies to ensure the robustness of your server's functionality and security measures. This step ensures that the Tyk Gateway properly communicates with your gRPC server and executes the custom logic defined by the plugin hooks. + +Test the API endpoint using tools like *Curl* or *Postman*. Ensure that your gRPC server is running and the gRPC plugin(s) are functioning. An example using *Curl* is listed below: + +```bash +curl -X GET https://www.your-gateway-server.com:8080/api/path +``` + +Replace `https://www.your-gateway-server.com:8080/api/path` with the actual endpoint of your API. + +--- + +#### Summary + +This guide has explained the key concepts and processes for writing gRPC plugins that integrate with Tyk Gateway. The following explanations have been given: + +- Prerequisites for developing a gRPC server for your target language. +- The *Dispatcher* service interface. +- How to configure Tyk Gateway to integrate with your gRPC server. +- How to configure Tyk Gateway with an optional external web server for fetching plugin configuration. +- How to configure gRPC plugins for your APIs. +- How to test your API integration with your gRPC server using curl. + +--- + +#### What's Next? + +- Consult the [Protocol Buffer messages](/api-management/plugins/rich-plugins#rich-plugins-data-structures) that Tyk Gateway uses when making a request to a gRPC server. +- Visit tutorial guides that explain how to implement a [Java](/api-management/plugins/rich-plugins#create-a-request-transformation-plugin-with-java), [.NET](/api-management/plugins/rich-plugins#create-custom-auth-plugin-with-dotnet) and [NodeJS](/api-management/plugins/rich-plugins#create-custom-auth-plugin-with-dotnet) gRPC server. +- Visit our [plugins hub](/api-management/plugins/overview#plugins-hub) to explore further gRPC development examples and resources. + +--- + + + + +### Getting Started: Creating A Python gRPC Server + +In the realm of API integration, establishing seamless connections between services is paramount. + +Understanding the fundamentals of gRPC server implementation is crucial, especially when integrating with a Gateway solution like Tyk. This guide aims to provide practical insights into this process, starting with the basic principles of how to implement a Python gRPC server that integrates with Tyk Gateway. + +#### Objectives + +By the end of this guide, you will be able to implement a gRPC server that will integrate with Tyk Gateway, setting the stage for further exploration in subsequent parts: + +- Establishing the necessary tools, Python libraries and gRPC service definition for implementing a gRPC server that integrates with Tyk Gateway. +- Developing a basic gRPC server that echoes the request payload to the console, showcasing the core principles of integration. +- Configuring Tyk Gateway to interact with our gRPC server, enabling seamless communication between the two services. + +Before implementing our first gRPC server it is first necessary to understand the service interface that defines how Tyk Gateway integrates with a gRPC server. + + +#### Tyk Dispatcher Service + +The *Dispatcher* service, defined in the [coprocess_object.proto](https://github.com/TykTechnologies/tyk/blob/master/coprocess/proto/coprocess_object.proto) file, contains the *Dispatch* RPC method, invoked by Tyk Gateway to request remote execution of gRPC plugins. Tyk Gateway dispatches accompanying data relating to the original client request and session. The service definition is listed below: + +```protobuf +service Dispatcher { + rpc Dispatch (Object) returns (Object) {} + rpc DispatchEvent (Event) returns (EventReply) {} +} +``` + +On the server side, we will implement the *Dispatcher* service methods and a gRPC server to handle requests from Tyk Gateway. The gRPC infrastructure decodes incoming requests, executes service methods and encodes service responses. + +Before we start developing our gRPC server we need to setup our development environment with the supporting libraries and tools. + + +#### Prerequisites + +Firstly, we need to download the [Tyk Protocol Buffers](https://github.com/TykTechnologies/tyk/tree/master/coprocess/proto) and install the Python protoc compiler. + +We are going to use the *protoc* compiler to generate the supporting classes and data structures to implement the *Dispatcher* service. + + +##### Tyk Protocol Buffers + +Issue the following command to download and extract the Tyk Protocol Buffers from the Tyk GitHub repository: + +```bash +curl -sL "https://github.com/TykTechnologies/tyk/archive/master.tar.gz " -o tyk.tar.gz && \ + mkdir tyk && \ + tar -xzvf tyk.tar.gz --strip-components=1 -C tyk && \ + mv tyk/coprocess/proto/* . && \ + rm -r tyk tyk.tar.gz +``` + +##### Install Dependencies + +We are going to setup a Python virtual environment and install some supporting dependencies. Assuming that you have Python [virtualenv](https://virtualenv.pypa.io/en/latest/installation.html) already installed, then issue the following commands to setup a Python virtual environment containing the grpcio and grpcio-tools libraries: + +```bash +python3 -m venv .venv +source .venv/bin/activate +pip install –upgrade pip +pip install grpcio grpcio-tools grpcio-reflection +``` + +The [grpcio](https://pypi.org/project/grpcio/) library offers essential functionality to support core gRPC features such as message serialisation and deserialisation. The [grpcio-tools](https://pypi.org/project/grpcio-tools/) library provides the Python *protoc* compiler that we will use to generate the supporting classes and data structures to implement our gRPC server. The [grpcio-reflection](https://pypi.org/project/grpcio-reflection/) library allows clients to query information about the services and methods provided by a gRPC server at runtime. It enables clients to dynamically discover available services, their RPC methods, in addition to the message types and field names associated with those methods. + + +##### Install grpcurl + +Follow the [installation instructions](https://github.com/fullstorydev/grpcurl?tab=readme-ov-file#installation) to install grpcurl. We will use grpcurl to send test requests to our gRPC server. + + +##### Generate Python Bindings + +We are now able to generate the Python classes and data structures to allow us to implement our gRPC server. To accomplish this we will use the Python *protoc* command as listed below: + +```bash +python -m grpc_tools.protoc --proto_path=. --python_out=. --grpc_python_out=. *.proto +``` + +This compiles the Protocol Buffer files (*.proto) from the current working directory and generates the Python classes representing the Protocol Buffer messages and services. A series of *.py* files should now exist in the current working directory. We are interested in the *coprocess_object_pb2_grpc.py* file, containing a default implementation of *Tyk’s Dispatcher* service. + +Inspect the generated Python file, *coprocess_object_pb2_grpc.py*, containing the *DispatcherServicer* class: + +```python expandable +class DispatcherServicer(object): + """ GRPC server interface, that must be implemented by the target language """ + def Dispatch(self, request, context): + """ Accepts and returns an Object message """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def DispatchEvent(self, request, context): + """ Dispatches an event to the target language """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') +``` + +This superclass contains a default stub implementation for the **Dispatch** and **DispatchEvent** RPC methods, each defining request and context parameters: + +The *request* parameter allows our server to access the message payload sent by Tyk Gateway. We can use this data, pertaining to the request and session, to process and generate a response. + +The *context* parameter provides additional information and functionalities related to the RPC call, such as timeout limits, cancelation signals etc. This is a [grpc.ServicerContext](https://grpc.github.io/grpc/python/grpc.html#grpc.ServicerContext) or a [grpc.aio.ServicerContext](https://grpc.github.io/grpc/python/grpc_asyncio.html#grpc.aio.ServicerContext), object depending upon whether a synchronous or AsyncIO gRPC server is implemented. + +In the next step we will implement a subclass that will handle requests made by Tyk Gateway for remote execution of custom plugins. + + +#### Implement Dispatcher Service + +We will now develop the *Dispatcher* service, adding implementations of the *Dispatch* and *DispatchEvent* methods, to allow our gRPC server to integrate with Tyk Gateway. Before we continue, create a file, *async_server.py*, within the same folder as the generated Protocol Buffer (.proto) files. + + +##### Dispatch + +Our implementation of the Dispatch RPC method will deserialize the request payload and output to the console as JSON format. This serves as a useful development and debugging aid, allowing inspection of the request and session state dispatched by Tyk Gateway to our gRPC server. + +Copy and paste the following source code into the *async_server.py* file. Notice that we have used type hinting to aid readability. The type hints are located within the type hint files (.pyi) we generated with the protoc compiler. + + +```python expandable +import asyncio +import grpc +import json +import signal +import logging +from google.protobuf.json_format import MessageToJson +from grpc_reflection.v1alpha import reflection +import coprocess_object_pb2_grpc +import coprocess_object_pb2 +from coprocess_common_pb2 import HookType +from coprocess_session_state_pb2 import SessionState +class PythonDispatcher(coprocess_object_pb2_grpc.DispatcherServicer): + async def Dispatch( + self, object: coprocess_object_pb2.Object, context: grpc.aio.ServicerContext + ) -> coprocess_object_pb2.Object: + logging.info(f"STATE for {object.hook_name}\n{MessageToJson(object)}\n") + if object.hook_type == HookType.Pre: + logging.info(f"Pre plugin name: {object.hook_name}") + logging.info(f"Activated Pre Request plugin from API: {object.spec.get('APIID')}") + elif object.hook_type == HookType.CustomKeyCheck: + logging.info(f"CustomAuth plugin: {object.hook_name}") + logging.info(f"Activated CustomAuth plugin from API: {object.spec.get('APIID')}") + elif object.hook_type == HookType.PostKeyAuth: + logging.info(f"PostKeyAuth plugin name: {object.hook_name}") + logging.info(f"Activated PostKeyAuth plugin from API: {object.spec.get('APIID')}") + elif object.hook_type == HookType.Post: + logging.info(f"Post plugin name: {object.hook_name}") + logging.info(f"Activated Post plugin from API: {object.spec.get('APIID')}") + elif object.hook_type == HookType.Response: + logging.info(f"Response plugin name: {object.hook_name}") + logging.info(f"Activated Response plugin from API: {object.spec.get('APIID')}") + logging.info("--------\n") + return object +``` + +Our *Dispatch* RPC method accepts the two parameters, *object* and *context*. The object parameter allows us to inspect the state and session of the request object dispatched by Tyk Gateway, via accessor methods. The *context* parameter can be used to set timeout limits etc. associated with the RPC call. + +The important takeaways from the source code listing above are: + +- The [MessageToJson](https://googleapis.dev/python/protobuf/latest/google/protobuf/json_format.html#google.protobuf.json_format.MessageToJson) function is used to deserialize the request payload as JSON. +- In the context of custom plugins we access the *hook_type* and *hook_name* attributes of the *Object* message to determine which plugin to execute. +- The ID of the API associated with the request is accessible from the spec dictionary, *object.spec.get('APIID')*. + +An implementation of the *Dispatch* RPC method must return the object payload received from Tyk Gateway. The payload can be modified by the service implementation, for example to add or remove headers and query parameters before the request is sent upstream. + + +##### DispatchEvent + +Our implementation of the *DispatchEvent* RPC method will deserialize and output the event payload as JSON. Append the following source code to the *async_server.py* file: + +```python expandable + async def DispatchEvent( + self, event: coprocess_object_pb2.Event, context: grpc.aio.ServicerContext + ) -> coprocess_object_pb2.EventReply: + event = json.loads(event.payload) + http://logging.info (f"RECEIVED EVENT: {event}") + return coprocess_object_pb2.EventReply() +``` + +The *DispatchEvent* RPC method accepts the two parameters, *event* and *context*. The event parameter allows us to inspect the payload of the event dispatched by Tyk Gateway. The context parameter can be used to set timeout limits etc. associated with the RPC call. + +The important takeaways from the source code listing above are: + +- The event data is accessible from the *payload* attribute of the event parameter. +- An implementation of the *DispatchEvent* RPC method must return an instance of *coprocess_object_pb2.EventReply*. + + +#### Create gRPC Server + +Finally, we will implement an AsyncIO gRPC server to handle requests from Tyk Gateway to the *Dispatcher* service. We will add functions to start and stop our gRPC server. Finally, we will use *grpcurl* to issue a test payload to our gRPC server to test that it is working. + + +##### Develop gRPC Server + +Append the following source code from the listing below to the *async_server.py* file: + +```python expandable +async def serve() -> None: + server = grpc.aio.server() + coprocess_object_pb2_grpc.add_DispatcherServicer_to_server( + PythonDispatcher(), server + ) + listen_addr = "[::]:50051" + SERVICE_NAMES = ( + coprocess_object_pb2.DESCRIPTOR.services_by_name["Dispatcher"].full_name, + reflection.SERVICE_NAME, + ) + + reflection.enable_server_reflection(SERVICE_NAMES, server) + server.add_insecure_port(listen_addr) + + logging.info ("Starting server on %s", listen_addr) + + await server.start() + await server.wait_for_termination() + +async def shutdown_server(server) -> None: + http://logging.info ("Shutting down server...") + await server.stop(None) +``` + +The *serve* function starts the gRPC server, listening for requests on port 50051 with reflection enabled. + +Clients can use reflection to list available services, obtain their RPC methods and retrieve their message types and field names dynamically. This is particularly useful for tooling and debugging purposes, allowing clients to discover server capabilities without prior knowledge of the service definitions. + + + +**note** + +A descriptor is a data structure that describes the structure of the messages, services, enums and other elements defined in a .proto file. The purpose of the descriptor is primarily metadata: it provides information about the types and services defined in the protocol buffer definition. The *coprocess_object_pb2.py* file that we generated using *protoc* contains a DESCRIPTOR field that we can use to retrieve this metadata. For further details consult the documentation for the Google's protobuf [FileDescriptor](https://googleapis.dev/python/protobuf/latest/google/protobuf/descriptor.html#google.protobuf.descriptor.FileDescriptor.services_by_name) class. + + + +The *shutdown_server* function stops the gRPC server via the *stop* method of the server instance. + +The key takeaways from the source code listing above are: + +- An instance of a gRPC server is created using *grpc.aio.server()*. +- A service implementation should be registered with the gRPC server. We register our *PythonDispatcher* class via *coprocess_object_pb2_grpc.add_DispatcherServicer_to_server(PythonDispatcher(), server)*. +- Reflection can be enabled to allow clients to dynamically discover the services available at a gRPC server. We enabled our *Dispatcher* service to be discovered via *reflection.enable_server_reflection(SERVICE_NAMES, server)*. SERVICE_NAMES is a tuple containing the full names of two gRPC services: the *Dispatcher* service obtained by using the DESCRIPTOR field within the *coprocess_object_pb2* module and the other being the standard reflection service. +- The server instance should be started via invoking and awaiting the *start* and *wait_for_termination* methods of the server instance. +- A port may be configured for the server. In this example we configured an insecure port of 50051 on the server instance via the [add_insecure_port function](https://grpc.github.io/grpc/python/grpc.html#grpc.Server.add_insecure_port). It is also possible to add a secure port via the [add_secure_port](https://grpc.github.io/grpc/python/grpc.html#grpc.Server.add_secure_port) method of the server instance, which accepts the port number in addition to an SSL certificate and key to enable TLS encryption. +- The server instance can be stopped via its stop method. + +Finally, we will allow our server to terminate upon receipt of SIGTERM and SIGINT signals. To achieve this, append the source code listed below to the *async_server.py* file. + +```python expandable +def handle_sigterm(sig, frame) -> None: + asyncio.create_task(shutdown_server(server)) + +async def handle_sigint() -> None: + loop = asyncio.get_running_loop() + for sig in (signal.SIGINT, signal.SIGTERM): + loop.add_signal_handler(sig, loop.stop) + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) + server = None + signal.signal(signal.SIGTERM, handle_sigterm) + try: + asyncio.get_event_loop().run_until_complete(serve()) + except KeyboardInterrupt: + pass +``` + + +##### Start gRPC Server + +Issue the following command to start the gRPC server: + +```bash +python3 -m async_server +``` + +A message should be output on the console, displaying the port number and confirming that the gRPC server has started. + + +##### Test gRPC Server + +To test our gRPC server is working, issue test requests to the *Dispatch* and *DispatchEvent* methods, using *grpcurl*. + + +####### Send Dispatch Request + +Use the *grpcurl* command to send a test dispatch request to our gRPC server: + +```bash expandable +grpcurl -plaintext -d '{ + "hookType": "Pre", + "hookName": "MyPreCustomPluginForBasicAuth", + "request": { + "headers": { + "User-Agent": "curl/8.1.2", + "Host": "tyk-gateway.localhost:8080", + "Authorization": "Basic ZGV2QHR5ay5pbzpwYXN0cnk=", + "Accept": "*/*" + }, + "url": "/basic-authentication-valid/get", + "returnOverrides": { + "responseCode": -1 + }, + "method": "GET", + "requestUri": "/basic-authentication-valid/get", + "scheme": "https" + }, + "spec": { + "bundle_hash": "d41d8cd98f00b204e9800998ecf8427e", + "OrgID": "5e9d9544a1dcd60001d0ed20", + "APIID": "04e911d3012646d97fcdd6c846fafc4b" + } +}' localhost:50051 coprocess.Dispatcher/Dispatch +``` + +Inspect the console output of your gRPC server. It should echo the payload that you sent in the request. + + +####### Send DispatchEvent Request + +Use the grpcurl command to send a test event payload to our gRPC server: + +```bash +grpcurl -plaintext -d '{"payload": "{\"event\": \"test\"}"}' localhost:50051 coprocess.Dispatcher/DispatchEvent +``` + +Inspect the console output of your gRPC server. It should display a log similar to that shown below: + +```bash +INFO:root:RECEIVED EVENT: {'event': 'test'} +``` + +The response received from the server should be an empty event reply, similar to that shown below: + +```bash +grpcurl -plaintext -d '{"payload": "{\"event\": \"test\"}"}' localhost:50051 coprocess.Dispatcher/DispatchEvent +{} +``` + +At this point we have tested, independently of Tyk Gateway, that our gRPC Server can handle an example request payload for gRPC plugin execution. In the next section we will create a test environment for testing that Tyk Gateway integrates with our gRPC server for API requests. + + +#### Configure Test Environment + +Now that we have implemented and started a gRPC server, Tyk Gateway needs to be configured to integrate with it. To achieve this we will enable the coprocess feature and configure the URL of the gRPC server. + +We will also create an API so that we can test that Tyk Gateway integrates with our gRPC server. + + +##### Configure Tyk Gateway + +Within the root of the *tyk.conf* file, add the following configuration, replacing host and port with values appropriate for your environment: + +```yaml +"coprocess_options": { + "enable_coprocess": true, + "coprocess_grpc_server": "tcp://host:port" +} +``` + +Alternatively, the following environment variables can be set in your .env file: + +```bash +TYK_GW_COPROCESSOPTIONS_ENABLECOPROCESS=true +TYK_GW_COPROCESSOPTIONS_COPROCESSGRPCSERVER=tcp://host:port +``` + +Replace host and port with values appropriate for your environment. + + +##### Configure API + +Before testing our gRPC server we will create and configure an API with 2 plugins: + +- **Pre Request**: Named *MyPreRequestPlugin*. +- **Response**: Named *MyResponsePlugin* and configured so that Tyk Gateway dispatches the session state with the request. + +Each plugin will be configured to use the *grpc* plugin driver. + +Tyk Gateway will forward details of an incoming request to the gRPC server, for each of the configured API plugins. + + +####### Tyk Classic API + +gRPC plugins can be configured within the *custom_middleware* section of the Tyk Classic ApiDefinition, as shown in the listing below: + +```yaml expandable +{ + "created_at": "2024-03-231T12:49:52Z", + "api_model": {}, + "api_definition": { + ... + ... + "custom_middleware": { + "pre": [ + { + "disabled": false, + "name": "MyPreRequestPlugin", + "path": "", + "require_session": false, + "raw_body_only": false + } + ], + "post": [], + "post_key_auth": [], + "auth_check": { + "disabled": false, + "name": "", + "path": "", + "require_session": false, + "raw_body_only": false + }, + "response": [ + { + "disabled": false, + "name": "MyResponsePlugin", + "path": "", + "require_session": true, + "raw_body_only": false + } + ], + "driver": "grpc", + "id_extractor": { + "disabled": false, + "extract_from": "", + "extract_with": "", + "extractor_config": {} + } + } +} +``` + +In the above listing, the plugin driver parameter has been configured with a value of *grpc*. Two plugins are configured within the *custom_middleware* section: a *Pre Request* plugin and a *Response* plugin. + +The *Response* plugin is configured with *require_session* enabled, so that Tyk Gateway will send details for the authenticated key / user with the gRPC request. Note, this is not configured for *Pre Request* plugins that are triggered before authentication in the request lifecycle. + + +####### Tyk OAS API + +To quickly get started, a Tyk OAS API schema can be created by importing the infamous [pet store](https://petstore3.swagger.io/api/v3/openapi.json) OAS schema. Then the [findByStatus](https://petstore3.swagger.io/api/v3/pet/findByStatus?status=available) endpoint can be used for testing. + +The resulting Tyk OAS API Definition contains the OAS JSON schema with an *x-tyk-api-gateway* section appended, as listed below. gRPC plugins can be configured within the middleware section of the *x-tyk-api-gateway* that is appended at the end of the OAS schema: + +```yaml expandable +"x-tyk-api-gateway": { + "info": { + "id": "6e2ae9b858734ea37eb772c666517f55", + "dbId": "65f457804773a600011af41d", + "orgId": "5e9d9544a1dcd60001d0ed20", + "name": "Swagger Petstore - OpenAPI 3.0 Custom Authentication", + "state": { + "active": true + } + }, + "upstream": { + "url": "https://petstore3.swagger.io/api/v3/" + }, + "server": { + "listenPath": { + "value": "/custom_auth", + "strip": true + }, + "authentication": { + "enabled": true, + "custom": { + "enabled": true, + "header": { + "enabled": false, + "name": "Authorization" + } + } + } + }, + "middleware": { + "global": { + "pluginConfig": { + "driver": "grpc" + } + }, + "cors": { + "enabled": false, + "maxAge": 24, + "allowedHeaders": [ + "Accept", + "Content-Type", + "Origin", + "X-Requested-With", + "Authorization" + ], + "allowedOrigins": [ + "*" + ], + "allowedMethods": [ + "GET", + "HEAD", + "POST" + ] + }, + "prePlugin": { + "api-management/plugins/overview#": [ + { + "enabled": true, + "functionName": "MyPreRequestPlugin", + "path": "" + } + ] + }, + "responsePlugin": { + "api-management/plugins/overview#": [ + { + "enabled": true, + "functionName": "MyResponsePlugin", + "path": "", + "requireSession": true + } + ] + } + } +} +``` + +In the above listing, the plugin driver parameter has been set to *grpc*. Two plugins are configured within the middleware section: a *Pre Request* plugin and a *Response* plugin. + +The *Response* plugin is configured with *requireSession* enabled, so that Tyk Gateway will send details for the authenticated key / user with the gRPC request. Note, this is not configurable for *Pre Request* plugins that are triggered before authentication in the request lifecycle. + +Tyk Gateway will forward details of an incoming request to the gRPC server, for each plugin. + + +#### Test API + +We have implemented and configured a gRPC server to integrate with Tyk Gateway. Furthermore, we have created an API that has been configured with two gRPC plugins: a *Pre Request* and *Response* plugin. + +When we issue a request to our API and observe the console output of our gRPC server we should see a JSON representation of the request headers etc. echoed in the terminal. + +Issue a request for your API in the terminal window. For example: + +```bash +curl -L http://.localhost:8080/grpc-http-bin +``` + +Observe the console output of your gRPC server. Tyk Gateway should have dispatched two requests to your gRPC server; a request for the *Pre Request* plugin and a request for the *Response* plugin. + +The gRPC server we implemented echoes a JSON representation of the request payload dispatched by Tyk Gateway. + +Note that this is a useful feature for learning how to develop gRPC plugins and understanding the structure of the request payload dispatched by Tyk Gateway to the gRPC server. However, in production environments care should be taken to avoid inadvertently exposing sensitive data such as secrets in the session. + + +#### Summary + +In this guide, we've delved into the integration of a Python gRPC server with Tyk Gateway. + +We have explained how to implement a Python gRPC server and equipped developers with the necessary tools, knowledge and capabilities to effectively utilize Tyk Gateway through gRPC services. + +The following essential groundwork has been covered: + +- Setting up tools, libraries and service definitions for the integration. +- Developing a basic gRPC server with functionality to echo the request payload, received from Tyk Gateway, in JSON format. +- Configuring Tyk Gateway for seamless communication with our gRPC server. + + +### Create a Request Transformation Plugin with Java + +This tutorial will guide you through the creation of a gRPC-based Java plugin for Tyk. +Our plugin will inject a header into the request before it gets proxied upstream. For additional information about gRPC, check the official documentation [here](https://grpc.io/docs/guides/index.html). + +The sample code that we'll use implements a request transformation plugin using Java and uses the proper gRPC bindings generated from our Protocol Buffers definition files. + +#### Requirements + +- Tyk Gateway: This can be installed using standard package management tools like Yum or APT, or from source code. See [here][1] for more installation options. +- The Tyk CLI utility, which is bundled with our RPM and DEB packages, and can be installed separately from [https://github.com/TykTechnologies/tyk-cli][2]. +- In Tyk 2.8 the Tyk CLI is part of the gateway binary, you can find more information by running "tyk help bundle". +- Gradle Build Tool: https://gradle.org/install/. +- gRPC tools: https://grpc.io/docs/quickstart/csharp.html#generate-grpc-code +- Java JDK 7 or higher. + +#### Create the Plugin + +##### Setting up the Java Project + +We will use the Gradle build tool to generate the initial files for our project: + +```bash +cd ~ +mkdir tyk-plugin +cd tyk-plugin +gradle init +``` + +We now have a `tyk-plugin` directory containing the basic skeleton of our application. + +Add the following to `build.gradle` + +```{.copyWrapper} expandable +buildscript { + repositories { + jcenter() + } + dependencies { + classpath 'com.google.protobuf:protobuf-gradle-plugin:0.8.1' + } +} + +plugins { + id "com.google.protobuf" version "0.8.1" + id "java" + id "application" + id "idea" +} + +protobuf { + protoc { + artifact = "com.google.protobuf:protoc:3.3.0" + } + plugins { + grpc { + artifact = 'io.grpc:protoc-gen-grpc-java:1.5.0' + } + } + generateProtoTasks { + all()*.plugins { + grpc {} + } + } + generatedFilesBaseDir = "$projectDir/src/generated" +} + +sourceCompatibility = 1.8 +targetCompatibility = 1.8 + +mainClassName = "com.testorg.testplugin.PluginServer" + +repositories { + mavenCentral() +} + +dependencies { + compile 'io.grpc:grpc-all:1.5.0' +} + +idea { + module { + sourceDirs += file("${projectDir}/src/generated/main/java"); + sourceDirs += file("${projectDir}/src/generated/main/grpc"); + } +} +``` + +##### Create the Directory for the Server Class + +```bash +cd ~/tyk-plugin +mkdir -p src/main/java/com/testorg/testplugin +``` + +##### Install the gRPC Tools + +We need to download the Tyk Protocol Buffers definition files, these files contains the data structures used by Tyk. See [Data Structures](/api-management/plugins/rich-plugins#rich-plugins-data-structures) for more information: + +```bash +cd ~/tyk-plugin +git clone https://github.com/TykTechnologies/tyk +mv tyk/coprocess/proto src/main/proto +``` + +##### Generate the Bindings + +To generate the Protocol Buffers bindings we use the Gradle build task: + +```bash +gradle build +``` + +If you need to customize any setting related to the bindings generation step, check the `build.gradle` file. + +##### Implement Server + +We need to implement two classes: one class will contain the request dispatcher logic and the actual middleware implementation. The other one will implement the gRPC server using our own dispatcher. + +From the `~/tyk-plugin/src/main/java/com/testorg/testplugin` directory, create a file named `PluginDispatcher.java` with the following code: + +```java expandable +package com.testorg.testplugin; + +import coprocess.DispatcherGrpc; +import coprocess.CoprocessObject; + +public class PluginDispatcher extends DispatcherGrpc.DispatcherImplBase { + + @Override + public void dispatch(CoprocessObject.Object request, + io.grpc.stub.StreamObserver responseObserver) { + CoprocessObject.Object modifiedRequest = null; + + switch (request.getHookName()) { + case "MyPreMiddleware": + modifiedRequest = MyPreHook(request); + default: + // Do nothing, the hook name isn't implemented! + } + + // Return the modified request (if the transformation was done): + if (modifiedRequest != null) { + responseObserver.onNext(modifiedRequest); + }; + + responseObserver.onCompleted(); + } + + CoprocessObject.Object MyPreHook(CoprocessObject.Object request) { + CoprocessObject.Object.Builder builder = request.toBuilder(); + builder.getRequestBuilder().putSetHeaders("customheader", "customvalue"); + return builder.build(); + } +} +``` + +In the same directory, create a file named `PluginServer.java` with the following code. This is the server implementation: + +```java expandable +package com.testorg.testplugin; + +import coprocess.DispatcherGrpc; + +import io.grpc.Server; +import io.grpc.ServerBuilder; +import io.grpc.stub.StreamObserver; +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class PluginServer { + + private static final Logger logger = Logger.getLogger(PluginServer.class.getName()); + static Server server; + static int port = 5555; + + public static void main(String[] args) throws IOException, InterruptedException { + System.out.println("Initializing gRPC server."); + + // Our dispatcher is instantiated and attached to the server: + server = ServerBuilder.forPort(port) + .addService(new PluginDispatcher()) + .build() + .start(); + + blockUntilShutdown(); + + } + + static void blockUntilShutdown() throws InterruptedException { + if (server != null) { + server.awaitTermination(); + } + } +} +``` + +To run the gRPC server we can use the following command: + +```bash +cd ~/tyk-plugin +gradle runServer +``` + +The gRPC server will listen on port 5555 (as defined in `Server.java`). In the next steps we'll setup the plugin bundle and modify Tyk to connect to our gRPC server. + + +#### Bundle the Plugin + +We need to create a manifest file within the `tyk-plugin` directory. This file contains information about our plugin and how we expect it to interact with the API that will load it. This file should be named `manifest.json` and needs to contain the following: + +```json expandable +{ + "custom_middleware": { + "driver": "grpc", + "pre": [{ + "name": "MyPreMiddleware" + }] + } +} +``` + +- The `custom_middleware` block contains the middleware settings like the plugin driver we want to use (`driver`) and the hooks that our plugin will expose. We use the `pre` hook for this tutorial. For other hooks see [here](/api-management/plugins/rich-plugins#coprocess-dispatcher---hooks). +- The `name` field references the name of the function that we implemented in our plugin code - `MyPreMiddleware`. This will be handled by our dispatcher gRPC method in `PluginServer.java`. + +To bundle our plugin run the following command in the `tyk-plugin` directory. Check your tyk-cli install path first: + +```bash +/opt/tyk-gateway/utils/tyk-cli bundle build -y +``` + +For Tyk 2.8 use: +```bash +/opt/tyk-gateway/bin/tyk bundle build -y +``` + +A plugin bundle is a packaged version of the plugin. It may also contain a cryptographic signature of its contents. The `-y` flag tells the Tyk CLI tool to skip the signing process in order to simplify the flow of this tutorial. + +For more information on the Tyk CLI tool, see [here](/api-management/plugins/overview#plugin-bundles). + +You should now have a `bundle.zip` file in the `tyk-plugin` directory. + +#### Publish the Plugin + +To publish the plugin, copy or upload `bundle.zip` to a local web server like Nginx, or Apache or storage like Amazon S3. For this tutorial we'll assume you have a web server listening on `localhost` and accessible through `http://localhost`. + + + +#### What's Next? {what-is-next} + +In this tutorial we learned how Tyk gRPC plugins work. For a production-level setup we suggest the following: + +- Configure an appropriate web server and path to serve your plugin bundles. + +[1]: /tyk-self-managed/install +[2]: https://github.com/TykTechnologies/tyk-cli +[3]: /img/dashboard/system-management/api_settings.png +[4]: /img/dashboard/system-management/plugin_options.png + +

Create Custom Authentication Plugin with .NET

+This tutorial will guide you through the creation of a custom authentication plugin for Tyk with a gRPC based plugin with .NET and C#. For additional information check the official gRPC [documentation](https://grpc.io/docs/guides/index.html). + +The sample code that we’ll use implements a very simple authentication layer using .NET and the proper gRPC bindings generated from our Protocol Buffers definition files. + +Using gRPC for plugins + +#### Requirements + +- Tyk Gateway: This can be installed using standard package management tools like Yum or APT, or from source code. See [here][1] for more installation options. +- The Tyk CLI utility, which is bundled with our RPM and DEB packages, and can be installed separately from [https://github.com/TykTechnologies/tyk-cli][2] +- In Tyk 2.8 the Tyk CLI is part of the gateway binary, you can find more information by running "tyk help bundle". +- .NET Core for your OS: https://www.microsoft.com/net/core +- gRPC tools: https://grpc.io/docs/quickstart/csharp.html#generate-grpc-code + +#### Create the Plugin + +
Create .NET Project
+We use the .NET CLI tool to generate the initial files for our project: + +```bash +cd ~ +dotnet new console -o tyk-plugin +``` + +We now have a `tyk-plugin` directory containing the basic skeleton of a .NET application. + +From the `tyk-plugin` directory we need to install a few packages that the gRPC server requires: + +```bash +dotnet add package Grpc --version 1.6.0 +dotnet add package System.Threading.ThreadPool --version 4.3.0 +dotnet add package Google.Protobuf --version 3.4.0 +``` + +- The `Grpc` package provides base code for our server implementation. +- The `ThreadPool` package is used by `Grpc`. +- The `Protobuf` package will be used by our gRPC bindings. + +
Install the gRPC Tools
+We need to install the gRPC tools to generate the bindings. We recommended you follow the official guide here: https://grpc.io/docs/quickstart/csharp.html#generate-grpc-code. + +Run the following Commands (both MacOS and Linux): + +```bash +cd ~/tyk-plugin +temp_dir=packages/Grpc.Tools.1.6.x/tmp +curl_url=https://www.nuget.org/api/v2/package/Grpc.Tools/ +mkdir -p $temp_dir && cd $temp_dir && curl -sL $curl_url > tmp.zip; unzip tmp.zip && cd .. && cp -r tmp/tools . && rm -rf tmp && cd ../.. +chmod -Rf +x packages/Grpc.Tools.1.6.x/tools/ +``` + +Then run the following, depending on your OS: + +**MacOS (x64)** + +```bash +export GRPC_TOOLS=packages/Grpc.Tools.1.6.x/tools/macosx_x64 +``` + +**Linux (x64)** + +```bash +export GRPC_TOOLS=packages/Grpc.Tools.1.6.x/tools/linux_x64 +``` + +The `GRPC_TOOLS` environment variable will point to the appropriate GrpcTools path that matches our operating system and architecture. The last step is to export a variable for the `protoc` program; this is the main program used to generate bindings: + +```bash +export GRPC_PROTOC=$GRPC_TOOLS/protoc +``` + +Now that we can safely run `protoc`, we can download the Tyk Protocol Buffers definition files. These files contain the data structures used by Tyk. See [Data Structures](/api-management/plugins/rich-plugins#rich-plugins-data-structures) for more information: + +```bash +cd ~/tyk-plugin +git clone https://github.com/TykTechnologies/tyk +``` + +##### Generate the bindings + +To generate the bindings, we create an empty directory and run the `protoc` tool using the environment variable that was set before: + +```bash +mkdir Coprocess +$GRPC_PROTOC -I=tyk/coprocess/proto --csharp_out=Coprocess --grpc_out=Coprocess --plugin=protoc-gen-grpc=$GRPC_TOOLS/grpc_csharp_plugin tyk/coprocess/proto/*.proto +``` + +Run the following command to check the binding directory: + +```bash +ls Coprocess +``` + +The output will look like this: + +``` +CoprocessCommon.cs CoprocessObject.cs CoprocessReturnOverrides.cs +CoprocessMiniRequestObject.cs CoprocessObjectGrpc.cs CoprocessSessionState.cs +``` + +##### Implement Server + +Create a file called `Server.cs`. + +Add the following code to `Server.cs`. + +```c# expandable +using System; +using System.Threading.Tasks; +using Grpc.Core; + +using Coprocess; + +class DispatcherImpl : Dispatcher.DispatcherBase +{ + public DispatcherImpl() + { + Console.WriteLine("Instantiating DispatcherImpl"); + } + + + // The Dispatch method will be called by Tyk for every configured hook, we'll implement a very simple dispatcher here: + public override Task Dispatch(Coprocess.Object thisObject, ServerCallContext context) + { + // thisObject is the request object: + Console.WriteLine("Receiving object: " + thisObject.ToString()); + + // hook contains the hook name, this will be defined in our plugin bundle and the implementation will be a method in this class (DispatcherImpl), we'll look it up: + var hook = this.GetType().GetMethod(thisObject.HookName); + + // If hook is null then a handler method for this hook isn't implemented, we'll log this anyway: + if (hook == null) + { + Console.WriteLine("Hook name: " + thisObject.HookName + " (not implemented!)"); + // We return the unmodified request object, so that Tyk can proxy this in the normal way. + return Task.FromResult(thisObject); + }; + + // If there's a handler method, let's log it and proceed with our dispatch work: + Console.WriteLine("Hook name: " + thisObject.HookName + " (implemented)"); + + // This will dynamically invoke our hook method, and cast the returned object to the required Protocol Buffers data structure: + var output = hook.Invoke(this, new object[] { thisObject, context }); + return (Task)output; + } + + // MyPreMiddleware implements a PRE hook, it will be called before the request is proxied upstream and before the authentication step: + public Task MyPreMiddleware(Coprocess.Object thisObject, ServerCallContext context) + { + Console.WriteLine("Calling MyPreMiddleware."); + // We'll inject a header in this request: + thisObject.Request.SetHeaders["my-header"] = "my-value"; + return Task.FromResult(thisObject); + } + + // MyAuthCheck implements a custom authentication mechanism, it will initialize a session object if the token matches a certain value: + public Task MyAuthCheck(Coprocess.Object thisObject, ServerCallContext context) + { + // Request.Headers contains all the request headers, we retrieve the authorization token: + var token = thisObject.Request.Headers["Authorization"]; + Console.WriteLine("Calling MyAuthCheck with token = " + token); + + // We initialize a session object if the token matches "abc123": + if (token == "abc123") + { + Console.WriteLine("Successful auth!"); + var session = new Coprocess.SessionState(); + session.Rate = 1000; + session.Per = 10; + session.QuotaMax = 60; + session.QuotaRenews = 1479033599; + session.QuotaRemaining = 0; + session.QuotaRenewalRate = 120; + session.Expires = 1479033599; + + session.LastUpdated = 1478033599.ToString(); + + thisObject.Metadata["token"] = token; + thisObject.Session = session; + return Task.FromResult(thisObject); + + } + + // If the token isn't "abc123", we return the request object in the original state, without a session object, Tyk will reject this request: + Console.WriteLine("Rejecting auth!"); + return Task.FromResult(thisObject); + } +} +``` + +Create a file called `Program.cs` to instantiate our dispatcher implementation and start a gRPC server. + +Add the following code to `Program.cs`. + +```bash expandable +using System; +using Grpc.Core; + +namespace tyk_plugin +{ + class Program + { + + // Port to attach the gRPC server to: + const int Port = 5555; + + static void Main(string[] args) + { + // We initialize a Grpc.Core.Server and attach our dispatcher implementation to it: + Server server = new Server + { + Services = { Coprocess.Dispatcher.BindService(new DispatcherImpl()) }, + Ports = { new ServerPort("localhost", Port, ServerCredentials.Insecure) } + }; + server.Start(); + + Console.WriteLine("gRPC server listening on " + Port); + Console.WriteLine("Press any key to stop the server..."); + Console.ReadKey(); + + server.ShutdownAsync().Wait(); + + } + } +} +``` + +To run the gRPC server use the following command from the plugin directory: + +```bash +dotnet run +``` + +The gRPC server will listen on port 5555 (as defined in `Program.cs`). In the next steps we'll setup the plugin bundle and modify Tyk to connect to our gRPC server. + +#### Bundle the Plugin + +We need to create a manifest file within the `tyk-plugin` directory. This file contains information about our plugin and how we expect it to interact with the API that will load it. This file should be named `manifest.json` and needs to contain the following: + +```json expandable +{ + "custom_middleware": { + "driver": "grpc", + "auth_check": { + "name": "MyAuthMiddleware", + "path": "", + "raw_body_only": false, + "require_session": false + } + } +} +``` + +- The `custom_middleware` block contains the middleware settings like the plugin driver we want to use (`driver`) and the hooks that our plugin will expose. We use the `auth_check` hook for this tutorial. For other hooks see [here](/api-management/plugins/rich-plugins#coprocess-dispatcher---hooks). +- The `name` field references the name of the function that we implement in our plugin code - `MyAuthMiddleware`. This will be handled by our dispatcher gRPC method (implemented in `Server.cs`). +- The `path` field is the path to the middleware component. +- The `raw_body_only` field +- The `require_session` field, if set to `true` gives you access to the session object. It will be supplied as a session variable to your middleware processor function + + +To bundle our plugin run the following command in the `tyk-plugin` directory. Check your tyk-cli install path first: + +```bash +/opt/tyk-gateway/utils/tyk-cli bundle build -y +``` + +From Tyk v2.8 upwards you can use: +```bash +/opt/tyk-gateway/bin/tyk bundle build -y +``` + +A plugin bundle is a packaged version of the plugin. It may also contain a cryptographic signature of its contents. The `-y` flag tells the Tyk CLI tool to skip the signing process in order to simplify the flow of this tutorial. + +For more information on the Tyk CLI tool, see [here](/api-management/plugins/overview#plugin-bundles). + +You should now have a `bundle.zip` file in the `tyk-plugin` directory. + +#### Publish the Plugin + +To publish the plugin, copy or upload `bundle.zip` to a local web server like Nginx, or Apache or storage like Amazon S3. For this tutorial we'll assume you have a web server listening on `localhost` and accessible through `http://localhost`. + + + +#### What's Next? + +In this tutorial we learned how Tyk gRPC plugins work. For a production-level setup we suggest the following: + +- Configure an appropriate web server and path to serve your plugin bundles. +- See the following [GitHub repo](https://github.com/TykTechnologies/tyk-plugin-demo-dotnet) for a gRPC based .NET plugin that incorporates authentication based on Microsoft SQL Server. + +[1]: /tyk-self-managed/install +[2]: https://github.com/TykTechnologies/tyk-cli +[3]: /img/dashboard/system-management/plugin_options.png +[4]: /img/dashboard/system-management/plugin_auth_mode.png + +### Create Custom Authentication Plugin with NodeJS + +This tutorial will guide you through the creation of a custom authentication plugin for Tyk with a gRPC based plugin written in NodeJS. For additional information about gRPC, check the official documentation [here](https://grpc.io/docs/guides/index.html). + +The sample code that we'll use implements a very simple authentication layer using NodeJS and the proper gRPC bindings generated from our Protocol Buffers definition files. + +gRPC Auth Diagram + +#### Requirements + +- Tyk Gateway: This can be installed using standard package management tools like Yum or APT, or from source code. See [here](/tyk-self-managed/install) for more installation options. +- The Tyk CLI utility, which is bundled with our RPM and DEB packages, and can be installed separately from [https://github.com/TykTechnologies/tyk-cli](https://github.com/TykTechnologies/tyk-cli) +- In Tyk 2.8 and upwards the Tyk CLI is part of the gateway binary, you can find more information by running "tyk help bundle". +- NodeJS v6.x.x [https://nodejs.org/en/download/](https://nodejs.org/en/download/) + +#### Create the Plugin + +##### Create NodeJS Project + +We will use the NPM tool to initialize our project, follow the steps provided by the `init` command: + +```bash +cd ~ +mkdir tyk-plugin +cd tyk-plugin +npm init +``` + +Now we'll add the gRPC package for this project: + +```bash +npm install --save grpc +``` + +##### Install gRPC Tools + +Typically to use gRPC and Protocol Buffers you need to use a code generator and generate bindings for the target language that you're using. For this tutorial we'll skip this step and use the dynamic loader that's provided by the NodeJS gRPC library. This mechanism allows a program to load Protocol Buffers definitions directly from `.proto` files. See [this section](https://grpc.io/docs/tutorials/basic/node.html#loading-service-descriptors-from-proto-files) in the gRPC documentation for more details. + +To fetch the required `.proto` files, you may use an official repository where we keep the Tyk Protocol Buffers definition files: + +```bash +cd ~/tyk-plugin +git clone https://github.com/TykTechnologies/tyk +``` + +##### Implement Server + +Now we're ready to implement our gRPC server, create a file called `main.js` in the project's directory + +Add the following code to `main.js`. + +```nodejs expandable +const grpc = require('grpc'), + resolve = require('path').resolve + +const tyk = grpc.load({ + file: 'coprocess_object.proto', + root: resolve(__dirname, 'tyk/coprocess/proto') +}).coprocess + +const listenAddr = '127.0.0.1:5555', + authHeader = 'Authorization' + validToken = '71f6ac3385ce284152a64208521c592b' + +// The dispatch function is called for every hook: +const dispatch = (call, callback) => { + var obj = call.request + // We dispatch the request based on the hook name, we pass obj.request which is the coprocess.Object: + switch (obj.hook_name) { + case 'MyPreMiddleware': + preMiddleware(obj, callback) + break + case 'MyAuthMiddleware': + authMiddleware(obj, callback) + break + default: + callback(null, obj) + break + } +} + +const preMiddleware = (obj, callback) => { + var req = obj.request + + // req is the coprocess.MiniRequestObject, we inject a header using the "set_headers" field: + req.set_headers = { + 'mycustomheader': 'mycustomvalue' + } + + // Use this callback to finish the operation, sending back the modified object: + callback(null, obj) +} + +const authMiddleware = (obj, callback) => { + var req = obj.request + + // We take the value from the "Authorization" header: + var token = req.headers[authHeader] + + // The token should be attached to the object metadata, this is used internally for key management: + obj.metadata = { + token: token + } + + // If the request token doesn't match the "validToken" constant we return the call: + if (token != validToken) { + callback(null, obj) + return + } + + // At this point the token is valid and a session state object is initialized and attached to the coprocess.Object: + var session = new tyk.SessionState() + session.id_extractor_deadline = Date.now() + 100000000000 + obj.session = session + callback(null, obj) +} + +main = function() { + server = new grpc.Server() + server.addService(tyk.Dispatcher.service, { + dispatch: dispatch + }) + server.bind(listenAddr, grpc.ServerCredentials.createInsecure()) + server.start() +} + +main() +``` + + +To run the gRPC server run: + +```bash +node main.js +``` + +The gRPC server will listen on port `5555` (see the `listenAddr` constant). In the next steps we'll setup the plugin bundle and modify Tyk to connect to our gRPC server. + + +#### Bundle the Plugin + +We need to create a manifest file within the `tyk-plugin` directory. This file contains information about our plugin and how we expect it to interact with the API that will load it. This file should be named `manifest.json` and needs to contain the following: + +```json expandable +{ + "custom_middleware": { + "driver": "grpc", + "auth_check": { + "name": "MyAuthMiddleware", + "path": "", + "raw_body_only": false, + "require_session": false + } + } +} +``` + +- The `custom_middleware` block contains the middleware settings like the plugin driver we want to use (`driver`) and the hooks that our plugin will expose. We use the `auth_check` hook for this tutorial. For other hooks see [here](/api-management/plugins/rich-plugins#coprocess-dispatcher---hooks). +- The `name` field references the name of the function that we implement in our plugin code - `MyAuthMiddleware`. The implemented dispatcher uses a switch statement to handle this hook, and calls the `authMiddleware` function in `main.js`. +- The `path` field is the path to the middleware component. +- The `raw_body_only` field +- The `require_session` field, if set to `true` gives you access to the session object. It will be supplied as a session variable to your middleware processor function + +To bundle our plugin run the following command in the `tyk-plugin` directory. Check your tyk-cli install path first: + +```bash +/opt/tyk-gateway/utils/tyk-cli bundle build -y +``` + +For Tyk 2.8 use: +```bash +/opt/tyk-gateway/bin/tyk bundle build -y +``` + +A plugin bundle is a packaged version of the plugin. It may also contain a cryptographic signature of its contents. The `-y` flag tells the Tyk CLI tool to skip the signing process in order to simplify the flow of this tutorial. + +For more information on the Tyk CLI tool, see [here](/api-management/plugins/overview#plugin-bundles). + +You should now have a `bundle.zip` file in the `tyk-plugin` directory. + +#### Publish the Plugin + +To publish the plugin, copy or upload `bundle.zip` to a local web server like Nginx, Apache or storage like Amazon S3. For this tutorial we'll assume you have a web server listening on `localhost` and accessible through `http://localhost`. + + + + +#### What's Next? + +In this tutorial we learned how Tyk gRPC plugins work. For a production-level setup we suggest the following: + +- Configure an appropriate web server and path to serve your plugin bundles. + +[1]: /tyk-self-managed/install +[2]: https://github.com/TykTechnologies/tyk-cli +[3]: /img/dashboard/system-management/plugin_options.png +[4]: /img/dashboard/system-management/plugin_auth_mode.png + +### Create Custom Authentication Plugin With Python + +In the realm of API security, HMAC-signed authentication serves as a foundational concept. In this developer-focused blog post, we'll use HMAC-signed authentication as the basis for learning how to write gRPC custom authentication plugins with Tyk Gateway. Why learn how to write Custom Authentication Plugins? + +- **Foundational knowledge**: Writing custom authentication plugins provides foundational knowledge of Tyk's extensibility and customization capabilities. +- **Practical experience**: Gain hands-on experience in implementing custom authentication logic tailored to specific use cases, starting with HMAC-signed authentication. +- **Enhanced control**: Exercise greater control over authentication flows and response handling, empowering developers to implement advanced authentication mechanisms beyond built-in features. + +While Tyk Gateway offers built-in support for HMAC-signed authentication, this tutorial serves as a practical guide for developers looking to extend Tyk's capabilities through custom authentication plugins. It extends the gRPC server that we developed in our [getting started guide](/api-management/plugins/rich-plugins#using-python). + +We will develop a basic gRPC server that implements the Tyk Dispatcher service with a custom authentication plugin to handle authentication keys, signed using the HMAC SHA512 algorithm. Subsequently, you will be able to make a request to your API with a HMAC signed authentication key in the *Authorization* header. Tyk Gateway will intercept the request and forward it to your Python gRPC server for HMAC signature and token verification. + +Our plugin will only verify the key against an expected value. In a production environment it will be necessary to verify the key against Redis storage. + +Before we continue ensure that you have: + +- Read and completed our getting started guide that explains how to implement a basic Python gRPC server to echo the request payload received from Tyk Gateway. This tutorial extends the source code of the tyk_async_server.py file to implement a custom authentication plugin for a HMAC signed authentication key. +- Read our HMAC signatures documentation for an explanation of HMAC signed authentication with Tyk Gateway. A brief summary is given in the HMAC Signed Authentication section below. + + +#### HMAC Signed Authentication + +Before diving in further, we will give a brief overview of HMAC signed authentication using our custom authentication plugin. + +- **Client request**: The journey begins with a client requesting access to a protected resource on the Tyk API. +- **HMAC signing**: Before dispatching the request, the client computes an HMAC signature using a secret key and request date, ensuring the payload's integrity. +- **Authorization header**: The HMAC signature, along with essential metadata such as the API key and HMAC algorithm, is embedded within the Authorization header. +- **Tyk Gateway verification**: Upon receipt, Tyk Gateway forwards the request to our gRPC server to execute the custom authentication plugin. This will validate the HMAC signature, ensuring the request's authenticity before proceeding with further processing. + +Requests should be made to an API that uses our custom authentication plugin as follows. A HMAC signed key should be included in the *Authorization* header and a date/time string in the *Date* header. An example request is shown in the curl command below: + +```bash +curl -v -H 'Date: Fri, 03 May 2024 12:00:42 GMT' \ +-H 'Authorization: Signature keyId="eyJvcmciOiI1ZTlkOTU0NGExZGNkNjAwMDFkMGVkMjAiLCJpZCI6ImdycGNfaG1hY19rZXkiLCJoIjoibXVybXVyNjQifQ==", \ +algorithm="hmac-sha512",signature="9kwBK%2FyrjbSHJDI7INAhBmhHLTHRDkIe2uRWHEP8bgQFQvfXRksm6t2MHeLUyk9oosWDZyC17AbGeP8EFqrp%2BA%3D%3D"' \ +http://localhost:8080/grpc-custom-auth/get +``` + +From the above example, it should be noted that: + +- The *Date* header contains a date string formatted as follows: *Fri, 03 May 2024 11:06:00 GMT*. +- The *Authorization* header is formatted as `Signature keyId="", algorithm="", signature=""` where: + + - **keyId** is a Tyk authentication key. + - **algorithm** is the HMAC algorithm used to sign the signature, *hmac-sha512* or *hmac-sha256*. + - **signature** is the HAMC signature calculated with the date string from the *Date* header, signed with a base64 encoded secret value, using the specified HMAC algorithm. The HMAC signature is then encoded as base64. + +#### Prerequisites + +Firstly, we need to create the following: + +- An API configured to use a custom authentication plugin. +- A HMAC enabled key with a configured secret for signing. + +This will enable us to issue a request to test that Tyk Gateway integrates with our custom authentication plugin on the gRPC server. + +###### Create API + +We will create an API served by Tyk Gateway, that will forward requests upstream to https://httpbin.org/. + +The API will have the following parameters configured: + +- **Listen path**: Tyk Gateway will listen to API requests on */grpc-custom-auth/* and will strip the listen path for upstream requests. +- **Target URL**: The target URL will be configured to send requests to *http://httpbin/*. +- **Authentication Mode**: The authentication mode will be configured for custom authentication. This is used to trigger CoProcess (gRPC), Python or JSVM plugins to handle custom authentication. + +You can use the following Tyk Classic API definition to get you started, replacing the *org_id* with the ID of your organization. + +```json expandable +{ + "api_definition": { + "id": "662facb2f03e750001a03500", + "api_id": "6c56dd4d3ad942a94474df6097df67ed", + "org_id": "5e9d9544a1dcd60001d0ed20", + "name": "Python gRPC Custom Auth", + "enable_coprocess_auth": true, + "auth": { + "auth_header_name": "Authorization" + }, + "proxy": { + "preserve_host_header": false, + "listen_path": "/grpc-custom-auth/", + "disable_strip_slash": true, + "strip_listen_path": true, + "target_url": "http://httpbin/" + }, + "version_data": { + "not_versioned": false, + "versions": { + "Default": { + "name": "Default", + "expires": "", + "use_extended_paths": true, + "extended_paths": { + "ignored": [], + "white_list": [], + "black_list": [] + } + } + }, + "default_version": "Default" + }, + "active": true + } +} +``` + +The Tyk API definition above can be imported via Tyk Dashboard. Alternatively, if using Tyk Gateway OSS, a POST request can be made to the *api/apis* endpoint of Tyk Gateway. Consult the [Tyk Gateway Open API Specification documentation](/tyk-gateway-api) for usage. + +An illustrative example using *curl* is given below. Please note that you will need to: + +- Update the location to use the protocol scheme, host and port suitable for your environment. +- Replace the value in the *x-tyk-authorization* header with the secret value in your *tyk.conf* file. +- Replace the *org_id* with the ID of your organization. + +```bash expandable +curl -v \ + --header 'Content-Type: application/json' \ + --header 'x-tyk-authorization: your Gateway admin secret' \ + --location http://localhost:8080/tyk/apis/ \ + --data '{\ + "api_definition": {\ + "id": "662facb2f03e750001a03502",\ + "api_id": "6c56dd4d3ad942a94474df6097df67ef",\ + "org_id": "5e9d9544a1dcd60001d0ed20",\ + "name": "Python gRPC Custom Auth",\ + "enable_coprocess_auth": true,\ + "auth": {\ + "auth_header_name": "Authorization"\ + },\ + "proxy": {\ + "preserve_host_header": false,\ + "listen_path": "/grpc-custom-auth-error/",\ + "disable_strip_slash": true,\ + "strip_listen_path": true,\ + "target_url": "http://httpbin/"\ + },\ + "version_data": {\ + "not_versioned": false,\ + "versions": {\ + "Default": {\ + "name": "Default",\ + "expires": "",\ + "use_extended_paths": true,\ + "extended_paths": {\ + "ignored": [],\ + "white_list": [],\ + "black_list": []\ + }\ + }\ + },\ + "default_version": "Default"\ + },\ + "active": true\ + }\ + }' +``` + +A response similar to that given below will be returned by Tyk Gateway: + +```bash +{ + "key": "f97b748fde734b099001ca15f0346dfe", + "status": "ok", + "action": "added" +} +``` + +###### Create HMAC Key + +We will create an key configured to use HMAC signing, with a secret of *secret*. The key will configured to have access to our test API. + +You can use the following configuration below, replacing the value of the *org_id* with the ID of your organization. + +```bash expandable +{ + "quota_max": 1000, + "quota_renews": 1596929526, + "quota_remaining": 1000, + "quota_reset": 1596843126, + "quota_used": 0, + "org_id": "5e9d9544a1dcd60001d0ed20", + "access_rights": { + "662facb2f03e750001a03500": { + "api_id": "662facb2f03e750001a03500", + "api_name": "Python gRPC Custom Auth", + "versions": ["Default"], + "allowed_urls": [], + "limit": null, + "quota_max": 1000, + "quota_renews": 1596929526, + "quota_remaining": 1000, + "quota_reset": 1596843126, + "quota_used": 0, + "per": 1, + "expires": -1 + } + }, + "enable_detailed_recording": true, + "hmac_enabled": true, + "hmac_string": "secret", + "meta_data": {} +} +``` + +You can use Tyk Gateway’s API to create the key by issuing a POST request to the *tyk/keys* endpoint. Consult the [Tyk Gateway Open API Specification documentation](/tyk-gateway-api) for usage. + +An illustrative example using *curl* is given below. Please note that you will need to: + +- Update the location to use the protocol scheme, host and port suitable for your environment. +- Replace the value in the *x-tyk-authorization* header with the secret value in your *tyk.conf* file. + +Replace the *org_id* with the ID of your organization. + +```bash expandable +curl --location 'http://localhost:8080/tyk/keys/grpc_hmac_key' \ +--header 'x-tyk-authorization: your Gateay admin secret' \ +--header 'Content-Type: application/json' \ +--data '{\ + "alias": "grpc_hmac_key",\ + "quota_max": 1000,\ + "quota_renews": 1596929526,\ + "quota_remaining": 1000,\ + "quota_reset": 1596843126,\ + "quota_used": 0,\ + "org_id": "5e9d9544a1dcd60001d0ed20",\ + "access_rights": {\ + "662facb2f03e750001a03500": {\ + "api_id": "662facb2f03e750001a03500",\ + "api_name": "python-grpc-custom-auth",\ + "versions": ["Default"],\ + "allowed_urls": [],\ + "limit": null,\ + "quota_max": 1000,\ + "quota_renews": 1596929526,\ + "quota_remaining": 1000,\ + "quota_reset": 1596843126,\ + "quota_used": 0,\ + "per": 1,\ + "expires": -1\ + }\ + },\ + "enable_detailed_recording": true,\ + "hmac_enabled": true,\ + "hmac_string": "secret",\ + "meta_data": {}\ +}\ +' +``` + +A response similar to that given below should be returned by Tyk Gateway: + +```json expandable +{ + "key": "eyJvcmciOiI1ZTlkOTU0NGExZGNkNjAwMDFkMGVkMjAiLCJpZCI6ImdycGNfaG1hY19rZXkiLCJoIjoibXVybXVyNjQifQ==", + "status": "ok", + "action": "added", + "key_hash": "a72fcdc09caa86b5" +} +``` + + + +Make a note of the key ID given in the response, since we will need this to test our API. + + + +#### Implement Plugin + +Our custom authentication plugin will perform the following tasks: + +- Extract the *Authorization* and *Date* headers from the request object. +- Parse the *Authorization* header to extract the *keyId*, *algorithm* and *signature* attributes. +- Compute the HMAC signature using the specific algorithm and date included in the header. +- Verify that the computed HMAC signature matches the signature included in the *Authorization* header. A 401 error response will be returned if verification fails. Our plugin will only verify the key against an expected value. In a production environment it will be necessary to verify the key against Redis storage. +- Verify that the *keyId* matches an expected value (VALID_TOKEN). A 401 error response will be returned to Tyk Gateway if verification fails. +- If verification of the signature and key passes then update the session with HMAC enabled and set the HMAC secret. Furthermore, add the key to the *Object* metadata. + +Return the request *Object* containing the updated session back to Tyk Gateway. When developing custom authentication plugins it is the responsibility of the developer to update the session state with the token, in addition to setting the appropriate response status code and error message when authentication fails. + +##### Import Python Modules + +Ensure that the following Python modules are imported at the top of your *tyk_async_server.py* file: + +```python expandable +import asyncio +import base64 +import hashlib +import hmac +import json +import re +import signal +import logging +import urllib.parse + +import grpc +from google.protobuf.json_format import MessageToJson +from grpc_reflection.v1alpha import reflection +import coprocess_object_pb2_grpc +import coprocess_object_pb2 +from coprocess_common_pb2 import HookType +from coprocess_session_state_pb2 import SessionState +``` + +##### Add Constants + +Add the following constants to the top of the *tyk_async_server.py* file, after the import statements: + +```bash +SECRET = "c2VjcmV0" +VALID_TOKEN = "eyJvcmciOiI1ZTlkOTU0NGExZGNkNjAwMDFkMGVkMjAiLCJpZCI6ImdycGNfaG1hY19rZXkiLCJoIjoibXVybXVyNjQifQ==" +``` + +- **SECRET** is a base64 representation of the secret used for HMAC signing. +- **VALID_TOKEN** is the key ID that we will authenticate against. + +The values listed above are designed to align with the examples provided in the *Prerequisites* section, particularly those related to HMAC key generation. If you've made adjustments to the HMAC secret or you've modified the key alias referred to in the endpoint path (for instance, *grpc_hmac_key*), you'll need to update these constants accordingly. + +##### Extract headers + +Add the following function to your *tyk_async_server.py* file to extract a dictionary of the key value pairs from the *Authorization* header. We will use a regular expression to extract the key value pairs. + +```python +def parse_auth_header(auth_header: str) -> dict[str,str]: + pattern = r'(\w+)\s*=\s*"([^"]+)"' + + matches = re.findall(pattern, auth_header) + + parsed_data = dict(matches) + + return parsed_data +``` + +##### Compute HMAC Signature + +Add the following function to your *tyk_async_server.py* to compute the HMAC signature. + +```python expandable +def generate_hmac_signature(algorithm: str, date_string: str, secret_key: str) -> str: + + if algorithm == "hmac-sha256": + hash_algorithm = hashlib.sha256 + elif algorithm == "hmac-sha512": + hash_algorithm = hashlib.sha512 + else: + raise ValueError("Unsupported hash algorithm") + + base_string = f"date: {date_string}" + + logging.info(f"generating signature from: {base_string}") + hmac_signature = hmac.new(secret_key.encode(), base_string.encode(), hash_algorithm) + + return base64.b64encode(hmac_signature.digest()).decode() +``` + +Our function accepts three parameters: + +- **algorithm** is the HMAC algorithm to use for signing. We will use HMAC SHA256 or HMAC SHA512 in our custom authentication plugin +- **date_string** is the date extracted from the date header in the request sent by Tyk Gateway. +- **secret_key** is the value of the secret used for signing. + +The function computes and returns the HMAC signature for a string formatted as *date: date_string*, where *date_string* corresponds to the value of the *date_string* parameter. The signature is computed using the secret value given in the *secret_key* parameter and the HMAC algorithm given in the *algorithm* parameter. A *ValueError* is raised if the hash algorithm is unrecognized. + +We use the following Python modules in our implementation: + +- hmac Python module to compute the HMAC signature. +- base64 Python module to encode the result. + +##### Verify HMAC Signature + +Add the following function to your *tyk_async_server.py* file to verify the HMAC signature provided by the client: + +```python expandable +def verify_hmac_signature(algorithm: str, signature: str, source_string) -> bool: + + expected_signature = generate_hmac_signature(algorithm, source_string, SECRET) + received_signature = urllib.parse.unquote(signature) + + if expected_signature != received_signature: + error = f"Signatures did not match\nreceived: {received_signature}\nexpected: {expected_signature}" + logging.error(error) + else: + logging.info("Signatures matched!") + + return expected_signature == received_signature +``` + +Our function accepts three parameters: + +- **algorithm** is the HMAC algorithm to use for signing. We will use hmac-sha256 or hmac-sha512 in our custom authentication plugin. +- **signature** is the signature string extracted from the *Authorization* header. +- **source_string** is the date extracted from the date header in the request sent by Tyk Gateway. +- **secret_key** is the value of the secret used for signing. + +The function calls *generate_hmac_signature* to verify the signatures match. It returns true if the computed and client HMAC signatures match, otherwise false is returned. + +##### Set Error Response + +Add the following helper function to *tyk_async_server.py* to allow us to set the response status and error message if authentication fails. + +```python +def set_response_error(object: coprocess_object_pb2.Object, code: int, message: str) -> None: + object.request.return_overrides.response_code = code + object.request.return_overrides.response_error = message +``` + +Our function accepts the following three parameters: + +- **object** is an instance of the [Object](/api-management/plugins/rich-plugins#object) message representing the payload sent by Tyk Gateway to the *Dispatcher* service in our gRPC server. For further details of the payload structure dispatched by Tyk Gateway to a gRPC server please consult our gRPC documentation. +- **code** is the HTTP status code to return in the response. +- **message** is the response message. + +The function modifies the *return_overrides* attribute of the request, updating the response status code and error message. The *return_overrides* attribute is an instance of a [ReturnOverrides](/api-management/plugins/rich-plugins#returnoverrides) message that can be used to override the response of a given HTTP request. When this attribute is modified the request is terminated and is not sent upstream. + +##### Authenticate + +Add the following to your *tyk_async_server.py* file to implement the main custom authentication function. This parses the headers to extract the signature and date from the request, in addition to verifying the HMAC signature and key: + +```python expandable +def authenticate(object: coprocess_object_pb2.Object) -> coprocess_object_pb2.Object: + keys_to_check = ["keyId", "algorithm", "signature"] + + auth_header = object.request.headers.get("Authorization") + date_header = object.request.headers.get("Date") + + parse_dict = parse_auth_header(auth_header) + + if not all(key in parse_dict for key in keys_to_check) or not all([auth_header, date_header]): + set_response_error(object, 400, "Custom middleware: Bad request") + return object + + try: + signature_valid = verify_hmac_signature( + parse_dict["algorithm"], + parse_dict["signature"], + date_header + ) + except ValueError: + set_response_error(object, 400, "Bad HMAC request, unsupported algorithm") + return object + + if not signature_valid or parse_dict["keyId"] != VALID_TOKEN: + set_response_error(object, 401, "Custom middleware: Not authorized") + else: + new_session = SessionState() + new_session.hmac_enabled = True + new_session.hmac_secret = SECRET + + object.metadata["token"] = VALID_TOKEN + object.session.CopyFrom(new_session) + + return object +``` + +The *Object* payload received from the Gateway is updated and returned as a response from the *Dispatcher* service: + +- If authentication fails then we set the error message and status code for the response accordingly, using our *set_response_error* function. +- If authentication passes then we update the session attribute in the *Object* payload to indicate that HMAC verification was performed and provide the secret used for signing. We also add the verified key to the meta data of the request payload. + +Specifically, our function performs the following tasks: + +- Extracts the *Date* and *Authorization* headers from the request and verifies that the *Authorization* header is structured correctly, using our *parse_auth_header* function. We store the extracted *Authorization* header fields in the *parse_dict* dictionary. If the structure is invalid then a 400 bad request response is returned to Tyk Gateway, using our *set_response_error* function. +- We use our *verify_hmac_signature* function to compute and verify the HMAC signature. A 400 bad request error is returned to the Gateway if HMAC signature verification fails, due to an unrecognized HMAC algorithm. +- A 401 unauthorized error response is returned to the Gateway under the following conditions: + + - The client HMAC signature and the computed HMAC signature do not match. + - The extracted key ID does not match the expected key value in VALID_TOKEN. + +- If HMAC signature verification passed and the key included in the *Authorization* header is valid then we update the *SessionState* instance to indicate that HMAC signature verification is enabled, i.e. *hmac_enabled* is set to true. We also specify the HMAC secret used for signing in the *hmac_secret* field and include the valid token in the metadata dictionary. + +##### Integrate Plugin + +Update the *Dispatch* method of the *PythonDispatcher* class in your *tyk_async_server.py* file so that our authenticate function is called when the a request is made by Tyk Gateway to execute a custom authentication (*HookType.CustomKeyCheck*) plugin. + +```python expandable +class PythonDispatcher(coprocess_object_pb2_grpc.DispatcherServicer): + async def Dispatch( + self, object: coprocess_object_pb2.Object, context: grpc.aio.ServicerContext + ) -> coprocess_object_pb2.Object: + + logging.info(f"STATE for {object.hook_name}\n{MessageToJson(object)}\n") + + if object.hook_type == HookType.Pre: + logging.info(f"Pre plugin name: {object.hook_name}") + logging.info(f"Activated Pre Request plugin from API: {object.spec.get('APIID')}") + + elif object.hook_type == HookType.CustomKeyCheck: + logging.info(f"CustomAuth plugin: {object.hook_name}") + logging.info(f"Activated CustomAuth plugin from API: {object.spec.get('APIID')}") + + authenticate(object) + + elif object.hook_type == HookType.PostKeyAuth: + logging.info(f"PostKeyAuth plugin name: {object.hook_name}") + logging.info(f"Activated PostKeyAuth plugin from API: {object.spec.get('APIID')}") + + elif object.hook_type == HookType.Post: + logging.info(f"Post plugin name: {object.hook_name}") + logging.info(f"Activated Post plugin from API: {object.spec.get('APIID')}") + + elif object.hook_type == HookType.Response: + logging.info(f"Response plugin name: {object.hook_name}") + logging.info(f"Activated Response plugin from API: {object.spec.get('APIID')}") + logging.info("--------\n") + + return object +``` + +#### Test Plugin + +Create the following bash script, *hmac.sh*, to issue a test request to an API served by Tyk Gateway. The script computes a HMAC signature and constructs the *Authorization* and *Date* headers for a specified API. The *Authorization* header contains the HMAC signature and key for authentication. + +Replace the following constant values with values suitable for your environment: + +- **KEY** represents the key ID for the HMAC signed key that you created at the beginning of this guide. +- **HMAC_SECRET** represents the base64 encoded value of the secret for your HMAC key that you created at the beginning of this guide. +- **BASE_URL** represents the base URL, containing the protocol scheme, host and port number that Tyk Gateway listens to for API requests. +- **ENDPOINT** represents the path of your API that uses HMAC signed authentication. + +```bash expandable +#!/bin/bash + +BASE_URL=http://localhost:8080 +ENDPOINT=/grpc-custom-auth/get +HMAC_ALGORITHM=hmac-sha512 +HMAC_SECRET=c2VjcmV0 +KEY=eyJvcmciOiI1ZTlkOTU0NGExZGNkNjAwMDFkMGVkMjAiLCJpZCI6ImdycGNfaG1hY19rZXkiLCJoIjoibXVybXVyNjQifQ== +REQUEST_URL=${BASE_URL}${ENDPOINT} + + +function urlencode() { + echo -n "$1" | perl -MURI::Escape -ne 'print uri_escape($_)' | sed "s/%20/+/g" +} + +# Set date in expected format +date="$(LC_ALL=C date -u +"%a, %d %b %Y %H:%M:%S GMT")" + +# Generate the signature using hmac algorithm with hmac secret from created Tyk key and +# then base64 encoded +signature=$(echo -n "date: ${date}" | openssl sha512 -binary -hmac "${HMAC_SECRET}" | base64) + +# Ensure the signature is base64 encoded +url_encoded_signature=$(echo -n "${signature}" | perl -MURI::Escape -ne 'print uri_escape($_)' | sed "s/%20/+/g") + +# Output the date, encoded date, signature and the url encoded signature +echo "request: ${REQUEST_URL}" +echo "date: $date" +echo "signature: $signature" +echo "url_encoded_signature: $url_encoded_signature" + +# Make the curl request using headers +printf "\n\n----\n\nMaking request to http://localhost:8080/grpc-custom-auth/get\n\n" +set -x +curl -v -H "Date: ${date}" \ + -H "Authorization: Signature keyId=\"${KEY}\",algorithm=\"${HMAC_ALGORITHM}\",signature=\"${url_encoded_signature}\"" \ + ${REQUEST_URL} +``` + +After creating and saving the script, ensure that it is executable by issuing the following command: + +```bash +chmod +x hmac.sh +``` + +Issue a test request by running the script: + +```bash +./hmac.sh +``` + +Observe the output of your gRPC server. You should see the request payload appear in the console output for the server and your custom authentication plugin should have been triggered. An illustrative example is given below: + +```bash expandable +2024-05-13 12:53:49 INFO:root:STATE for CustomHMACCheck +2024-05-13 12:53:49 { +2024-05-13 12:53:49 "hookType": "CustomKeyCheck", +2024-05-13 12:53:49 "hookName": "CustomHMACCheck", +2024-05-13 12:53:49 "request": { +2024-05-13 12:53:49 "headers": { +2024-05-13 12:53:49 "User-Agent": "curl/8.1.2", +2024-05-13 12:53:49 "Date": "Mon, 13 May 2024 11:53:49 GMT", +2024-05-13 12:53:49 "Host": "localhost:8080", +2024-05-13 12:53:49 "Authorization": "Signature keyId=\"eyJvcmciOiI1ZTlkOTU0NGExZGNkNjAwMDFkMGVkMjAiLCJpZCI6ImdycGNfaG1hY19rZXkiLCJoIjoibXVybXVyNjQifQ==\",algorithm=\"hmac-sha512\",signature=\"e9OiifnTDgi3PW2EGJWfeQXCuhuhi6bGLiGhUTFpjEfgdKmX%2FQOFrePAQ%2FAoSFGU%2FzpP%2FCabmQi4zQDPdRh%2FZg%3D%3D\"", +2024-05-13 12:53:49 "Accept": "*/*" +2024-05-13 12:53:49 }, +2024-05-13 12:53:49 "url": "/grpc-custom-auth/get", +2024-05-13 12:53:49 "returnOverrides": { +2024-05-13 12:53:49 "responseCode": -1 +2024-05-13 12:53:49 }, +2024-05-13 12:53:49 "method": "GET", +2024-05-13 12:53:49 "requestUri": "/grpc-custom-auth/get", +2024-05-13 12:53:49 "scheme": "http" +2024-05-13 12:53:49 }, +2024-05-13 12:53:49 "spec": { +2024-05-13 12:53:49 "bundle_hash": "d41d8cd98f00b204e9800998ecf8427e", +2024-05-13 12:53:49 "OrgID": "5e9d9544a1dcd60001d0ed20", +2024-05-13 12:53:49 "APIID": "6c56dd4d3ad942a94474df6097df67ed" +2024-05-13 12:53:49 } +2024-05-13 12:53:49 } +2024-05-13 12:53:49 +2024-05-13 12:53:49 INFO:root:CustomAuth plugin: CustomHMACCheck +2024-05-13 12:53:49 INFO:root:Activated CustomAuth plugin from API: 6c56dd4d3ad942a94474df6097df67ed +2024-05-13 12:53:49 INFO:root:generating signature from: date: Mon, 13 May 2024 11:53:49 GMT +2024-05-13 12:53:49 INFO:root:Signatures matched! +2024-05-13 12:53:49 INFO:root:-------- +``` + +Try changing the SECRET and/or KEY constants with invalid values and observe the output of your gRPC server. You should notice that authentication fails. An illustrative example is given below: + +``` expandable +2024-05-13 12:56:37 INFO:root:STATE for CustomHMACCheck +2024-05-13 12:56:37 { +2024-05-13 12:56:37 "hookType": "CustomKeyCheck", +2024-05-13 12:56:37 "hookName": "CustomHMACCheck", +2024-05-13 12:56:37 "request": { +2024-05-13 12:56:37 "headers": { +2024-05-13 12:56:37 "User-Agent": "curl/8.1.2", +2024-05-13 12:56:37 "Date": "Mon, 13 May 2024 11:56:37 GMT", +2024-05-13 12:56:37 "Host": "localhost:8080", +2024-05-13 12:56:37 "Authorization": "Signature keyId=\"eyJvcmciOiI1ZTlkOTU0NGExZGNkNjAwMDFkMGVkMjAiLCJpZCI6ImdycGNfaG1hY19rZXkiLCJoIjoibXVybXVyNjQifQ==\",algorithm=\"hmac-sha512\",signature=\"KXhkWOS01nbxuFfK7wEBggkydXlKJswxbukiplboJ2n%2BU6JiYOil%2Bx4OE4edWipg4EcG9T49nvY%2Fc9G0XFJcfg%3D%3D\"", +2024-05-13 12:56:37 "Accept": "*/*" +2024-05-13 12:56:37 }, +2024-05-13 12:56:37 "url": "/grpc-custom-auth/get", +2024-05-13 12:56:37 "returnOverrides": { +2024-05-13 12:56:37 "responseCode": -1 +2024-05-13 12:56:37 }, +2024-05-13 12:56:37 "method": "GET", +2024-05-13 12:56:37 "requestUri": "/grpc-custom-auth/get", +2024-05-13 12:56:37 "scheme": "http" +2024-05-13 12:56:37 }, +2024-05-13 12:56:37 "spec": { +2024-05-13 12:56:37 "bundle_hash": "d41d8cd98f00b204e9800998ecf8427e", +2024-05-13 12:56:37 "OrgID": "5e9d9544a1dcd60001d0ed20", +2024-05-13 12:56:37 "APIID": "6c56dd4d3ad942a94474df6097df67ed" +2024-05-13 12:56:37 } +2024-05-13 12:56:37 } +2024-05-13 12:56:37 +2024-05-13 12:56:37 INFO:root:CustomAuth plugin: CustomHMACCheck +2024-05-13 12:56:37 INFO:root:Activated CustomAuth plugin from API: 6c56dd4d3ad942a94474df6097df67ed +2024-05-13 12:56:37 INFO:root:generating signature from: date: Mon, 13 May 2024 11:56:37 GMT +2024-05-13 12:56:37 ERROR:root:Signatures did not match +2024-05-13 12:56:37 received: KXhkWOS01nbxuFfK7wEBggkydXlKJswxbukiplboJ2n+U6JiYOil+x4OE4edWipg4EcG9T49nvY/c9G0XFJcfg== +2024-05-13 12:56:37 expected: zT17C2tgDCYBJCgFFN/mknf6XydPaV98a5gMPNUHYxZyYwYedIPIhyDRQsMF9GTVFe8khCB1FhfyhpmzrUR2Lw== +``` + +#### Summary + +In this guide, we've explained how to write a Python gRPC custom authentication plugin for Tyk Gateway, using HMAC-signed authentication as a practical example. Through clear instructions and code examples, we've provided developers with insights into the process of creating custom authentication logic tailored to their specific API authentication needs. + +While Tyk Gateway already supports HMAC-signed authentication out of the box, this guide goes beyond basic implementation by demonstrating how to extend its capabilities through custom plugins. By focusing on HMAC-signed authentication, developers have gained valuable experience in crafting custom authentication mechanisms that can be adapted and expanded to meet diverse authentication requirements. + +It's important to note that the authentication mechanism implemented in this guide solely verifies the HMAC signature's validity and does not include access control checks against specific API resources. Developers should enhance this implementation by integrating access control logic to ensure authenticated requests have appropriate access permissions. + +By mastering the techniques outlined in this guide, developers are better equipped to address complex authentication challenges and build robust API security architectures using Tyk Gateway's extensibility features. This guide serves as a foundation for further exploration and experimentation with custom authentication plugins, empowering developers to innovate and customize API authentication solutions according to their unique requirements. + +--- + +
+ +### Performance + +These are some benchmarks performed on gRPC plugins. + +gRPC plugins may use different transports, we've tested TCP and Unix Sockets. + +#### TCP + +TCP Response Times + +TCP Hit Rate + +#### Unix Socket + +Unix Socket Response Times + +Unix Socket Hit Rate + +## Using Lua + +### Overview + +#### Requirements + +Tyk uses [LuaJIT](http://luajit.org/). The main requirement is the LuaJIT shared library, you may find this as `libluajit-x` in most distros. + +For Ubuntu 14.04 you may use: + +`$ apt-get install libluajit-5.1-2 +$ apt-get install luarocks` + +The LuaJIT required modules are as follows: + +* [lua-cjson](https://github.com/mpx/lua-cjson): in case you have `luarocks`, run: `$ luarocks install lua-cjson` + +#### How to write LuaJIT Plugins + +We have a demo plugin hosted in the repo [tyk-plugin-demo-lua](https://github.com/TykTechnologies/tyk-plugin-demo-lua). The project implements a simple middleware for header injection, using a Pre hook (see [Tyk custom middleware hooks](/api-management/plugins/javascript#using-javascript-with-tyk)) and [mymiddleware.lua](https://github.com/TykTechnologies/tyk-plugin-demo-lua/blob/master/mymiddleware.lua). +#### Lua Performance +Lua support is currently in beta stage. We are planning performance optimizations for future releases. +#### Tyk Lua API Methods +Tyk Lua API methods aren’t currently supported. + +### Lua Plugin Tutorial + +#### Settings in the API Definition + +To add a Lua plugin to your API, you must specify the bundle name using the `custom_middleware_bundle` field: + +```json expandable +{ + "name": "Tyk Test API", + "api_id": "1", + "org_id": "default", + "definition": { + "location": "header", + "key": "version" + }, + "auth": { + "auth_header_name": "authorization" + }, + "use_keyless": true, + "version_data": { + "not_versioned": true, + "versions": { + "Default": { + "name": "Default", + "expires": "3000-01-02 15:04", + "use_extended_paths": true, + "extended_paths": { + "ignored": [], + "white_list": [], + "black_list": [] + } + } + } + }, + "proxy": { + "listen_path": "/quickstart/", + "target_url": "http://httpbin.org", + "strip_listen_path": true + }, + "custom_middleware_bundle": "test-bundle", +} +``` + +#### Global settings + +To enable Lua plugins you need to add the following block to `tyk.conf`: + +```json expandable +"coprocess_options": { + "enable_coprocess": true, +}, +"enable_bundle_downloader": true, +"bundle_base_url": "http://my-bundle-server.com/bundles/", +"public_key_path": "/path/to/my/pubkey", +``` + +`enable_coprocess` enables the rich plugins feature. + +`enable_bundle_downloader` enables the bundle downloader. + +`bundle_base_url` is a base URL that will be used to download the bundle, in this example we have "test-bundle" specified in the API settings, Tyk will fetch the following URL: `http://my-bundle-server.com/bundles/test-bundle`. + +`public_key_path` sets a public key, this is used for verifying signed bundles, you may omit this if unsigned bundles are used. + +#### Running the Tyk Lua build + +To use Tyk with Lua support you will need to use an alternative binary, it is provided in the standard Tyk package but it has a different service name. + +Firstly stop the standard Tyk version: + +```console +service tyk-gateway stop +``` + +and then start the Lua build: + +```console +service tyk-gateway-lua start +``` + + diff --git a/api-management/policies.mdx b/api-management/policies.mdx new file mode 100644 index 00000000..543b39d5 --- /dev/null +++ b/api-management/policies.mdx @@ -0,0 +1,1054 @@ +--- +title: "Security Policy and Access Keys" +'og:description': "How to create and use policies and access keys in Tyk" +tags: ['Policies', 'Security', 'Security Policy', 'Access Key', 'API Key'] +sidebarTitle: "Security Policy and Access Keys" +--- + +## Introduction + +In Tyk, a security policy acts as a template for access control and rate limiting. It can be applied to multiple access keys, OAuth clients, or JWT tokens, allowing you to manage API access at scale. + +Access keys, on the other hand, are the tokens that clients use to authenticate and access your APIs. These keys can either have their own individual settings or inherit settings from one or more security policies. + +By leveraging security policies and access keys together, you can: + +- Standardize access control across multiple users or applications. +- Easily update access rights for groups of users. +- Implement tiered access levels (e.g., basic, premium, enterprise). +- Manage and monitor API usage effectively. + +In the following sections, we'll explore how to create and manage security policies and access keys using both the Tyk Dashboard and API. + +## What is a Security Policy + +A Tyk security policy incorporates several security options that can be applied to an API key. It acts as a template that can override individual sections of an API key (or identity) in Tyk. For example, if you had 10,000 API keys issued, how would you ensure that all 10,000 users received an upgraded quota or access a new API that you have published? + +Using policies provides a more scalable and manageable way to control access compared to configuring each key separately, especially when dealing with large numbers of keys. +You could manually modify all 10,000 keys, or you could apply a policy to each of those keys when you create them, and then just modify the policy once. + +**Policies can set:** + +* Access lists for API and versions +* Access lists for method and path (granular control) +* Rate limit for a user +* Quota for a user +* Add tags and metadata + +Each of these can also be overridden in isolation using the partitioning options. When partitioning a policy, only one segment of the policy will be applied to the key. So, for example, if you need to set quotas and rate limits on the user level, but want to manage access control across all of your users, a partitioned policy with only the ACL enabled would achieve this. + +## Relationship between Security Policy and Access Key + +A security policy acts as a template that defines access rights, rate limits, quotas and other security settings whereas an access key (API key) is issued to a client/user to authenticate and access APIs. + +When creating an access key, you can apply one or more security policies to it. This associates the policy settings with that specific key. The policy settings then override the individual settings on the key itself. + +This allows you to manage access controls and limits for groups of keys by just modifying the policy, rather than updating each key individually. You can apply multiple policies to a single key, allowing for flexible combinations of access rights. + +When a request comes in with an access key, Tyk will evaluate the associated policies to determine the permissions and limits for that key. + +In essence, security policies provide a reusable template of settings that can be easily applied to many access keys, simplifying management and allowing for centralized control of API access and usage limits. + +## Policies Guide + +A Tyk policy looks just like the session object that is used when you create a new key: + +```{.copyWrapper} expandable +{ + org_id: "53ac07777cbb8c2d53000002", + rate: 3, + per: 1, + quota_max: 1000, + quota_renewal_rate: 90000, + access_rights: { + b605a6f03cc14f8b74665452c263bf19: { + apiname: "Tyk Test API", + apiid: "b605a6f03cc14f8b74665452c263bf19", + versions: [ + "Default" + ], + allowed_urls: [] + }, + "3b7e73fd18794f146aab9c2e07b787bf": { + apiname: "Second Test API", + apiid: "3b7e73fd18794f146aab9c2e07b787bf", + versions: [ + "Test" + ], + allowed_urls: [] + } + }, + active: true, + is_inactive: false, + tags: [], + key_expires_in: 0 +} +``` + +Here you can see the various fields as they are applied to Tyk keys, these are all described in the Keys section of the [Gateway API](/tyk-gateway-api). + +The important differences here are two new additions: + +- The `active` flag must be set to `true` for Tyk to load the policy into memory, this makes it easy to enable or disable policies without deleting them. + +- Secondly, the `is_inactive` flag applies to the key itself. If you set this value to `true`, any key with this policy will be disabled, you can actually set this same value on a key object to make the single key inactive, but as part of a policy it makes it possible to deny access to a whole block of users with a single change. + +### Trial keys + +It is possible to have a policy create "Trial" keys, these are keys with a fixed expiry date set in the number of seconds from the time of the keys creations. + +Although key expiry can be set in the session object on creation, when a key is created using the portal or a key request it will have a default expiry time. + +To set a trial key expiry, simply add: + +```{.copyWrapper} +`key_expires_in: 50000` +``` + +To the policy object, when the key is generated, the expiry will be forced. + +### Configuring Pro Edition to use a policy list + +Tyk Pro (The Dashboard) has policies enabled by default. + +### Configuring the Open Source Edition to use a policy list + +If your Tyk configuration is standalone and configuration is being managed via the Gateway API without the support of the dashboard, then you will need to set the `policies section` in your configuration file as follows: + +```{.copyWrapper} +"policies": { + "policy_source": "file", + "policy_record_name": "./policies/policies.json" +}, +``` + +Here the `policy_source` section is set to `file` and tells Tyk to look for policy record in the file specified in the `policy_record_name` field. An example file is shipped with Tyk, and it will look like this: + +```{.copyWrapper} expandable +{ + "default": { + "rate": 1000, + "per": 1, + "quota_max": 100, + "quota_renewal_rate": 60, + "access_rights": { + "41433797848f41a558c1573d3e55a410": { + "api_name": "My API", + "api_id": "41433797848f41a558c1573d3e55a410", + "versions": [ + "Default" + ] + } + }, + "org_id": "54de205930c55e15bd000001", + "hmac_enabled": false + } +} +``` + +The record is a single JSON object, with each named key representing the policy ID, so you can list multiple policies within the single JSON object. In the above example we have only defined a single policy called `default`. + +### Applying a policy to a key + +To apply the above policy to a key, we simply need to call the `/create` (or `/add`) endpoint in the Tyk REST API with a session object that has the `apply_policy_id` flag set to the name `default` (or whatever you named your policy). + + + +Although `apply_policy_id` is still supported, it is now deprecated. `apply_policies` is now used to list your policy IDs as an array. This supports the **Multiple Policy** feature introduced in the **v2.4** release. + + + + +```{.copyWrapper} expandable +{ + "allowance": 2, + "rate": 3, + "per": 1, + "expires": 0, + "quota_max": 1000, + "quota_renews": 1429804261, + "quota_remaining": 1000, + "quota_renewal_rate": 90000, + "access_rights": {}, + "org_id": "53ac07777cbb8c2d53000002", + "EnableHTTPSignatureValidation": false, + "hmac_enabled": false, + "hmac_string": "", + "is_inactive": false, + "apply_policy_id": "default", + "apply_policies": [ + "59672779fa4387000129507d", + "53222349fa4387004324324e", + "543534s9fa4387004324324d" + ] +} +``` + +Although we have set the main factors of the key, they will be overridden by the policy as soon as the key is loaded, this will happen each time the key appears, so modifying a policy will have an instant effect on the token. + +### How You Can Create Policies + +[With the Dashboard API](/api-management/gateway-config-managing-classic#create-a-security-policy-with-the-api) + +[With the Gateway API - Open Source tab](/api-management/gateway-config-managing-classic#secure-an-api) + +[With the Dashboard](/api-management/gateway-config-managing-classic#create-a-security-policy-with-the-dashboard) + +--- +--- +--- + +### Secure your APIs by Method and Path + +Tyk already lets you set version access rights, allowed, and blocked paths to control how your users access your APIs, however what has not been easy to do is to restrict access based on specific paths, per key or policy. + +Granular path control allows you to define which methods and paths a key is allowed to access on a per API-version basis. This can be done on a key-by-key basis, or, for even more power and control, through the Policies feature. + +With this feature it is possible to set up tiered access policies for your users, so if you offer read only, free and extended access to your APIs and are charging for the higher levels, you can encode these tiers into policies, and use the granular path control feature to limit what paths and methods the keys with those access policies can access. + +Or, alternatively, you could just upgrade a single key to have more access, both methods use the same, or similar areas of the configuration to make this possible. + + + +Granular permissions are applied *after* version-based (global) allowlist/blocklist rules. + + + + +#### Setting granular paths on a per-key basis + +Let's take a look at a key session definition: + +```{.copyWrapper} expandable +{ + "last_check": 0, + "allowance": 2, + "rate": 3, + "per": 1, + "expires": -1, + "quota_max": 1000, + "quota_renews": 1429804261, + "quota_remaining": 994, + "quota_renewal_rate": 90000, + "access_rights": { + "3b7e73fd18794f146aab9c2e07b787bf": { + "api_name": "Second Test API", + "api_id": "3b7e73fd18794f146aab9c2e07b787bf", + "versions": [ + "Test" + ], + "allowed_urls": [] + }, + "b605a6f03cc14f8b74665452c263bf19": { + "api_name": "Tyk Test API", + "api_id": "b605a6f03cc14f8b74665452c263bf19", + "versions": [ + "Default" + ], + "allowed_urls": [] + } + }, + "org_id": "53ac07777cbb8c2d53000002", + "oauth_client_id": "", + "basic_auth_data": {}, + "hmac_enabled": false, + "hmac_string": "", + "is_inactive": false +} +``` + +Within the `access_rights` section, in each version definition, we can see an `allowed_urls` section, here we can define which URLs are enabled in this key as follows: + +```{.copyWrapper} expandable + "allowed_urls": [ + { + "url": "/resource/(.*)", + "methods": ["GET", "POST"] + } + ] +``` + +Each entry must be a valid Regex pattern and use the [Go syntax](https://golang.org/pkg/regexp/syntax/) (unfortunately Tyk does not accept regular expressions written with the Perl syntax at this time). Methods are case sensitive. This is an allow list, and can be used to define exactly what kind of access a key can have to your API. + +#### Using granular control with a key template + +This feature is much more powerful when applied to key templates and the policies feature, within the policy definition you can add the same section: + +```{.copyWrapper} expandable +{ + "default": { + "rate": 1000, + "per": 1, + "quota_max": 100, + "quota_renewal_rate": 60, + "access_rights": { + "41433797848f41a558c1573d3e55a410": { + "api_name": "My API", + "api_id": "41433797848f41a558c1573d3e55a410", + "versions": [ + "Default" + ], + "allowed_urls": [ + { + "url": "/resource/(.*), + "methods": ["GET", "POST"] + } + ] + } + }, + "org_id": "54de205930c55e15bd000001", + "hmac_enabled": false + } +} +``` + +These paths will be copied into the active key session the next time a key that is using this policy appears. + +## Partitioned Policies + +Creating a policy where access rights, usage quota and rate limit are set in stone may not suit your use case. Instead, you may wish to have only one or two segments of a token managed at policy level and the other segments managed at key level or by another policy. + +### Example Use Case + +You have different tiers of rate limiting as follows: + +* Tier A has access to the API at a rate of 1000 per 60 seconds +* Tier B a rate of 500 per 60 seconds +* Tier C a rate of 250 per 60 seconds + +You could create three separate policies that allow the same access rights and usage quota but have different rate limiting, or, you could create one policy and partition it by enforcing only access rights and usage quota, leaving rate limiting to be defined at key level or by another policy. + +Because the access rights and usage quota are enforced at policy level, you can only make changes to them within the policy. Any changes will then be inherited by all keys with that policy applied without affecting the rate limit defined at key level. + +A partitioned policy can enforce any of these elements individually or together on a key: + +* The Access Control List (ACL), configured using the `access_rights` field + * When applying partitioned policies to a key, at least one of these policies needs to enforce ACL +* The Rate limit +* The Quota limit +* The GraphQL complexity (currently only query-depth limit is supported) + +### Set up a partition in an API + +You can partition your policy by adding a `partitions` section to your policy object: + +```{.json} expandable +"partitions": { + "quota": false, + "rate_limit": false, + "acl": false, + "complexity": false +} +``` + +* `quota`: If set to `true`, enforce the quota element of this policy +* `rate_limit`: If set to `true`, enforce the rate limit of this policy +* `acl`: If set to `true`, enforce the access control rules of this policy +* `complexity`: If set to `true`, enforce the GraphQL complexity rules of this policy + +Partitions can be applied together, if you select all of them then essentially the whole policy will be enforced. + +### Set up a partition in the Tyk Dashboard + +Once you have added access rights to your policy, open the Global Limits and Quota panel. You’ll see the Policy Partitioning section where you can uncheck Access Rights, Usage Quota or Rate Limiting to enable their value to be defined at key level. + +For example, the screenshot below shows that rate limit has not been enforced and therefore can be defined at key level when this policy is applied to a key. + +Global Limits + +### Partitioned Policy Functionality + +In Gateway v2.4 and Dashboard v1.4 We extended support for partitioned policies, and you can now apply multiple when creating a key. We’ll cover all combinations and how you can expect the key to react. + + + +#### Applying partitioned policies to a key with the same segments enforced + +If you apply partitioned policies to a key with the same segments enforced, you will be able to override any segment that has not been enforced and define new rules specific to that key. + +**Example One** - Single Policy: Policy A has access rights and usage quota enforced meaning the rate limiting can be defined at key level. + +**Example Two** - Multiple Policies: Policy A and Policy B have access rights and usage quota enforced meaning the rate limiting defined at key level will be inherited by both policies. + +```{.json} expandable +{ + "policy_a": { + "access_rights": { + "1": { + "api_name": "API One", + "api_id": "1", + "versions": [ + "Default" + ] + } + }, + "active": true, + "id": "policy_a", + "name": "policy_a", + "partitions": { + "acl": true, + "complexity": false, + "per_api": false, + "quota": true, + "rate_limit": false + }, + "quota_max": 100, + "quota_renewal_rate": 3600, + "state": "active", + "tags": [] + }, + "policy_b": { + "access_rights": { + "2": { + "api_name": "API Two", + "api_id": "2", + "versions": [ + "Default" + ] + } + }, + "active": true, + "id": "policy_b", + "name": "policy_b", + "partitions": { + "acl": true, + "complexity": false, + "per_api": false, + "quota": true, + "rate_limit": false + }, + "quota_max": 50, + "quota_renewal_rate": 3600, + "state": "active", + "tags": [] + } +} +``` + +##### Use Case + +You want to give access to the same API with the same usage quota but define separate rate limits for various developers. + +#### Applying partitioned policies to a key with different segments enforced + +For ultimate flexibility, you can create policies that each have only one segment enforced. Instead of creating multiple policies that cover a variety of scenarios you can create a few as building blocks to create unique combinations that suit your needs. + +**Example:** + +Policy A has API 1 enforced +Policy B has API 2 enforced +Policy C has a rate limit of 1000 per 60 seconds enforced +Policy D has a rate limit of 2000 per 60 seconds enforced +Policy E has an unlimited request usage quota enforced +Policy F has 10,000 requests per hour usage quota enforced + +If Policy A, C and E is applied to a key it will give access to API 1 at a rate of 1000 per 60 seconds with unlimited requests. + +If Policy A, D and E is applied to a key it will give access to API 1 at a rate of 2000 per 60 seconds with unlimited requests. + +```{.json} expandable +{ + "policy_a": { + "access_rights": { + "1": { + "api_name": "API 1", + "api_id": "1", + "versions": [ + "Default" + ] + } + }, + "active": true, + "id": "policy_a", + "name": "policy_a", + "partitions": { + "acl": true, + "complexity": false, + "per_api": false, + "quota": false, + "rate_limit": false + }, + "state": "active", + "tags": [] + }, + "policy_b": { + "access_rights": { + "2": { + "api_name": "API 2", + "api_id": "2", + "versions": [ + "Default" + ] + } + }, + "active": true, + "id": "policy_b", + "name": "policy_b", + "partitions": { + "acl": true, + "complexity": false, + "per_api": false, + "quota": false, + "rate_limit": false + }, + "state": "active", + "tags": [] + }, + "policy_c": { + "access_rights": {}, + "active": true, + "id": "policy_c", + "name": "policy_c", + "partitions": { + "acl": false, + "complexity": false, + "per_api": false, + "quota": false, + "rate_limit": true + }, + "per": 60, + "rate": 1000, + "state": "active", + "tags": [], + "throttle_interval": -1, + "throttle_retry_limit": -1 + }, + "policy_d": { + "access_rights": {}, + "active": true, + "id": "policy_d", + "name": "policy_d", + "partitions": { + "acl": false, + "complexity": false, + "per_api": false, + "quota": false, + "rate_limit": true + }, + "per": 60, + "rate": 2000, + "state": "active", + "tags": [], + "throttle_interval": -1, + "throttle_retry_limit": -1 + }, + "policy_e": { + "access_rights": {}, + "active": true, + "id": "policy_e", + "name": "policy_e", + "partitions": { + "acl": false, + "complexity": false, + "per_api": false, + "quota": true, + "rate_limit": false + }, + "quota_max": -1, + "quota_renewal_rate": -1, + "state": "active", + "tags": [], + "throttle_interval": -1, + "throttle_retry_limit": -1 + }, + "policy_f": { + "access_rights": {}, + "active": true, + "id": "policy_f", + "name": "policy_f", + "partitions": { + "acl": false, + "complexity": false, + "per_api": false, + "quota": true, + "rate_limit": false + }, + "quota_max": 10000, + "quota_renewal_rate": 3600, + "state": "active", + "tags": [], + "throttle_interval": -1, + "throttle_retry_limit": -1 + } +} +``` + +##### Use Case + +You have 20 developer keys that use a combination of Policy A, B, C, D, E and F and have decided that you’d now like to alter Policy D’s rate limit to 3000 per 60 seconds. All keys with Policy D applied will now inherit the new value instantly. If you had created each of the keys without using policies you would have to find and edit each key manually. + +#### Applying both a partitioned policy and a non-partitioned policy to a key + +If you apply both a partitioned policy and a non-partitioned policy to the same key, any segments that have not been enforced in the partitioned policy will inherit the values in the non-partitioned policy. + +##### Example + +Policy A has enforced access to API 1 with a rate limit of 1000 per 60 seconds and unlimited requests for the usage quota. +Policy B only has enforced access to API 2 + +If both policies were applied to a key, Policy B would automatically inherit Policy A’s rate limit and usage quota because Policy B did not have rate limit or usage quota enforced. + +```{.json} expandable +{ + "policy_a": { + "access_rights": { + "1": { + "api_name": "API One", + "api_id": "1", + "versions": [ + "Default" + ] + } + }, + "active": true, + "partitions": { + "acl": true, + "complexity": false, + "per_api": false, + "quota": true, + "rate_limit": true + }, + "per": 60, + "quota_max": -1, + "quota_renewal_rate": -1, + "rate": 1000, + "state": "active", + "tags": [], + "throttle_interval": -1, + "throttle_retry_limit": -1 + }, + "policy_b": { + "access_rights": { + "2": { + "api_name": "API Two", + "api_id": "2", + "versions": [ + "Default" + ] + } + }, + "active": true, + "partitions": { + "acl": true, + "complexity": false, + "per_api": false, + "quota": false, + "rate_limit": false + }, + "state": "active", + "tags": [] + } +} +``` + +##### Use Case + +A developer already has a key that gives access to Policy A and now requires access to another API product. The developer is already paying for a specific rate and limit and just needs access to the additional API. Instead of editing Policy A to allow for the additional API access (which would then affect all keys with this policy applied), we can instead create Policy B and combine the two, allowing the additional API in Policy B to inherit the same rate and limit the developer requires. + + + +For v2.4 and 1.4 multiple policies are only supported only via the Add Key section and via the API. +Support oAuth, and Portal API Catalogs are planned for subsequent releases. +Support of multiple policies for JWT and OIDC is done through the API definition when using scopes. + + + +## Access Keys + +### Access Key Expiry + +Key Expiry allows you to set the lifetime of tokens, ensuring a regular re-cycling of API tokens. If a key has expired Tyk will no longer let requests through on a token, however this **does not mean** that Tyk will remove the key. + +#### Token Expiry Behavior and Time-To-Live + +If a key is expired, Tyk will return a warning that the token has expired to the end user. If a token has been deleted, then Tyk will return an access denied response to the client. This is an important difference. In some cases, API tokens are hard-coded (this is terrible practice, but it does happen far more often than you might think). In this case it is extremely expensive to replace the token if it has expired. + +In the above case, if a token had been deleted because the **Time To Live** of the token matched it's expiry time, then the end user would need to replace the token with a new one. However, because we do not expire the key it is possible for an administrator to reset the expiry of the token to allow access and manage renewal in a more granular way. + +#### Timestamp format on a session object + +Tyk manages timestamps in the Unix timestamp format - this means that when a date is set for expiry it should be converted to a Unix timestamp (usually a large integer) which shows seconds since the epoch (Jan 1 1970). This format is used because it allows for faster processing and takes into account timezone differences without needing localisation. + +Key sessions are created and updated using the Tyk Gateway API, in order to set the expiry date for a key, update the `expires` value with a Unix timestamp of when the key should expire. + +
+ + + +`expires` can only be a positive number, or `0` if you don't want the key to expire. + + + + +#### How to delete expired tokens + +In order to not clutter the database with expired tokens, Tyk provides a way to force a TTL on all keys, this is a maximum time to live and should always be significantly larger than your maximum expiry setting. This setting must be set on a per-API basis. + +To enforce a TTL, set the `session_lifetime` value (in seconds) in your API Definition Object, this will need to be managed via the Dashboard REST API. + +### Access Key Hashing + +Tyk stores all API Tokens and their equivalent Session Objects in a Redis DB. Because of this, Tyk will, by default, obfuscate the tokens in Redis using a key hash. + +#### Default Key Hash Algorithm + +To find a balance between performance and security, the default algorithm used by Tyk to do the hashing is `murmur3`, and serves more to obfuscate than to cryptographically secure the tokens. + +It is possible to disable key hashing in Tyk using `hash_keys` set to `false` in your `tyk.conf` and `tyk_analytics.conf`. + +See the [Gateway Configuration Options](/tyk-oss-gateway/configuration) for more details. + +#### Custom Key Hash Algorithms + +To set a custom algorithm, you need to set `hash_key_function` in your `tyk.conf` to one of the following options: + +* `murmur32` +* `murmur64` +* `murmur128` +* `sha256` + +MurMur non-cryptographic hash functions are considered as the industry fastest and conflict-prone algorithms up to date, which gives a nice balance between security and performance. With this change you now you can choose the different hash length, depending on your organization security policies. We have also introduced a new `sha256` cryptographic key hashing algorithm, for cases when you are willing to sacrifice some performance for additional security. + +Performance wise, setting new key hashing algorithms can increase the key hash length, as well as key length itself, so expect that your analytics data size to grow (but not that much, up to about 10%). Additionally, if you set the `sha256` algorithm, it will significantly slowdown Tyk, because cryptographic functions are slow by design but very secure. + +Technically wise, it is implemented by new key generation algorithms, which now embed additional metadata to the key itself, and if you are curious about the actual implementation details, feel free to check the following [pull request](https://github.com/TykTechnologies/tyk/pull/1753). + +Changing hashing algorithm is entirely backward compatible. All your existing keys will continue working with the old `murmur32` hashing algorithm, and your new keys will use the algorithm specified in your `tyk.conf`. Moreover, changing algorithms is also backward compatible, and Tyk will maintain keys with multiple hashing algorithms without any issues. + +A hashed installation imposes some constraints on how Tyk is used: + +* Listing tokens requires setting `enable_hashed_keys_listing` to `true` in your `tyk.conf` file +* Tokens appear in Analytics in their hashed form + + + + + Switching from a hashed installation to non-hashed means all existing tokens cannot be used (they will not be correctly validated). + + + + +#### Using Hashed Keys Endpoints + +- endpoints `POST /keys/create`, `POST /keys` and `POST /keys/{keyName}` also return the field `"key_hash"` for future use +- endpoint `GET /keys` get all (or per API) key hashes. You can disable this endpoint by using the new `tyk.conf` setting `enable_hashed_keys_listing` (set to `false` by default) +- endpoint `GET /keys/{keyName}` was modified to be able to get a key by hash. You just need provide the key hash as a `keyName` +and call it with the new optional query parameter `hashed=true`. So the new format is `GET /keys/{keyName}?hashed=true"` +- we also have the same optional parameter for endpoint `DELETE /keys/{keyName}?hashed=true` and call it with the optional query parameter `hashed=true`. So the format is `GET /keys/{keyName}?hashed=true"` +- The same optional parameter is available for the `DELETE /keys/{keyName}?hashed=true` endpoint + +See the Keys section of [Tyk Gateway API Swagger page](/tyk-gateway-api) for more details. + +### Access Key Level Security + +Tyk supports the concept of access control at the key level. Access control is managed via three important settings in a session object. In order to be fully clear on how Tyk handles access control, it's worth looking at the key settings that go into a user session object. A full description of each of the options can be found in the [Tyk Gateway API documentation](/tyk-gateway-api). + +Tyk will store each access key as a record in your Redis database, and this key will have certain metadata attached to it. The record takes this form: + +```{.copyWrapper} expandable +{ + "allowance": 1000, + "rate": 1000, + "per": 60, + "expires": -1, + "quota_max": -1, + "quota_renews": 1406121006, + "quota_remaining": 0, + "quota_renewal_rate": 60, + "access_rights": { + "APIID1": { + "api_name": "HMAC API", + "api_id": "APIID1", + "versions": [ + "Default" + ] + } + }, + "org_id": "1", + "hmac_enabled": false, + "hmac_string": "" +} +``` + +The important elements that manage access control are the following fields: + +* `allowance` & `rate`: these should be set to the same value, these are the users allowance during a period as set by `per`. +* `per`: The time in seconds where a rate limit is applied. +* `expires`: The date when the key expires. **Note:** `expires` can only be a positive number, or -1 (unlimited). +* `quota_max`: The usage quota for the user. **Note:** `quota_max` can only be a positive number, or -1 (unlimited). +* `quota_renews`: the Unix timestamp when the quota is renewed. +* `quota_renewal_rate`: The time, in seconds, of when the quota gets renewed (e.g. 86400 would represent 1 day). + +These settings can be used exclusively or in conjunction with one another to create usage patterns and tiers of access for your users. Each time a request is processed by Tyk, the session will be updated with an updated quota (if used), and updated throttling rate depending on the time-frame specified. + +Creating new keys is done by POSTing an object such as the above to the Tyk create key API endpoint. See the keys section of the [Tyk Gateway API OpenAPI/Swagger](/tyk-gateway-api) page. + +The three types of access control are: + +#### Rate limiting + +Also known as throttling, the API will actively only allow a key to make x requests per y time period. this is very useful if you want to ensure your API does not get flooded with requests. + +In order to apply a rate limit: + +1. Ensure that `allowance` and `rate` are set to the same value, this should be number of requests to be allowed in a time period, so if you wanted 100 requests every second, set this value to 100. +2. Ensure that `per` is set to the time limit. Again, as in the above example, if you wanted 100 requests per second, set this value to 1. If you wanted 100 per 5 seconds, set this value to 5 etc. + +#### Quotas + +A quota is similar to a rate limit, as it allows a certain number of requests through in a time period. However, traditionally these periods are much longer, so for example if you would like to limit a user to only 10,000 requests to the API per month, you can create a key that has no rate limiting but will disallow access once the quota is empty. Tyk will automatically reset the quota if the time limit on reset has been exceeded. + +In order to set a quota for a user: + +1. Ensure that `quota_max` is set to the maximum amount of requests that a user is allowed to make in a time period. +2. Ensure `quota_remaining` is set to the same value as `quota_max`, this is the value that will decrement on each request (failed or successful). +3. Set the `quota_renewal_rate` to the value, in seconds, of when the quota should renew. For example, if you would like it to renew every 30 days, you would have `2592000` seconds (`((60*60) * 24) * 30 = 2592000`). + +To set an unlimited quota, set `quota_max` to `-1`. + + + +`quota_max` can only be a positive number, or -1 (unlimited). + + + +#### Key Expiry + +If you set a date in the key expiry field, when the key is created (or updated), the expiry time is also set as the keys deletion time from Redis. If a key has expired Tyk will no longer let requests through on this key. + +Tyk manages timestamps in the Unix timestamp format - this means that when a date is set for expiry it should be converted to a Unix timestamp (usually a large integer) which shows seconds since the epoch (Jan 1 1970). This format is used because it allows for faster processing and takes into account timezone differences without needing localisation. + +Key sessions are created and updated using the Tyk REST API, in order to set the expiry date for a key, update the `expires` value with the timestamp of when the key should expire. + +Leave this field empty for it never to expire. + +## Understanding Tyk Session + +### What is a Session Object + +In Tyk, all identities are mapped to a session object. Identities can be in the form of Bearer Tokens, HMAC Keys, JSON Web Tokens, OpenID Connect identities and Basic Auth users. + +You should think about a session object as the metadata associated with a user, or the identity trying to gain access to your services. + +In Tyk, a session object encapsulates the following details for any given identity: + +* What rate limit to apply +* What quota to apply +* What Access Control List to apply +* What policy ID to use to override the above (if set) +* When the session holder's access expires + +Tyk also allows some additional metadata for a session object which is valuable for transformation or upstream identification purposes: + +* Metadata (a string key/value map that can hold any data) +* Alias (a human-readable name for the identity) + + + + + Expiry is not the same as invalidation, in Tyk, a session object will be "expired" but will still be in the database in order to inform the session owner that their token has expired and they should renew, if the token was invalidated (deleted after the expiry period), then the user would simply be denied access and their token would be invalid. This is important for developers that have (but shouldn't) hard-coded their token into their app so it is hard to change. + + + + +#### Where are session objects stored? + +Session objects are stored in Redis, not in MongoDB or in the Gateway itself. Session objects are stored as a token string / JSON object key/value pair in the Redis DB. + +By default, the token itself is hashed and therefore **obfuscated**, this means using the Alias is important to identify token data in analytics and logs. + +#### Where can I get more information? + +A session object is just a JSON object. For more details of each parameter in the session object, see [Tyk Token Session Object Details](/api-management/policies#session-object). + +#### Session Object + +```{.copyWrapper} expandable +{ + "last_check": 0, + "allowance": 1000, + "rate": 1000, + "per": 1, + "expires": 1458669677, + "quota_max": 1000, + "quota_renews": 1458667309, + "quota_remaining": 1000, + "quota_renewal_rate": 3600, + "access_rights": { + "e1d21f942ec746ed416ab97fe1bf07e8": { + "api_name": "Closed", + "api_id": "e1d21f942ec746ed416ab97fe1bf07e8", + "versions": ["Default"], + "allowed_urls": null + } + }, + "org_id": "53ac07777cbb8c2d53000002", + "oauth_client_id": "", + "basic_auth_data": { + "password": "", + "hash_type": "" + }, + "jwt_data": { + "secret": "" + }, + "hmac_enabled": false, + "hmac_string": "", + "is_inactive": false, + "apply_policy_id": "", + "apply_policies": [ + "59672779fa4387000129507d", + "53222349fa4387004324324e", + "543534s9fa4387004324324d" + ], + "data_expires": 0, + "monitor": { + "trigger_limits": null + }, + "meta_data": { + "test": "test-data" + }, + "tags": ["tag1", "tag2"], + "alias": "john@smith.com" +} +``` + +* `last_check` (**deprecated**): No longer used, but this value is related to rate limiting. + +* `allowance` (**deprecated**): No longer directly used, this value, no key creation, should be the same as `rate`. + +* `rate`: The number of requests that are allowed in the specified rate limiting window. + +* `per`: The number of seconds that the rate window should encompass. + +* `expires`: A Unix timestamp that defines when the key should expire. You can set this to `0` (zero) if you don't want the key to expire. + +* `quota_max`: The maximum number of requests allowed during the quota period. + +* `quota_renews`: An epoch that defines when the quota renews. + +* `quota_remaining`: The number of requests remaining for this user's quota (unrelated to rate limit). + +* `quota_renewal_rate`: The time, in seconds. during which the quota is valid. So for `1000` requests per hour, this value would be `3600` while `quota_max` and `quota_remaining` would be `1000`. + +* `access_rights`: This section is defined in the Access Control section of this documentation, use this section define what APIs and versions this token has access to. + +* `org_id`: The organization this user belongs to, this can be used in conjunction with the `org_id` setting in the API Definition object to have tokens "owned" by organizations. See the Organizations Quotas section of the [Tyk Gateway API](/tyk-gateway-api). + +* `oauth_client_id`: This is set by Tyk if the token is generated by an OAuth client during an OAuth authorization flow. + +* `basic_auth_data`: This section defines the basic auth password and hashing method. + +* `jwt_data`: This section contains a JWT shared secret if the ID matches a JWT ID. + +* `hmac_enabled`: If this token belongs to an HMAC user, this will set the token as a valid HMAC provider. + +* `hmac_string`: The value of the HMAC shared secret. + +* `is_inactive`: Set this value to `true` to deny access. + +* `apply_policy_id` (**supported but now deprecated**): The policy ID that is bound to this token. + +* `apply_policies`: This replaces `apply_policy_id` and lists your policy IDs as an array. This supports the **Multiple Policy** feature introduced in the **v2.4 of the Gateway**. + +* `data_expires`: An value, in seconds, that defines when data generated by this token expires in the analytics DB (must be using Pro edition and MongoDB). + +* `monitor`: Rate monitor trigger settings, defined elsewhere in the documentation. + +* `meta_data`: Metadata to be included as part of the session, this is a key/value string map that can be used in other middleware such as transforms and header injection to embed user-specific data into a request, or alternatively to query the providence of a key. + +* `tags`: Tags are embedded into analytics data when the request completes. If a policy has tags, those tags will supersede the ones carried by the token (they will be overwritten). + +* `alias`: As of v2.1, an Alias offers a way to identify a token in a more human-readable manner, add an Alias to a token in order to have the data transferred into Analytics later on so you can track both hashed and un-hashed tokens to a meaningful identifier that doesn't expose the security of the underlying token. + +### What is a Session Metadata + +As described in [What is a Session Object?](/api-management/policies#what-is-a-session-object), all Tyk tokens can contain a metadata field. This field is a string key/value map that can store any kind of information about the underlying identity of a session. + +The metadata field is important, because it can be used in various ways: + +- to inform an admin of the provenance of a token +- values can be injected into headers for upstream services to consume (e.g. a user ID or an email address provided at the time of creation) +- values can be used in dynamic [JavaScript](/api-management/plugins/javascript#accessing-external-and-dynamic-data) middleware and Virtual Endpoints for further validation or request modification + +Metadata is also injected by other Tyk Components when keys are created using "generative" methods, such as JSON Web Token and OIDC session creation and via the Developer Portal, to include information about the underlying identity of the token when it comes from a third-party such as an OAuth IDP (e.g. OIDC). + +#### Middleware that can use metadata + +Metadata is exposed in several middleware for use in the middleware configuration: + +- [URL Rewrite](/transform-traffic/url-rewriting#pattern) +- [Request Header Transformation](/api-management/traffic-transformation/request-headers#injecting-dynamic-data-into-headers) +- [Response Header Transformation](/api-management/traffic-transformation/request-headers#injecting-dynamic-data-into-headers) +- [Request Body Transformation](/api-management/traffic-transformation/request-body#data-accessible-to-the-middleware) +- [Response Body Transformation](/api-management/traffic-transformation/request-body#data-accessible-to-the-middleware) +- [Virtual Endpoints](/api-management/traffic-transformation/virtual-endpoints) + +You can also access and update metadata from your [custom plugins](/api-management/plugins/overview#). For an example of this, take a look at this [gRPC enabled GO Server](https://github.com/TykTechnologies/tyk-grpc-go-basicauth-jwt). It's a PoC middleware that injects a JWT value into metadata and then accesses it later in the stream. + + +## Set Physical Key Expiry and Deletion +Tyk makes a clear distinction between an API authorization key expiring and being deleted from the Redis storage. + +- When a key expires, it remains in the Redis storage but is no longer valid. Consequently, it is no longer authorized to access any APIs. If a key in Redis has expired and is passed in an API request, Tyk will return `HTTP 401 Key has expired, please renew`. + - When a key is deleted from Redis, Tyk no longer knows about it, so if it is passed in an API request, Tyk will return `HTTP 400 Access to this API has been disallowed`. + +Tyk provides separate control for the expiration and deletion of keys. + +Note that where we talk about keys here, we are referring to [Session Objects](/api-management/policies#what-is-a-session-object), also sometimes referred to as Session Tokens + +### Key expiry + +Tyk's API keys ([token session objects](/api-management/policies#session-object)) have an `expires` field. This is a UNIX timestamp and, when this date/time is reached, the key will automatically expire; any subsequent API request made using the key will be rejected. + +### Key lifetime + +Tyk does not automatically delete keys when they expire. You may prefer to leave expired keys in Redis storage, so that they can be renewed (for example if a user has - inadvisedly - hard coded the key into their application). Alternatively, you may wish to delete keys to avoid cluttering up Redis storage with obsolete keys. + +You have two options for configuring the lifetime of keys when using Tyk: + +1. At the API level +2. At the Gateway level + +#### API-level key lifetime control + +You can configure Tyk to delete keys after a configurable period (lifetime) after they have been created. Simply set the `session_lifetime` field in your API Definition and keys created for that API will automatically be deleted when that period (in seconds) has passed. + +The default value for `session_lifetime` is 0, this is interpreted as an infinite lifetime which means that keys will not be deleted from Redis. + +For example, to have keys live in Redis for only 24 hours (and be deleted 24 hours after their creation) set: + +```{.json} +"session_lifetime": 86400 +``` + + + +There is a risk, when configuring API-level lifetime, that a key will be deleted before it has expired, as `session_lifetime` is applied regardless of whether the key is active or expired. To protect against this, you can configure the [session_lifetime_respects_key_expiration](/tyk-oss-gateway/configuration#session_lifetime_respects_key_expiration) parameter in your `tyk.conf`, so that keys that have exceeded their lifetime will not be deleted from Redis until they have expired. + + + +This feature works nicely with [JWT](/basic-config-and-security/security/authentication-authorization/json-web-tokens) or [OIDC](/api-management/client-authentication#integrate-with-openid-connect-deprecated) authentication methods, as the keys are created in Redis the first time they are in use so you know when they will be removed. Be extra careful in the case of keys created by Tyk (Auth token or JWT with individual secrets) and set a long `session_lifetime`, otherwise the user might try to use the key **after** it has already been removed from Redis. + +#### Gateway-level key lifetime control + +You can set a global lifetime for all keys created in the Redis by setting [global_session_lifetime](/tyk-oss-gateway/configuration#global_session_lifetime) in the `tyk.conf` file; this parameter is an integer value in seconds. + +To enable this global lifetime, you must also set the [force_global_session_lifetime](/tyk-oss-gateway/configuration#force_global_session_lifetime) parameter in the `tyk.conf` file. + +#### Summary of key lifetime precedence + +The table below shows the key lifetime assigned for the different permutations of `force_global_session_lifetime` and `session_lifetime_respects_key_expiration` configuration parameters. +| `force_global_session_lifetime` | `session_lifetime_respects_key_expiration` | Assigned lifetime | +| :--------------------------------- | :-------------------------------------------- | :------------------------------------------- | +| `true` | `true` | `global_session_lifetime` | +| `true` | `false` | `global_session_lifetime` | +| `false` | `true` | larger of `session_lifetime` or `expires` | +| `false` | `false` | `session_lifetime` | + + + +It is important to remember that a value of `0` in `session_lifetime` or `global_session_lifetime` is interpreted as infinity (i.e. key will not be deleted if that control is in use) - and if a field is not set, this is treated as `0`. +
+If you want the key to be deleted when it expires (i.e. to use the expiry configured in `expires` within the key to control deletion) then you must set a non-zero value in `session_lifetime` and configure both `session_lifetime_respects_key_expiration:true` and `force_global_session_lifetime:false`. + + diff --git a/api-management/rate-limit.mdx b/api-management/rate-limit.mdx new file mode 100644 index 00000000..ddbfd855 --- /dev/null +++ b/api-management/rate-limit.mdx @@ -0,0 +1,791 @@ +--- +title: "Rate Limiting" +'og:description': "Overview of Rate Limiting with the Tyk Gateway" +tags: ['Rate Limit', 'Rate Limiting', 'Rate Limit Algorithms', 'Distributed Rate Limiter', 'Redis Rate Limiter', 'Fixed Window', 'Spike Arrest', 'Rate Limit Scope', 'Local', 'Local rate Limits', 'Tyk Classic', 'Tyk Classic API', 'Tyk OAS', 'Tyk OAS API', 'Rate Limiting', 'Global limits', 'Per API limits'] +sidebarTitle: "Rate Limiting" +--- + +## Introduction + +API rate limiting is a technique that allows you to control the rate at which clients can consume your APIs and is one of the fundamental aspects of managing traffic to your services. It serves as a safeguard against abuse, overloading, and denial-of-service attacks by limiting the rate at which an API can be accessed. By implementing rate limiting, you can ensure fair usage, prevent resource exhaustion, and maintain system performance and stability, even under high traffic loads. + +## What is rate limiting? + +Rate limiting involves setting thresholds for the maximum number of requests that can be made within a specific time window, such as requests per second, per minute, or per day. Once a client exceeds the defined rate limit, subsequent requests may be delayed, throttled, or blocked until the rate limit resets or additional capacity becomes available. + +## When might you want to use rate limiting? + +Rate limiting may be used as an extra line of defense around attempted denial of service attacks. For instance, if you have load-tested your current system and established a performance threshold that you would not want to exceed to ensure system availability and/or performance then you may want to set a global rate limit as a defense to ensure it hasn't exceeded. + +Rate limiting can also be used to ensure that one particular user or system accessing the API is not exceeding a determined rate. This makes sense in a scenario such as APIs which are associated with a monetization scheme where you may allow so many requests per second based on the tier in which that consumer is subscribed or paying for. + +Of course, there are plenty of other scenarios where applying a rate limit may be beneficial to your APIs and the systems that your APIs leverage behind the scenes. + +## How does rate limiting work? + +At a basic level, when rate limiting is in use, Tyk Gateway will compare the incoming request rate against the configured limit and will block requests that arrive at a higher rate. For example, let’s say you only want to allow a client to call the API a maximum of 10 times per minute. In this case, you would apply a rate limit to the API expressed as "10 requests per 60 seconds". This means that the client will be able to successfully call the API up to 10 times within any 60 second interval (or window) and after for any further requests within that window, the user will get an [HTTP 429 (Rate Limit Exceeded)](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429) error response stating the rate limit has been exceeded. + +Tyk's rate limiter is configured using two variables: +- `rate` which is the maximum number of requests that will be permitted during the interval (window) +- `per` which is the length of the interval (window) in seconds + +So for this example you would configure `rate` to 10 (requests) and `per` to 60 (seconds). + +### Rate limiting scopes: API-level vs key-level + +Rate limiting can be applied at different scopes to control API traffic effectively. This section covers the two primary scopes - API-level rate limiting and key-level rate limiting. Understanding the distinctions between these scopes will help you configure appropriate rate limiting policies based on your specific requirements. + +#### API-level rate limiting + +API-level rate limiting aggregates the traffic coming into an API from all sources and ensures that the overall rate limit is not exceeded. Overwhelming an endpoint with traffic is an easy and efficient way to execute a denial of service attack. By using a API-level rate limit you can easily ensure that all incoming requests are within a specific limit so excess requests are rejected by Tyk and do not reach your service. You can calculate the rate limit to set by something as simple as having a good idea of the maximum number of requests you could expect from users of your API during a period. You could alternatively apply a more scientific and precise approach by considering the rate of requests your system can handle while still performing at a high-level. This limit may be easily determined with some performance testing of your service under load. + +#### Key-level rate limiting + +Key-level rate limiting is more focused on controlling traffic from individual sources and making sure that users are staying within their prescribed limits. This approach to rate limiting allows you to configure a policy to rate limit in two ways: + +- **key-level global limit** limiting the rate of calls the user of a key can make to all APIs authorized by that key +- **key-level per-API limit** limiting the rate of calls the user of a key can make to specific individual APIs +- **key-level per-endpoint limit** limiting the rate of calls the user of a key can make to specific individual endpoints of an API + +These guides include explanation of how to configure key-level rate limits when using [API Keys](/api-management/gateway-config-managing-classic#access-an-api) and [Security Policies](/api-management/gateway-config-managing-classic#secure-an-api). + +#### Which scope should I use? + +The simplest way to figure out which level of rate limiting you’d like to apply can be determined by asking a few questions: + +- do you want to protect your service against denial of service attacks or overwhelming amounts of traffic from **all users** of the API? **You’ll want to use an API-level rate limit!** +- do you have a health endpoint that consumes very little resource on your service and can handle significantly more requests than your other endpoints? **You'll want to use an API-level per-endpoint rate limit!** +- do you want to limit the number of requests a specific user can make to **all APIs** they have access to? **You’ll want to use a key-level global rate limit!** +- do you want to limit the number of requests a specific user can make to **specific APIs** they have access to? **You’ll want to use a key-level per-API rate limit.** +- do you want to limit the number of requests a specific user can make to a **specific endpoint of an API** they have access to? **You’ll want to use a key-level per-endpoint rate limit.** + +### Applying multiple rate limits + +When multiple rate limits are configured, they are assessed in this order (if applied): + +1. API-level per-endpoint rate limit (configured in API definition) +2. API-level rate limit (configured in API definition) +3. Key-level per-endpoint rate limit (configured in access key) +4. Key-level per-API rate limit (configured in access key) +5. Key-level global rate limit (configured in access key) + +### Combining multiple policies configuring rate limits + +If more than one policy defining a rate limit is applied to a key then Tyk will apply the highest request rate permitted by any of the policies that defines a rate limit. + +If `rate` and `per` are configured in multiple policies applied to the same key then the Gateway will determine the effective rate limit configured for each policy and apply the highest to the key. + +Given, policy A with `rate` set to 90 and `per` set to 30 seconds (3rps) and policy B with `rate` set to 100 and `per` set to 10 seconds (10rps). If both are applied to a key, Tyk will take the rate limit from policy B as it results in a higher effective request rate (10rps). + + + +Prior to Tyk 5.4.0 there was a long-standing bug in the calculation of the effective rate limit applied to the key where Tyk would combine the highest `rate` and highest `per` from the policies applied to the key, so for the example above the key would have `rate` set to 100 and `per` set to 30 giving an effective rate limit of 3.33rps. This has now been corrected. + + + +## Rate limiting algorithms + +Different rate limiting algorithms are employed to cater to varying requirements, use cases and gateway deployments. A one-size-fits-all approach may not be suitable, as APIs can have diverse traffic patterns, resource constraints, and service level objectives. Some algorithms are more suited to protecting the upstream service from overload whilst others are suitable for per-client limiting to manage and control fair access to a shared resource. + +Tyk offers the following rate limiting algorithms: + +1. [Distributed Rate Limiter](#distributed-rate-limiter): recommended for most use cases, implements the [token bucket algorithm](https://en.wikipedia.org/wiki/Token_bucket) +2. [Redis Rate Limiter](#redis-rate-limiter): implements the [sliding window log algorithm](https://developer.redis.com/develop/dotnet/aspnetcore/rate-limiting/sliding-window) +3. [Fixed Window Rate Limiter](#fixed-window-rate-limiter): implements the [fixed window algorithm](https://redis.io/learn/develop/dotnet/aspnetcore/rate-limiting/fixed-window) + +When the rate limits are reached, Tyk will block requests with an [HTTP 429 (Rate Limit Exceeded)](https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429) response. + + + +Tyk supports selection of the rate limit algorithm at the Gateway level, so the same algorithm will be applied to all APIs. +It can be configured to switch dynamically between two algorithms depending on the request rate, as explained [here](#dynamic-algorithm-selection-based-on-request-rate). + + + +### Distributed Rate Limiter + +The Distributed Rate Limiter (DRL) is the default rate limiting mechanism in Tyk Gateway. It is +implemented using a token bucket implementation that does not use Redis. +In effect, it divides the configured rate limit between the number of +addressable gateway instances. + +The characteristics of DRL are: + +- a rate limit of 100 requests/min with 2 gateways yields 50 requests/min per gateway +- unreliable at low rate limits where requests are not fairly balanced + +DRL can face challenges in scenarios where traffic is not evenly +distributed across gateways, such as with sticky sessions or keepalive +connections. These conditions can lead to certain gateways becoming +overloaded while others remain underutilized, compromising the +effectiveness of configured rate limiting. This imbalance is particularly +problematic in smaller environments or when traffic inherently favors +certain gateways, leading to premature rate limits on some nodes and +excess capacity on others. + +DRL will be used automatically unless one of the other rate limit +algorithms are explicitly enabled via configuration. + +It's important to note that this algorithm will yield approximate results due to the nature of the local +rate limiting, where the total allowable request rate is split between the gateways; uneven distribution +of requests could lead to exhaustion of the limit on some gateways before others. + +### Redis Rate Limiter + +This algorithm implements a sliding window log algorithm and can be enabled via the [enable_redis_rolling_limiter](/tyk-oss-gateway/configuration#enable_redis_rolling_limiter) configuration option. + +The characteristics of the Redis Rate Limiter (RRL) are: + +- using Redis lets any gateway respect a cluster-wide rate limit (shared counter) +- a record of each request, including blocked requests that return `HTTP 429`, is written to the sliding log in Redis +- the log is constantly trimmed to the duration of the defined window +- requests are blocked if the count in the log exceeds the configured rate limit + +An important behavior of this rate limiting algorithm is that it blocks +access to the API when the rate exceeds the rate limit and does not let +further API calls through until the rate drops below the specified rate +limit. For example, if the configured rate limit is 3000 requests/minute the call rate would +have to be reduced below 3000 requests/minute for a whole minute before the `HTTP 429` +responses stop and traffic is resumed. This behavior is called **spike arrest**. + +The complete request log is stored in Redis so resource usage when using this rate limiter is high. +This algorithm will use significant resources on Redis even when blocking requests, as it must +maintain the request log, mostly impacting CPU usage. Redis resource +usage increases with traffic therefore shorter `per` values are recommended to +limit the amount of data being stored in Redis. + +If you wish to avoid spike arrest behavior but the DRL is not suitable, you might use the [Fixed Window Rate Limiter](#fixed-window-rate-limiter) algorithm. + +You can configure [Rate Limit Smoothing](#rate-limit-smoothing) to manage the traffic spike, allowing time to increase upstream capacity if required. + +The [Redis Sentinel Rate Limiter](#redis-sentinel-rate-limiter) reduces latency for clients, however increases resource usage on Redis and Tyk Gateway. + +#### Rate Limit Smoothing + +Rate Limit Smoothing is an optional mechanism of the RRL that dynamically adjusts the request +rate limit based on the current traffic patterns. It helps in managing +request spikes by gradually increasing or decreasing the rate limit +instead of making abrupt changes or blocking requests excessively. + +This mechanism uses the concept of an intermediate *current allowance* (rate limit) that moves between an initial lower +bound (`threshold`) and the maximum configured request rate (`rate`). As the request rate approaches the current +*current allowance*, Tyk will emit an event to notify you that smoothing has been triggered. When the event is emitted, +the *current allowance* will be increased by a defined increment (`step`). A hold-off counter (`delay`) must expire +before another event is emitted and the *current allowance* further increased. If the request rate exceeds the +*current allowance* then the rate limiter will block further requests, returning `HTTP 429` as usual. + +As the request rate falls following the spike, the *current allowance* will gradually reduce back to the lower bound (`threshold`). + +Events are emitted and adjustments made to the *current allowance* based on the following calculations: + +- when the request rate rises above `current allowance - (step * trigger)`, + a `RateLimitSmoothingUp` event is emitted and *current allowance* increases by `step`. +- when the request rate falls below `allowance - (step * (1 + trigger))`, + a `RateLimitSmoothingDown` event is emitted and *current allowance* decreases by `step`. + +##### Configuring rate limit smoothing + +When Redis Rate Limiter is in use, rate limit smoothing is configured with the following options within the `smoothing` object alongside the standard `rate` and `per` parameters: + +- `enabled` (boolean) to enable or disable rate limit smoothing +- `threshold` is the initial rate limit (*current allowance*) beyond which smoothing will be applied +- `step` is the increment by which the *current allowance* will be increased or decreased each time a smoothing event is emitted +- `trigger` is a fraction (typically in the range 0.1-1.0) of the `step` at which point a smoothing event will be emitted as the request rate approaches the *current allowance* +- `delay` is a hold-off between smoothing events and controls how frequently the current allowance will step up or down (in seconds). + +Rate Limit Smoothing is configured using the `smoothing` object within access keys and policies. For API-level rate limiting, this configuration is within the `access_rights[*].limit` object. + +An example configuration would be as follows: + +```yaml expandable + "smoothing": { + "enabled": true, + "threshold": 5, + "trigger": 0.5, + "step": 5, + "delay": 30 + } +``` + +#### Redis Sentinel Rate Limiter + +The Redis Sentinel Rate Limiter option will: + +- write a sentinel key into Redis when the request limit is reached +- use the sentinel key to block requests immediately for `per` duration +- requests, including blocked requests, are written to the sliding log in a background thread + +This optimizes the latency for connecting clients, as they don't have to +wait for the sliding log write to complete. This algorithm exhibits spike +arrest behavior the same as the basic Redis Rate Limiter, however recovery may take longer as the blocking is in +effect for a minimum of the configured window duration (`per`). Gateway and Redis +resource usage is increased with this option. + +This option can be enabled using the following configuration option +[enable_sentinel_rate_limiter](/tyk-oss-gateway/configuration#enable_sentinel_rate_limiter). + +To optimize performance, you may configure your rate limits with shorter +window duration values (`per`), as that will cause Redis to hold less +data at any given moment. + +Performance can be improved by enabling the [enable_non_transactional_rate_limiter](/tyk-oss-gateway/configuration#enable_non_transactional_rate_limiter). This leverages Redis Pipelining to enhance the performance of the Redis operations. Please consult the [Redis documentation](https://redis.io/docs/manual/pipelining/) for more information. + +Please consider the [Fixed Window Rate Limiter](#fixed-window-rate-limiter) algorithm as an alternative, if Redis performance is an issue. + +### Fixed Window Rate Limiter + +The Fixed Window Rate Limiter will limit the number of requests in a +particular window in time. Once the defined rate limit has been reached, +the requests will be blocked for the remainder of the configured window +duration. After the window expires, the counters restart and again allow +requests through. + +- the implementation uses a single counter value in Redis +- the counter expires after every configured window (`per`) duration. + +The implementation does not smooth out traffic bursts within a window. For any +given `rate` in a window, the requests are processed without delay, until +the rate limit is reached and requests are blocked for the remainder of the +window duration. + +When using this option, resource usage for rate limiting does not +increase with traffic. A simple counter with expiry is created for every +window and removed when the window elapses. Regardless of the traffic +received, Redis is not impacted in a negative way, resource usage remains +constant. + +This algorithm can be enabled using the following configuration option [enable_fixed_window_rate_limiter](/tyk-oss-gateway/configuration#enable_fixed_window_rate_limiter). + +If you need spike arrest behavior, the [Redis Rate Limiter](#redis-rate-limiter) should be used. + +### Dynamic algorithm selection based on request rate + +The Distributed Rate Limiter (DRL) works by distributing the +rate allowance equally among all gateways in the cluster. For example, +with a rate limit of 1000 requests per second and 5 gateways, each +gateway can handle 200 requests per second. This distribution allows for +high performance as gateways do not need to synchronize counters for each +request. + +DRL assumes an evenly load-balanced environment, which is typically +achieved at a larger scale with sufficient requests. In scenarios with +lower request rates, DRL may generate false positives for rate limits due +to uneven distribution by the load balancer. For instance, with a rate of +10 requests per second across 5 gateways, each gateway would handle only +2 requests per second, making equal distribution unlikely. + +It's possible to configure Tyk to switch automatically between the Distributed Rate Limiter +and the Redis Rate Limiter by setting the `drl_threshold` configuration. + +This threshold value is used to dynamically switch the rate-limiting +algorithm based on the volume of requests. This option sets a +minimum number of requests per gateway that triggers the Redis Rate +Limiter. For example, if `drl_threshold` is set to 2, and there are 5 +gateways, the DRL algorithm will be used if the rate limit exceeds 10 +requests per second. If it is 10 or fewer, the system will fall back to +the Redis Rate Limiter. + +See [DRL Threshold](/tyk-oss-gateway/configuration#drl_threshold) for details on how to configure this feature. + + +## Rate Limiting Layers + +You can protect your upstream services from being flooded with requests by configuring rate limiting in Tyk Gateway. Rate limits in Tyk are configured using two parameters: allow `rate` requests in any `per` time period (given in seconds). + +As explained in the [Rate Limiting Concepts](/api-management/rate-limit#introduction) section, Tyk supports configuration of rate limits at both the API-Level and Key-Level for different use cases. + +The API-Level rate limit takes precedence over Key-Level, if both are configured for a given API, since this is intended to protect your upstream service from becoming overloaded. The Key-Level rate limits provide more granular control for managing access by your API clients. + +### Configuring the rate limiter at the API-Level + +If you want to protect your service with an absolute limit on the rate of requests, you can configure an API-level rate limit. You can do this from the API Designer in Tyk Dashboard as follows: + +1. Navigate to the API for which you want to set the rate limit +2. From the **Core Settings** tab, navigate to the **Rate Limiting and Quotas** section +3. Ensure that **Disable rate limiting** is not selected +4. Enter in your **Rate** and **Per (seconds)** values +5. **Save/Update** your changes + +Tyk will now accept a maximum of **Rate** requests in any **Per** period to the API and will reject further requests with an `HTTP 429 Too Many Requests` error. + +Check out the following video to see this being done. + + + +### Configuring the rate limiter at the Key-Level + +If you want to restrict an API client to a certain rate of requests to your APIs, you can configure a Key-Level rate limit via a [Security Policy](/api-management/policies). The allowance that you configure in the policy will be consumed by any requests made to APIs using a key generated from the policy. Thus, if a policy grants access to three APIs with `rate=15 per=60` then a client using a key generated from that policy will be able to make a total of 15 requests - to any combination of those APIs - in any 60 second period before receiving the `HTTP 429 Too Many Requests` error. + + + +It is assumed that the APIs being protected with a rate limit are using the [auth token](/api-management/authentication/bearer-token) client authentication method and policies have already been created. + + + +You can configure this rate limit from the API Designer in Tyk Dashboard as follows: + +1. Navigate to the Tyk policy for which you want to set the rate limit +2. Ensure that API(s) that you want to apply rate limits to are selected +3. Under **Global Limits and Quota**, make sure that **Disable rate limiting** is not selected and enter your **Rate** and **Per (seconds)** values +4. **Save/Update** the policy + +### Setting up a Key-Level Per-API rate limit + +If you want to restrict API clients to a certain rate of requests for a specific API you will also configure the rate limiter via the security policy. However this time you'll assign per-API limits. The allowance that you configure in the policy will be consumed by any requests made to that specific API using a key generated from that policy. Thus, if a policy grants access to an API with `rate=5 per=60` then three clients using keys generated from that policy will each independently be able to make 5 requests in any 60 second period before receiving the `HTTP 429 Too Many Requests` error. + + + +It is assumed that the APIs being protected with a rate limit are using the [auth token](/api-management/authentication/bearer-token) client authentication method and policies have already been created. + + + +You can configure this rate limit from the API Designer in Tyk Dashboard as follows: + +1. Navigate to the Tyk policy for which you want to set the rate limit +2. Ensure that API that you want to apply rate limits to is selected +3. Under **API Access**, turn on **Set per API Limits and Quota** +4. You may be prompted with "Are you sure you want to disable partitioning for this policy?". Click **CONFIRM** to proceed +5. Under **Rate Limiting**, make sure that **Disable rate limiting** is not selected and enter your **Rate** and **Per (seconds)** values +6. **Save/Update** the policy + +Check out the following video to see this being done. + + + +### Setting up a key-level per-endpoint rate limit + +To restrict the request rate for specific API clients on particular endpoints, you can use the security policy to assign per-endpoint rate limits. These limits are set within the policy and will be #enforced for any requests made to that endpoint by clients using keys generated from that policy. + +Each key will have its own independent rate limit allowance. For example, if a policy grants access to an endpoint with a rate limit of 5 requests per 60 seconds, each client with a key from that policy can make 5 requests to the endpoint in any 60-second period. Once the limit is reached, the client will receive an HTTP `429 Too Many Requests` error. + +If no per-endpoint rate limit is defined, the endpoint will inherit the key-level per-API rate limit or the global rate limit, depending on what is configured. + + + +The following assumptions are made: + - The [ignore authentication](/api-management/traffic-transformation/ignore-authentication) middleware should not be enabled for the relevant endpoints. + - If [path-based permissions](/api-management/gateway-config-managing-classic#path-based-permissions) are configured, they must grant access to these endpoints for keys generated from the policies. + + + +You can configure per-endpoint rate limits from the API Designer in Tyk Dashboard as follows: + +1. Navigate to the Tyk policy for which you want to set the rate limit +2. Ensure that API that you want to apply rate limits to is selected +3. Under **API Access** -> **Set endpoint-level usage limits** click on **Add Rate Limit** to configure the rate limit. You will need to provide the rate limit and the endpoint path and method. +4. **Save/Update** the policy + + +### Setting Rate Limits in the Tyk Community Edition Gateway (CE) + +#### Configuring the rate limiter at the (Global) API-Level + +Using the `global_rate_limit` field in the API definition you can specify the API-level rate limit in the following format: `{"rate": 10, "per": 60}`. + +An equivalent example using Tyk Operator is given below: + +```yaml {linenos=table,hl_lines=["14-17"],linenostart=1} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-global-rate-limit +spec: + name: httpbin-global-rate-limit + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true + # setting a global rate-limit for the API of 10 requests per 60 seconds + global_rate_limit: + rate: 10 + per: 60 +``` + +### Configuring the rate limiter on the session object + +All actions on the session object must be done via the Gateway API. + +1. Ensure that `allowance` and `rate` are set to the same value: this should be number of requests to be allowed in a time period, so if you wanted 100 requests every second, set this value to 100. + +2. Ensure that `per` is set to the time limit. Again, as in the above example, if you wanted 100 requests per second, set this value to 1. If you wanted 100 requests per 5 seconds, set this value to 5. + +#### Can I disable the rate limiter? + +Yes, the rate limiter can be disabled for an API Definition by selecting **Disable Rate Limits** in the API Designer, or by setting the value of `disable_rate_limit` to `true` in your API definition. + +Alternatively, you could also set the values of `Rate` and `Per (Seconds)` to be 0 in the API Designer. + + + +Disabling the rate limiter at the API-Level does not disable rate limiting at the Key-Level. Tyk will enforce the Key-Level rate limit even if the API-Level limit is not set. + + + +#### Can I set rate limits by IP address? + +Not yet, though IP-based rate limiting is possible using custom pre-processor middleware JavaScript that generates tokens based on IP addresses. See our [Middleware Scripting Guide](/api-management/plugins/javascript#using-javascript-with-tyk) for more details. + +## Rate Limiting by API + +### Tyk Classic API Definition + +The per-endpoint rate limit middleware allows you to enforce rate limits on specific endpoints. This middleware is configured in the Tyk Classic API Definition, either via the Tyk Dashboard API or in the API Designer. + +To enable the middleware, add a new `rate_limit` object to the `extended_paths` section of your API definition. + +The `rate_limit` object has the following configuration: + +- `path`: the endpoint path +- `method`: the endpoint HTTP method +- `enabled`: boolean to enable or disable the rate limit +- `rate`: the maximum number of requests that will be permitted during the interval (window) +- `per`: the length of the interval (window) in seconds + +You can set different rate limits for various endpoints by specifying multiple `rate_limit` objects. + +#### Simple endpoint rate limit + +For example: + +```json {linenos=true, linenostart=1} expandable +{ + "use_extended_paths": true, + "extended_paths": { + "rate_limit": [ + { + "path": "/anything", + "method": "GET", + "enabled": true, + "rate": 60, + "per": 1 + } + ] + } +} +``` + +In this example, the rate limit middleware has been configured for HTTP +`GET` requests to the `/anything` endpoint, limiting requests to 60 per +second. + +#### Advanced endpoint rate limit + +For more complex scenarios, you can configure rate limits for multiple +paths. The order of evaluation matches the order defined in the +`rate_limit` array. For example, if you wanted to limit the rate of +`POST` requests to your API allowing a higher rate to one specific +endpoint you could configure the API definition as follows: + +```json {linenos=true, linenostart=1} expandable +{ + "use_extended_paths": true, + "extended_paths": { + "rate_limit": [ + { + "path": "/user/login", + "method": "POST", + "enabled": true, + "rate": 100, + "per": 1 + }, + { + "path": "/.*", + "method": "POST", + "enabled": true, + "rate": 60, + "per": 1 + } + ] + } +} +``` + +In this example, the first rule limits `POST` requests to `/user/login` +to 100 requests per second (rps). Any other `POST` request matching the +regex pattern `/.*` will be limited to 60 requests per second. The order +of evaluation ensures that the specific `/user/login` endpoint is matched +and evaluated before the regex pattern. + +The per-endpoint rate limit middleware allows you to enforce rate limits on specific endpoints. This middleware is configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation), either via the Tyk Dashboard API or in the API Designer. + +If you’re using the legacy Tyk Classic APIs, then check out the [Tyk Classic](/api-management/rate-limit#tyk-classic-api-definition) page. + +### Tyk OAS API Definition + +The design of the Tyk OAS API Definition takes advantage of the +`operationId` defined in the OpenAPI Document that declares both the path +and method for which the middleware should be added. Endpoint `paths` +entries (and the associated `operationId`) can contain wildcards in the +form of any string bracketed by curly braces, for example +`/status/{code}`. These wildcards are so they are human-readable and do +not translate to variable names. Under the hood, a wildcard translates to +the β€œmatch everything” regex of: `(.*)`. + +The rate limit middleware (`rateLimit`) can be added to the `operations` section of the +Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition +for the appropriate `operationId` (as configured in the `paths` section +of your OpenAPI Document). + +The `rateLimit` object has the following configuration: + +- `enabled`: enable the middleware for the endpoint +- `rate`: the maximum number of requests that will be permitted during the interval (window) +- `per`: the length of the interval (window) in time duration notation (e.g. 10s) + +#### Simple endpoint rate limit + +For example: + +```json {hl_lines=["39-43"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-rate-limit", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/status/200": { + "get": { + "operationId": "status/200get", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-rate-limit", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-rate-limit/", + "strip": true + } + }, + "middleware": { + "operations": { + "status/200get": { + "rateLimit": { + "enabled": true, + "rate": 60, + "per": "1s" + } + } + } + } + } +} +``` + +In this example, a rate limit has been configured for the `GET +/status/200` endpoint, limiting requests to 60 per second. + +The configuration above is a complete and valid Tyk OAS API Definition +that you can import into Tyk to try out the Per-endpoint Rate Limiter +middleware. + +#### Advanced endpoint rate limit + +For more complex scenarios, you can configure rate limits for multiple +paths. The order of evaluation matches the order that endpoints are +defined in the `paths` section of the OpenAPI description. For example, +if you wanted to limit the rate of `POST` requests to your API allowing a +higher rate to one specific endpoint you could configure the API +definition as follows: + +```json {hl_lines=["49-53", "56-60"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "advanced-rate-limit", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/user/login": { + "post": { + "operationId": "user/loginpost", + "responses": { + "200": { + "description": "" + } + } + } + }, + "/{any}": { + "post": { + "operationId": "anypost", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "advanced-rate-limit", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/advanced-rate-limit/", + "strip": true + } + }, + "middleware": { + "operations": { + "user/loginpost": { + "rateLimit": { + "enabled": true, + "rate": 100, + "per": "1s" + } + }, + "anypost": { + "rateLimit": { + "enabled": true, + "rate": 60, + "per": "1s" + } + } + } + } + } +} +``` + +In this example, the first rule limits requests to the `POST /user/login` +endpoint to 100 requests per second (rps). Any other `POST` request to an +endpoint path that matches the regex pattern `/{any}` will be limited to +60 rps. The order of evaluation ensures that the specific `POST +/user/login` endpoint is matched and evaluated before the regex pattern. + +The configuration above is a complete and valid Tyk OAS API Definition +that you can import into Tyk to try out the Per-endpoint Rate Limiter +middleware. + +### Configuring the middleware in the API Designer + +Configuring per-endpoint rate limits for your API endpoints is easy when using the API Designer in the Tyk Dashboard, simply follow these steps: + +1. **Add an endpoint** + + From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + + Tyk OAS API Designer showing no endpoints created + + Adding an endpoint to an API using the Tyk OAS API Designer + + Tyk OAS API Designer showing no middleware enabled on endpoint + +2. **Select the Rate Limit middleware** + + Select **ADD MIDDLEWARE** and choose **Rate Limit** from the *Add Middleware* screen. + + Adding the Rate Limit middleware + +3. **Configure the middleware** + + You must provide the path to the compiled plugin and the name of the Go function that should be invoked by Tyk Gateway when the middleware is triggered. + + Configuring the per-endpoint custom plugin + +4. **Save the API** + + Select **ADD MIDDLEWARE** to save the middleware configuration. Remember to select **SAVE API** to apply the changes. + +## Rate Limiting with Tyk Streams + +A rate limit is a strategy for limiting the usage of a shared resource across parallel components in a Tyk Streams instance, or potentially across multiple instances. They are configured as a resource: + +```yaml +rate_limit_resources: + - label: foobar + local: + count: 500 + interval: 1s +``` + +And most components that hit external services have a field `rate_limit` for specifying a rate limit resource to use. For example, if we wanted to use our `foobar` rate limit with a HTTP request: + +```yaml +input: + http_client: + url: TODO + verb: GET + rate_limit: foobar +``` + +By using a rate limit in this way we can guarantee that our input will only poll our HTTP source at the rate of 500 requests per second. + +{/* TODO: when rate-limit processor supported: +Some components don't have a `rate_limit` field but we might still wish to throttle them by a rate limit, in which case we can use the rate_limit processor that applies back pressure to a processing pipeline when the limit is reached. For example: + +```yaml expandable +input: + http_server: + path: /post +output: + http_server: + ws_path: /subscribe +pipeline: + processors: + - rate_limit: + resource: example_rate_limit +rate_limit_resources: + - label: example_rate_limit + local: + count: 3 + interval: 20s +``` */} + + +### Local + +The local rate limit is a simple X every Y type rate limit that can be shared across any number of components within the pipeline but does not support distributed rate limits across multiple running instances of Tyk Streams. + +```yml +# Config fields, showing default values +label: "" +local: + count: 1000 + interval: 1s +``` + +**Configuration Fields** + +**count** + +The maximum number of requests to allow for a given period of time. + + +Type: `int` +Default: `1000` + +**interval** + +The time window to limit requests by. + + +Type: `string` +Default: `"1s"` \ No newline at end of file diff --git a/api-management/request-quotas.mdx b/api-management/request-quotas.mdx new file mode 100644 index 00000000..f8ba8f07 --- /dev/null +++ b/api-management/request-quotas.mdx @@ -0,0 +1,772 @@ +--- +title: "Request Quotas" +'og:description': "Overview of Rate Quotas with the Tyk Gateway" +tags: ['Request Quotas', 'API Quotas', 'Usage Limits', 'Consumption Control'] +sidebarTitle: "Request Quotas" +--- + +## Introduction + +Request Quotas in Tyk Gateway allow you to set a maximum number of API requests for a specific API key or Policy over longer, defined periods (e.g., day, week, month). This feature is distinct from rate limiting (which controls requests per second), and it is essential for managing API consumption, enforcing service tiers, and protecting your backend services from sustained overuse over time. + +```mermaid +flowchart LR + Client[API Client] -->|Makes Request| Gateway[Tyk Gateway] + Gateway -->|Check Quota| Redis[(Redis)] + Redis -->|Quota OK| Gateway + Redis -->|Quota Exceeded| Gateway + Gateway -->|If Quota OK| Upstream[Upstream API] + Gateway -->|If Quota Exceeded| Reject[Reject Request] + Upstream -->|Response| Gateway + Gateway -->|Response| Client +``` + +### Key Benefits + +* **Enforce Usage Limits:** Cap the total number of requests allowed over extended periods (days, weeks, months) per consumer. +* **Implement Tiered Access:** Easily define different usage allowances for various subscription plans (e.g., Free, Basic, Pro). +* **Protect Backend Services:** Prevent individual consumers from overwhelming upstream services with consistently high volume over long durations. +* **Enable Usage-Based Monetization:** Provide a clear mechanism for charging based on consumption tiers. + +--- +## Quick Start + +### Overview + +In this tutorial, we will configure Request Quotas on a Tyk Security Policy to limit the number of requests an API key can make over a defined period. Unlike rate limits (requests per second), quotas control overall volume. We'll set a low quota limit with a short renewal period for easy testing, associate a key with the policy, observe blocked requests once the quota is exhausted, and verify that the quota resets after the period elapses. This guide primarily uses the Tyk Dashboard for configuration. + +### Prerequisites + +- **Working Tyk Environment:** You need access to a running Tyk instance that includes both the Tyk Gateway and Tyk Dashboard components. For setup instructions using Docker, please refer to the [Tyk Quick Start](https://github.com/TykTechnologies/tyk-pro-docker-demo?tab=readme-ov-file#quick-start). +- **Curl, Seq and Sleep**: These tools will be used for testing. + +### Instructions + +#### Create an API + +1. **Create an API:** + 1. Log in to your Tyk Dashboard. + 2. Navigate to **API Management > APIs** + 3. Click **Add New API** + 4. Click **Import** + 5. Select **Import Type** as **Tyk API** + 6. Copy the below Tyk OAS definition in the text box and click **Import API** to create an API. + + + + + ```json expandable + { + "components": { + "securitySchemes": { + "authToken": { + "in": "header", + "name": "Authorization", + "type": "apiKey" + } + } + }, + "info": { + "title": "Request Quota Test", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": {}, + "security": [ + { + "authToken": [] + } + ], + "servers": [ + { + "url": "http://tyk-gateway.localhost:8080/request-quota-test/" + } + ], + "x-tyk-api-gateway": { + "info": { + "name": "Request Quota Test", + "state": { + "active": true + } + }, + "middleware": { + "global": { + "contextVariables": { + "enabled": true + }, + "trafficLogs": { + "enabled": true + } + } + }, + "server": { + "authentication": { + "enabled": true, + "securitySchemes": { + "authToken": { + "enabled": true + } + } + }, + "listenPath": { + "strip": true, + "value": "/request-quota-test/" + } + }, + "upstream": { + "url": "http://httpbin.org/" + } + } + } + ``` + + + + +

Configure Policy and Quota

+2. **Create and Configure a Security Policy with a Request Quota:** + + + + + 1. Navigate to **API Security > Policies** in the Tyk Dashboard sidebar. + 2. Click the **Add Policy** button. + 3. Under the **1. Access Rights** tab, in the **Add API Access Rule** section, select the `Request Quota Test` API. + 4. Scroll down to the **Global Limits and Quota** section (still under the **1. Access Rights** tab): + * **Important:** Disable **Rate Limiting** by selecting **Disable rate limiting** option, so it doesn't interfere with testing the quota. + * Set the following values for `Usage Quotas`: + * Uncheck the `Unlimited requests` checkbox + * Enter `10` into the **Max Requests per period** field. (This is our low quota limit for testing). + * Select `1 hour` from the **Quota resets every:** dropdown. (In the next step, we will modify it to 60 seconds via API for quick testing, as 1 hour is a very long period. From the dashboard, we can only select pre-configured options.) + 5. Select the **2. Configuration** tab. + 6. In the **Policy Name** field, enter `Request Quota Policy`. + 7. From the **Key expire after** dropdown, select `1 hour`. + 8. Click the **Create Policy** button. + + + + + policy with request quota configured + +4. **Update Quota Reset Period via API:** + + As the Dashboard UI doesn't allow setting a shorter duration, we will set the Quota reset period to a value of 1 minute for testing purposes. The following commands search for the policy, modify its `quota_renewal_rate` to 60 seconds, and update the API. + + **Note:** Obtain your Dashboard API key by clicking on the User profile at the top right corner, then click on `Edit Profile`, and select the key available under `Tyk Dashboard API Access Credentials`. Now in the below command replace `` with the API key you obtained from the Dashboard UI. + + ``` + curl -s --location 'http://localhost:3000/api/portal/policies/search?q=Request%20Quota%20Policy' \ + -H "Authorization: " \ + -H "Accept: application/json" > policy.json + + jq '.Data[0] | .quota_renewal_rate = 60' policy.json > updated_policy.json + jq -r '.Data[0]._id' policy.json > policy_id.txt + + curl --location "http://localhost:3000/api/portal/policies/$(cat policy_id.txt)" \ + -H "Authorization: " \ + -H "Content-Type: application/json" \ + -H "Accept: application/json" \ + -X PUT \ + -d @updated_policy.json + ``` + +3. **Associate an Access Key with the Policy:** + + + + + 1. Navigate to **API Security > Keys** in the Tyk Dashboard sidebar. + 2. Click the **Add Key** button. + 3. Under the **1. Access Rights** tab: + * In the **Apply Policy** section, select the `Request Quota Policy`. + 4. Select the **2. Configuration** tab. + 5. In the **Alias** field, enter `Request Quota Key`. This provides a human-readable identifier. + 6. From the **Expires** dropdown, select `1 hour`. + 7. Click the **Create Key** button. + 8. A pop-up window **"Key created successfully"** will appear displaying the key details. **Copy the Key ID** value shown and save it securely. You will need this key to make API requests in the following steps. + 9. Click **OK** to close the pop-up. + + + + +#### Testing + +4. **Test Quota Exhaustion:** + + We've set a quota of 10 requests per 60 seconds. Let's send more than 10 requests within that window to observe the quota being enforced. + + 1. Open your terminal. + 2. Execute the following command, replacing `` with the API Key ID you saved earlier. This command attempts to send 15 requests sequentially. + + ```bash + for i in $(seq 1 15); do \ + echo -n "Request $i: "; \ + curl -s -o /dev/null -w "%{http_code}\n" -H "Authorization: " http://tyk-gateway.localhost:8080/request-quota-test/get; \ + sleep 0.1; \ + done + ``` + + *(Note: The `sleep 0.1` adds a tiny delay, but ensure all 15 requests execute well within the 60-second quota window).* + + 3. **Expected Observation:** You should see the first 10 requests succeed, returning an HTTP status code `200`. After the 10th request, the subsequent requests (11 through 15) should be blocked by the quota limit, returning an HTTP status code `403` (Forbidden). + + **Sample Output:** + + ```bash + Request 1: 200 + Request 2: 200 + Request 3: 200 + Request 4: 200 + Request 5: 200 + Request 6: 200 + Request 7: 200 + Request 8: 200 + Request 9: 200 + Request 10: 200 + Request 11: 403 + Request 12: 403 + Request 13: 403 + Request 14: 403 + Request 15: 403 + ``` + +5. **Test Quota Reset:** + + Now, let's wait for the quota period (60 seconds) to elapse and then send another request to verify that the quota allowance has been reset. + + 1. Wait slightly longer than the reset period in the same terminal. The command below waits for 70 seconds. + + ```bash + echo "Waiting for quota to reset (70 seconds)..." + sleep 70 + echo "Wait complete. Sending one more request..." + ``` + + 2. Send one more request using the same API key: + + ```bash + curl -s -o /dev/null -w "%{http_code}\n" -H "Authorization: " http://tyk-gateway.localhost:8080/request-quota-test/get + ``` + + 3. **Expected Observation:** This request should now succeed, returning an HTTP status code `200`. This demonstrates that because the 60-second quota period ended, the *next* request made after that period triggered the quota reset, replenishing the allowance. + + **Sample Output:** + + ```bash + Waiting for quota to reset (70 seconds)... + Wait complete. Sending one more request... + 200 + ``` + +This quick start demonstrates the fundamental behaviour of Request Quotas: they limit the total number of requests allowed within a specific period and automatically reset the allowance once that period renews (triggered by the next request). + +--- +## Configuration Options + +Request Quotas in Tyk can be configured at various levels. + +The configuration involves setting two specific fields: + +1. **QuotaMax**: The maximum number of requests allowed during the quota period. + - Set to `-1` for unlimited requests + - Set to a positive integer (e.g., `1000`) to limit total requests + +2. **QuotaRenewalRate**: The time in seconds for which the quota applies. + - Example: `3600` for the hourly quota (1 hour = 3600 seconds) + - Example: `86400` for the daily quota (24 hours = 86400 seconds) + - Example: `2592000` for the monthly quota (30 days = 2592000 seconds) + +### System-Level Configuration + +Global quota settings are configured in the Tyk Gateway configuration file (`tyk.conf`). These settings affect how quotas are enforced across the entire gateway. + + + + +```json +{ +// Partial config from tyk.conf + "enforce_org_quotas": true, + "enforce_org_data_detail_logging": false, + "monitor": { + "enable_trigger_monitors": true, + "global_trigger_limit": 80.0, + "monitor_user_keys": true, + "monitor_org_keys": true + }, +// ... more config follows +} +``` expandable + +- `enforce_org_quotas`: When set to `true`, enables organization-level quota enforcement +- `monitor.enable_trigger_monitors`: Enables quota monitoring and webhook triggers +- `monitor.global_trigger_limit`: Percentage of quota usage that triggers alerts (e.g., 80.0 means 80%) +- `monitor.monitor_user_keys`: Enables monitoring for individual API keys +- `monitor.monitor_org_keys`: Enables monitoring for organization quotas + + + +```bash +export TYK_GW_ENFORCEORGQUOTAS=true +``` expandable + + + +Refer to the [Tyk Gateway Configuration Reference](/tyk-oss-gateway/configuration#enforce_org_quotas) for more details on this setting. + +{/* Why we are commenting org quotas: Organization quotas are a hangover from the Classic Cloud, where all clients shared a deployment. They are not documented anywhere presently, and I’m not sure why we would start to do so - but if we’re going to, we need to be very careful not to add complexity to the way users configure things. */} + +{/* ### Organization-Level Configuration + +Organization quotas limit the total number of requests across all APIs for a specific organization. These are enforced by the `OrganizationMonitor` middleware when `enforce_org_quotas` is enabled. + +- `quota_max`: Maximum number of requests allowed during the quota period +- `quota_renewal_rate`: Time in seconds for the quota period (e.g., 3600 for hourly quotas) + +Organization quotas are configured through the Tyk Dashboard API or Gateway API: + +```bash +curl -X POST -H "Authorization: {your-api-key}" \ + -H "Content-Type: application/json" \ + -d '{ + "quota_max": 1000, + "quota_renewal_rate": 3600, + }' \ + http://tyk-gateway:8080/tyk/org/keys/{org-id} +``` */} expandable + +### API-Level Configuration + +You **cannot set** quota values within an API Definition, but you can **disable** quota checking entirely for all requests proxied through that specific API, regardless of Key or Policy settings. This is useful if an API should never have quota limits applied. + + + + + +In a Tyk OAS API Definition, you can globally disable quotas for specific APIs: + +- **skipQuota**: When set to true, disables quota enforcement for the API. +- **skipQuotaReset**: When set to true, prevents quota counters from being reset when creating or updating quotas. + +```json +{ + // Partial config from Tyk OAS API Definition + "middleware": { + "global": { + "skipQuota": true, + "skipQuotaReset": false + } + }, + // ... more config follows +} +``` + +Refer to the [Tyk OAS API Definition reference](/api-management/gateway-config-tyk-oas#global) for details. + + + + +In a Tyk Classic API Definition (JSON), set the `disable_quota` field to `true`. + +```json +{ + // Partial config from Tyk Classic API Definition + "disable_quota": true // Set to true to disable quota checks + // ... more config follows +} + +``` expandable + +Refer to the [Tyk Classic API Definition reference](/api-management/gateway-config-tyk-classic) for details. + + + + + + + +### Configure via UI + +The Tyk Dashboard provides a straightforward interface to set request quota parameters on Security Policies and Access Keys. + + + + + +The image below shows a policy with request quotas. Any key using this policy will inherit the quota settings and behave as follows: each key will be permitted 1000 requests per 24-hour (86400 seconds) cycle before the quota resets. + +policy with request quota configured + +
+ +1. Navigate to **API Security > Policies** in the Tyk Dashboard sidebar +2. Click the **Add Policy** button +3. Under the **1. Access Rights** tab and in the **Add API Access Rule** section, select the required API +4. Scroll down to the **Global Limits and Quota** section (still under the **1. Access Rights** tab): + * Enable `Request Quotas` by setting the following values in the `Usage Quotas` section: + * Uncheck the `Unlimited Requests` checkbox + * Field **Requests (or connection attempts) per period** - Enter the total number of requests a client can use during the defined quota period. + * Field **Quota resets every:** - Select the duration of the quota period. +5. Select the **2. Configuration** tab +6. In the **Policy Name** field, enter a name +7. From the **Key expire after** dropdown, select an option +8. Click the **Create Policy** button + + + + + + +The image below shows an access key with request quotas. This access key behaves as follows: each key will be permitted 1000 requests per 24-hour (86400 seconds) cycle before the quota resets. + +**Note:** Direct key configuration overrides policy settings only for that specific key. + +policy with request quota configured + +
+ +1. Navigate to **API Security > Keys** in the Tyk Dashboard sidebar +2. Click the **Create Key** button +3. Under the **1. Access Rights** tab: + * Select **Choose API** + * In the **Add API Access Rule** section, select the required API +4. Scroll down to the **Global Limits and Quota** section (still under the **1. Access Rights** tab): + * Enable `Request Quotas` by setting the following values in the `Usage Quotas` section: + * Uncheck the `Unlimited Requests` checkbox + * Field **Requests (or connection attempts) per period** - Enter the total number of requests a client can use during the defined quota period. + * Field **Quota resets every:** - Select the duration of the quota period. +5. Select the **2. Configuration** tab +6. In the **Alias** field, enter a name. This human-readable identifier makes tracking and managing this specific access key easier in your analytics and logs. +7. From the **Expires** dropdown, select an option +8. Click the **Create Key** button + + + + + + +### Configure via API + +These are the fields that you can set directly in the Policy object or the Access Key: + +```json +{ + // Partial policy/session object fields + "quota_max": 1000, // Allow one thousand requests + "quota_renewal_rate": 86400, // 1 day or 24 hours + // ... more config follows +} +``` expandable + + + + + +To update the policy, do the following: +1. Retrieve the policy object using `GET /api/portal/policies/{POLICY_ID}` +2. Add or modify the `quota_max` and `quota_renewal_rate` fields within the policy JSON object +3. Update the policy using `PUT /api/portal/policies/{POLICY_ID}` with the modified object, or create a new one using `POST /api/portal/policies/` + +**Explanation:** +The above adds request quotas to a policy. Any key using this policy will inherit the quotas settings and behaves as follows: each key will be permitted 1000 requests per 24-hour (86400 seconds) cycle before the quota resets. + + + + + +**Note:** Direct key configuration overrides policy settings only for that specific key. + +To update the access key do the following: +1. Retrieve the key's session object using `GET /api/keys/{KEY_ID}` +2. Add or modify the `quota_max` and `quota_renewal_rate` fields within the session object JSON +3. Update the key using `PUT /api/keys/{KEY_ID}` with the modified session object + +**Explanation:** +The above adds quotas to an access key. Any request made by the key will behave as follows: each key will be permitted 1000 requests per 24-hour (86400 seconds) cycle before the quota resets. + + + + + +### Important Considerations + +* **Policy Precedence:** Quotas set on a Security Policy apply to all keys using that policy *unless* overridden by a specific quota set directly on the key (using the "Set per API Limits and Quota" option). +* **Unlimited Quota:** Setting `quota_max` to `-1` grants unlimited requests for the quota period. +* **Event-Driven Resets:** Quotas reset *after* the `quota_renewal_rate` (in seconds) has passed *and* upon the next request using the key. They do not reset automatically on a fixed schedule (e.g., precisely at midnight or the 1st of the month) unless external automation updates the session object. +* **Response Headers:** When quotas are active, Tyk typically adds `X-RateLimit-Limit`, `X-RateLimit-Remaining`, and `X-RateLimit-Reset` headers to responses, allowing clients to track their usage. (Note: Header names might be configurable). + +--- +## How It Works + +Request Quotas in Tyk limit a client's total number of API requests within a defined period (hours, days, months). Unlike rate limits that control the frequency of requests over short intervals (like seconds or minutes) to prevent immediate system overload, Request Quotas control the total volume of requests allowed over much longer periods to manage overall consumption and align with service tiers. + +When clients reach their quota limit, further requests are rejected until the quota period renews. It helps API providers implement usage-based pricing tiers, prevent API abuse, control infrastructure costs, and ensure fair resource distribution among clients. + +Think of Request Quotas as a prepaid phone plan with a fixed number of minutes per month. When you sign up, you get allocated a specific number of call minutes (API requests) that you can use over the billing period. You can make calls (API requests) at any pace you want – all at once or spread throughout the month – but once you've used up your allocated minutes, you can't make any more calls until the next billing cycle begins. + +```mermaid +flowchart LR + Client[API Client] -->|Makes Request| Gateway[Tyk Gateway] + Gateway -->|Check Quota| Redis[(Redis)] + Redis -->|Quota OK| Gateway + Redis -->|Quota Exceeded| Gateway + Gateway -->|If Quota OK| Upstream[Upstream API] + Gateway -->|If Quota Exceeded| Reject[Reject Request] + Upstream -->|Response| Gateway + Gateway -->|Response| Client +``` + +### How Tyk Implements Quotas + +Tyk implements request quotas using a Redis-based counter mechanism with time-based expiration. Here's a detailed breakdown of the implementation: + +```mermaid +graph LR + A[API Request Received] --> B(Check Redis Quota Counter); + B -- Counter < QuotaMax --> C{Increment Redis Counter}; + C --> D[Calculate quota_remaining = QuotaMax - Counter]; + D --> E[Update Session State]; + E --> F[Forward Request to Upstream]; + B -- Counter >= QuotaMax --> G[Reject Request with 403]; +``` expandable + +#### Core Components + +1. **Redis Storage**: Quotas are tracked in Redis using incrementing counters for each API key. The TTL is set to the quota renewal period, and the counter is reset to 0 on the next request after expiration. + + Here is a sample Redis key for a Request Quota: + ``` + quota-[scope]-[key_hash] + ``` + + Where: + - `scope` is optional and represents an API-specific allowance scope + - `key_hash` is the hashed API key (if hash keys are enabled) + +2. **Session State**: Quota configuration is stored in the user's `SessionState`, which contains several quota-related fields: + + - `QuotaMax`: Maximum number of requests allowed during the quota period. + - `QuotaRemaining`: Number of requests remaining for the current period. **Note:** This is a derived value, not the primary counter. + - `QuotaRenews`: Unix timestamp when the quota will reset. + - `QuotaRenewalRate`: Time in seconds for the quota period (e.g., 3600 for hourly quotas). + +3. **Middleware**: The quota check is performed by the `RateLimitAndQuotaCheck` middleware + +#### Quota Enforcement + +The core logic for checking and enforcing Request Quotas is executed within the `RateLimitAndQuotaCheck` middleware, which is a step in the request processing pipeline. Here's a breakdown of this process: + +1. **Initiation:** As a request enters the Tyk Gateway, it passes through configured middleware. The quota validation process begins when it hits the `RateLimitAndQuotaCheck` middleware. + +2. **Applicability Check:** The middleware first determines if quota enforcement is relevant: + * It checks the API Definition to see if quotas are globally disabled. If so, the process stops here for quotas and the request proceeds. + * It identifies the API key for the request and retrieves its associated `SessionState`. + +3. **Retrieve Limits:** The middleware accesses the `SessionState` to get the specific quota parameters applicable to this key and potentially the specific API being accessed (if per-API quotas are configured): + * `QuotaMax`: The maximum number of requests allowed. + * `QuotaRenewalRate`: The duration (in seconds) of the quota period for setting the TTL in Redis. + +4. **Redis Interaction & Enforcement:** This is the core enforcement step, interacting directly with Redis: + * **Construct Key:** Generates the unique Redis key for tracking this specific quota counter (e.g., `quota-{scope}-{api-key-hash}`). + * **Check Expiry/Existence:** It checks Redis to see if the key exists and if its TTL is still valid. + * **Handle Renewal (If Expired/Missing):** If the key doesn't exist or its TTL has passed, Tyk initiates the renewal logic described previously (attempting a distributed lock, setting the counter to 0, and applying the `QuotaRenewalRate` as the new TTL). + * **Increment Counter:** Tyk atomically increments the Redis counter value. This operation returns the *new* value of the counter *after* the increment. + * **Compare Against Limit:** The middleware compares this *new* counter value against the `QuotaMax` retrieved from the session state. + * **Decision:** + * If `new_counter_value <= QuotaMax`: The request is within the allowed quota. + * If `new_counter_value > QuotaMax`: This request has exceeded the quota limit. + +5. **Outcome:** + * **Quota OK:** The middleware allows the request to proceed to the next stage in the processing pipeline (e.g., other middleware, upstream service). + * **Quota Exceeded:** The middleware halts further request processing down the standard pipeline. It prepares and returns an error response to the client, typically `HTTP 403 Forbidden` with a "Quota exceeded" message. + +6. **Session State Update:** Regardless of whether the request was allowed or blocked due to the quota, the middleware calls an internal function (like `updateSessionQuota`) to update the in-memory `SessionState` associated with the API key. This update synchronizes the `QuotaRemaining` field in the session with the latest calculated state based on the Redis counter and its expiry. It ensures that subsequent operations within the same request lifecycle (if any) or diagnostic information have access to the most recent quota status. + +#### Quota Reset Mechanisms + +The available allowance (`QuotaRemaining`) for an API key is replenished back to its maximum (`QuotaMax`) through several distinct mechanisms: + +1. **Event-Driven Renewal (Primary Mechanism):** + * **Condition:** This occurs *after* the time duration specified by `QuotaRenewalRate` (in seconds) has elapsed since the quota period began (i.e., since the last reset or key creation/update). In Redis, this corresponds to the Time-To-Live (TTL) expiring on the quota tracking key. + * **Trigger:** The reset is **not** automatic based on a timer. It is triggered by the **next API request** made using that specific key *after* the `QuotaRenewalRate` duration has passed (and the Redis TTL has expired). + * **Process:** Upon detecting the expired TTL during that next request, Tyk resets the Redis counter (typically by setting it to 0 and immediately incrementing it to 1 for the current request) and applies a *new* TTL based on the `QuotaRenewalRate`. This effectively makes the full `QuotaMax` available for the new period starting from that moment. + + ```mermaid + graph LR + A[Request After Quota Period] --> B{Redis Key Expired?}; + B -- Yes --> C[Reset Counter to 0]; + C --> D[Set New Expiration]; + D --> E[Process Request Normally]; + B -- No --> F[Continue Normal Processing]; + ``` + +2. **Manual Reset via API:** + * **Mechanism:** You can force an immediate quota reset for a specific API key by calling an endpoint on the Tyk Gateway Admin API. + * **Effect:** This action directly deletes the corresponding quota tracking key in Redis. The *next* request using this API key will find no existing key, triggering the renewal logic (Step 1) as if the period had just expired, immediately granting the full `QuotaMax` and setting a new TTL. This provides an immediate, on-demand refresh of the quota allowance. + +3. **Key Creation or Update:** + * **Trigger:** When a new API key is created or an existing key's configuration is updated (e.g., via the Dashboard or the Gateway API), Tyk reapplies the quota settings based on the current policy or key-specific configuration. + * **Process:** This typically involves setting the `QuotaRemaining` value to `QuotaMax` in the key's session data and ensuring the corresponding Redis key is created with the correct initial value (or implicitly reset) and its TTL set according to the `QuotaRenewalRate`. This ensures the key starts with a fresh quota allowance according to its defined limits. + * **Exception:** This behavior can be suppressed if the API definition includes the `DontSetQuotasOnCreate` field (referred to as `SkipQuotaReset` in the OAS specification), which prevents automatic quota resets during key creation or updates. + +#### Key Technical Aspects + +1. **Time-Based Reset**: Unlike rate limiting, which uses sliding windows, quotas have a fixed renewal time determined by `QuotaRenewalRate` (in seconds) + +2. **Atomic Operations**: Redis pipelining is used to ensure atomic increment and expiration setting: + +3. **Race Condition Handling**: Distributed locks prevent multiple servers from simultaneously resetting quotas + +4. **Quota Scope Support**: The implementation supports both global quotas and API-specific quotas through the scoping mechanism + +--- +## FAQs + + + +Request Quotas in Tyk limit the total number of API requests a client can make within a specific time period. Unlike rate limits (which control requests per second), quotas control the total number of requests over longer periods like hours, days, or months. Once a quota is exhausted, further requests are rejected until the quota is renewed. + + + +While both control API usage, they serve different purposes: +- **Rate Limits** control the frequency of requests (e.g., 10 requests per second) to prevent traffic spikes and ensure consistent performance +- **Request Quotas** control the total volume of requests over a longer period (e.g., 10,000 requests per month) to manage overall API consumption and often align with business/pricing models + + + +Refer this [documentation](#configuration-options). + + + +The main parameters for configuring quotas are: +- `quota_max`: Maximum number of requests allowed during the quota period +- `quota_remaining`: Number of requests remaining for the current period +- `quota_renewal_rate`: Time in seconds during which the quota is valid (e.g., 3600 for hourly quotas) +- `quota_renews`: Timestamp indicating when the quota will reset + + + +You can disable quotas for specific APIs by setting the `disable_quota` flag to `true` in the API definition. This config will bypass quota checking for all requests to that API, regardless of any quotas set at the key or policy level. + +Refer this [documentation](#api-level-configuration). + + + +When a quota is exceeded: +1. The request is rejected with a 403 Forbidden status code +2. A "QuotaExceeded" event is triggered (which can be used for notifications or monitoring) +3. The client must wait until the quota renewal period before making additional requests +4. The quota violation is logged and can be monitored in the Tyk Dashboard + + + +Tyk stores quota information in Redis: +- Quota keys are prefixed with "quota-" followed by the key identifier +- For each request, Tyk increments a counter in Redis and checks if it exceeds the quota_max +- When a quota period expires, the counter is reset +- For distributed deployments, quota information is synchronized across all Tyk nodes + + + +Yes, you can implement per-endpoint quotas using policies enabling the "per_api" partitioning. This config allows you to define different quota limits for API endpoints, giving you fine-grained control over resource usage. + + + +Organization quotas allow you to limit the total number of requests across all keys belonging to an organization. When enabled (using `enforce_org_quotas`), Tyk tracks the combined usage of all keys in the organization and rejects requests when the organization quota is exceeded, regardless of individual key quotas. + + + +Yes, Tyk provides quota monitoring capabilities: +- You can set up trigger monitors with percentage thresholds +- When usage reaches a threshold (e.g., 80% of quota), Tyk can trigger notifications +- These notifications can be sent via webhooks to external systems +- The monitoring configuration is set in the `monitor` section of your Tyk configuration + + + +Tyk's quota renewal is event-driven rather than time-driven. Quotas don't automatically reset at specific times (like midnight); instead, they reset when the first request is made after the renewal period has passed. If no requests are made after renewal, the quota counter remains unchanged until the next request triggers the check and renewal process. + + + +You can manually reset a quota for a specific key in two ways: + +**Via Tyk Dashboard:** +1. Navigate to the "Keys" section +2. Find and select the key you want to reset +3. Click on "Reset Quota" button + +**Via Tyk Gateway API:** +``` +POST /tyk/keys/reset/{key_id} +Authorization: {your-gateway-secret} +``` +This endpoint will immediately reset the quota for the specified key, allowing the key to make requests up to its quota limit again. + + + +Yes, by default, Tyk counts all requests against the quota regardless of the response status code (2xx, 4xx, 5xx). This means that even failed requests with error responses will decrement the available quota. This behavior is designed to prevent abuse and ensure consistent quota enforcement regardless of the upstream API's response. + + + +In multi-datacenter or multi-region setups, quota inconsistencies can occur due to: + +1. **Redis replication lag**: If you're using separate Redis instances with replication, there may be delays in syncing quota information +2. **Network latency**: In geographically distributed setups, network delays can cause temporary inconsistencies +3. **Configuration issues**: Each gateway must be properly configured to use the same Redis database for quota storage + +To resolve this, ensure all gateways are configured to use the same Redis database or a properly configured Redis cluster with minimal replication lag. Consider using Redis Enterprise or a similar solution with cross-region synchronization capabilities for multi-region deployments. + + + +In some older versions of Tyk, setting `quota_max` to -1 (to disable quotas) would generate an error log message: `Quota disabled: quota max <= 0`. This was a known issue that has been fixed in more recent versions. + +If you're still seeing these logs, consider: +1. Upgrading to the latest version of Tyk +2. Adjusting your log level to reduce noise +3. Using the API definition's `disable_quota` flag instead of setting `quota_max` to -1 + +This log message is informational and doesn't indicate a functional problem with your API. + + + +By default, Tyk counts all requests against the quota regardless of the response code. There is no built-in configuration to count only successful (2xx) responses toward quota limits. + +If you need this functionality, you have two options: +1. Implement a custom middleware plugin that conditionally decrements the quota based on response codes +2. Use the Tyk Pump to track successful vs. failed requests separately in your analytics platform and implement quota management at the application level + + + +If you modify a quota configuration mid-period (before the renewal time): + +1. For **increasing** the quota: The new maximum will apply, but the current remaining count stays the same +2. For **decreasing** the quota: If the new quota is less than what's already been used, further requests will be rejected +3. For **changing the renewal rate**: The new renewal period will apply from the next renewal + +Changes to quota settings take effect immediately, but don't reset the current usage counter. Use the "Reset Quota" functionality to apply new settings and reset the counter immediately. + + + +Yes, Tyk provides several ways to implement different quota plans: + +1. **Policies**: Create different policies with varying quota limits and assign them to keys based on subscription level +2. **Key-specific settings**: Override policy quotas for individual keys when necessary +3. **Meta Data**: Use key metadata to adjust quota behavior through middleware dynamically +4. **Multiple APIs**: Create separate API definitions with different quota configurations for different service tiers + +This flexibility allows you to implement complex quota schemes that align with your business model and customer tiers. + + + +When troubleshooting quota issues: + +1. **Check Redis**: Ensure Redis is functioning properly and examine the quota keys directly +2. **Review logs**: Look for quota-related messages in the Tyk Gateway logs +3. **Verify configuration**: Confirm that quota settings are correctly configured in policies and API definitions +4. **Test with the API**: Use the Tyk Gateway API to check quota status for specific keys +5. **Monitor request headers**: Examine the `X-Rate-Limit-Remaining` headers in API responses + +For multi-gateway setups, verify that all gateways use the same Redis instance and that there are no synchronization issues between Redis clusters. + + + diff --git a/api-management/request-throttling.mdx b/api-management/request-throttling.mdx new file mode 100644 index 00000000..67383a09 --- /dev/null +++ b/api-management/request-throttling.mdx @@ -0,0 +1,436 @@ +--- +title: "Request Throttling" +'og:description': "What is Request Throttling in Tyk Gateway?" +tags: ['Request Throttling'] +sidebarTitle: "Request Throttling" +--- + +## Introduction + +Tyk's Request Throttling feature provides a mechanism to manage traffic spikes by queuing and automatically retrying client requests that exceed [rate limits](/api-management/rate-limit), rather than immediately rejecting them. This helps protect upstream services from sudden bursts and improves the resilience of API interactions during temporary congestion. + +--- +## Quick Start + +### Overview + +In this tutorial, we will configure Request Throttling on a Tyk Security Policy to protect a backend service from sudden traffic spikes. We'll start by defining a basic rate limit on a policy, then enable throttling with specific retry settings to handle bursts exceeding that limit, associate a key with the policy, and finally test the behaviour using simulated traffic. This guide primarily uses the Tyk Dashboard for configuration. + +### Prerequisites + +- **Working Tyk Environment:** You need access to a running Tyk instance that includes both the Tyk Gateway and Tyk Dashboard components. For setup instructions using Docker, please refer to the [Tyk Quick Start](https://github.com/TykTechnologies/tyk-pro-docker-demo?tab=readme-ov-file#quick-start). +- **Curl, seq and xargs**: These tools will be used for testing. + +### Instructions + +#### Create an API + +1. **Create an API:** + 1. Log in to your Tyk Dashboard. + 2. Navigate to **API Management > APIs** + 3. Click **Add New API** + 4. Click **Import** + 5. Select **Import Type** as **Tyk API** + 6. Copy the below Tyk OAS definition in the text box and click **Import API** to create an API + + + + + ```json expandable + { + "components": { + "securitySchemes": { + "authToken": { + "in": "header", + "name": "Authorization", + "type": "apiKey" + } + } + }, + "info": { + "title": "Request Throttling Test", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": {}, + "security": [ + { + "authToken": [] + } + ], + "servers": [ + { + "url": "http://tyk-gateway.localhost:8080/request-throttling-test/" + } + ], + "x-tyk-api-gateway": { + "info": { + "name": "Request Throttling Test", + "state": { + "active": true + } + }, + "middleware": { + "global": { + "contextVariables": { + "enabled": true + }, + "trafficLogs": { + "enabled": true + } + } + }, + "server": { + "authentication": { + "enabled": true, + "securitySchemes": { + "authToken": { + "enabled": true + } + } + }, + "listenPath": { + "strip": true, + "value": "/request-throttling-test/" + } + }, + "upstream": { + "url": "http://httpbin.org/" + } + } + } + ``` + + + + +

Configure Policy and Rate Limit

+2. **Create and Configure an Security Policy with Rate Limiting:** + + + + + 1. Navigate to **API Security > Policies** in the Tyk Dashboard sidebar + 2. Click the **Add Policy** button + 3. Under the **1. Access Rights** tab, in the **Add API Access Rule** section, select the `Request Throttling Test` API + 4. Scroll down to the **Global Limits and Quota** section (still under the **1. Access Rights** tab): + * Set the following values for `Rate Limiting` + * Enter `5` into the **Requests (or connection attempts)** field + * Enter `10` into the **Per (seconds):** field + 5. Select the **2. Configuration** tab + 6. In the **Policy Name** field, enter `Request Throttling Policy` + 7. From the **Key expire after** dropdown, select `1 hour` + 8. Click the **Create Policy** button + + + + + policy with throttling configured + +3. **Associate an Access Key with the Policy:** + + + + + 1. Navigate to **API Security > Keys** in the Tyk Dashboard sidebar + 2. Click the **Add Key** button + 3. Under the **1. Access Rights** tab: + * In the **Apply Policy** section, select the `Request Throttling Policy` API + 5. Select the **2. Configuration** tab + 6. In the **Alias** field, enter `Request Throttling Key`. This provides a human-readable identifier that makes tracking and managing this specific access key easier in your analytics and logs. + 7. From the **Expires** dropdown, select `1 hour` + 8. Click the **Create Key** button + 9. A pop-up window **"Key created successfully"** will appear displaying the key details. **Copy the Key ID** value shown and save it securely. You will need this key to make API requests in the following steps + 10. Click **OK** to close the pop-up + + + + +4. **Test Rate Limit** + + So far, we've created a policy for an API definition and created a key that complies with that policy. Before enabling throttling, let's observe the standard rate limiting behaviour. We'll send 10 requests in parallel using `xargs` to simulate a burst that exceeds our configured limit (5 requests per 10 seconds). + + 1. Open your terminal. + 2. Execute the following command, replacing `` with the API Key ID you saved earlier: + + ```bash + seq 10 | xargs -n1 -P10 -I {} bash -c 'curl -s -I -H "Authorization: " http://tyk-gateway.localhost:8080/request-throttling-test/ | head -n 1' + ``` + + 3. **Expected Observation:** You should see some requests succeed with `HTTP/1.1 200 OK`, and other requests failing with `HTTP/1.1 429 Too Many Requests` as the rate limit is immediately enforced. The order of `200s` vs `429s` might vary depending upon the processing time, but you will see immediate rejections once the limit is hit. + + **Sample Output (Illustrative):** + + ```bash + HTTP/1.1 429 Too Many Requests + HTTP/1.1 429 Too Many Requests + HTTP/1.1 429 Too Many Requests + HTTP/1.1 429 Too Many Requests + HTTP/1.1 429 Too Many Requests + HTTP/1.1 200 OK + HTTP/1.1 200 OK + HTTP/1.1 200 OK + HTTP/1.1 200 OK + HTTP/1.1 200 OK + ``` + +#### Configure Throttling + +Now that the policy enforces a basic rate limit, we will enable and configure Request Throttling. This adds the queue-and-retry behavior for requests that exceed the limit, preventing immediate rejection and helping to smooth out traffic spikes. + +5. **Configure Request Throttling by Updating the Security Policy** + + 1. Navigate to **API Security > Policies** in the Tyk Dashboard sidebar + 2. Click on the `Request Throttling Policy` + 3. Under the **1. Access Rights** tab: + * In the **Global Limits and Quota** section + * Set the following values for `Throttling` + * Uncheck the `Disable Throttling` checkbox + * Enter `3` into the **Throttle retries (or connection attempts)** field + * Enter `5` into the **Per (seconds):** field + 4. Click the **Update** button + 5. A pop-up window will appear to confirm the changes. Click **Update** to close the pop-up + +#### Testing + +6. **Test Request Throttling** + + 1. **Repeat the Test:** Open your terminal and execute the *exact same command* as in step 4: + + ```bash + seq 10 | xargs -n1 -P10 -I {} bash -c 'curl -s -I -H "Authorization: " http://tyk-gateway.localhost:8080/request-throttling-test/ | head -n 1' + ``` + + 2. **Expected Observation:** + * You will still see the first ~5 requests return `HTTP/1.1 200 OK` quickly + * Critically, the subsequent requests (6 through 10) will **not** immediately return `429`. Instead, you should observe a **delay** before their status lines appear + * After the delay (`throttle_interval`), Tyk will retry the queued requests. Some might now succeed (return `200 OK`) if the rate limit window allows + * If a request is retried `throttle_retry_limit` (3) times and still encounters the rate limit, *then* it will finally return `HTTP/1.1 429 Too Many Requests` + * Overall, you might see more `200 OK` responses compared to the previous test, and any `429` responses will appear significantly later + + **Sample Output (Illustrative - timing is key):** + + ```bash + HTTP/1.1 200 OK # Appears quickly + HTTP/1.1 200 OK # Appears quickly + HTTP/1.1 200 OK # Appears quickly + HTTP/1.1 200 OK # Appears quickly + HTTP/1.1 200 OK # Appears quickly + # --- Noticeable pause here --- + HTTP/1.1 200 OK + # --- Noticeable pause here --- + HTTP/1.1 200 OK + # --- Noticeable pause here --- + HTTP/1.1 200 OK + HTTP/1.1 200 OK + HTTP/1.1 200 OK + ``` + *(The exact mix of 200s and 429s on the delayed requests depends heavily on timing relative to the 10-second rate limit window reset and the retry attempts).* + +This comparison clearly shows how Request Throttling changes the behaviour from immediate rejection to queued retries, smoothing the traffic flow and potentially allowing more requests to succeed during bursts. + +--- +## Configuration Options + +Request Throttling is configured within Tyk [Security Policies](/api-management/policies) or directly on individual [Access Keys](/api-management/authentication/bearer-token). + +The configuration involves setting two specific fields: + +- `throttle_interval`: Defines the wait time (in seconds) between retry attempts for a queued request. (*Note*: Do not set it to `0`. If you do, no delay is applied, and the request is immediately retried. This will creates a β€œbusy waiting” scenario that consumes more resources than a positive interval value) +- `throttle_retry_limit`: Sets the maximum number of retry attempts before the request is rejected. (*Note*: Do not set it to `0`. Setting it to `0` means that there will be no throttling on the request) + +To enable throttling, both fields must be set to a value greater than `0`. + +### Disable throttling + +The default value is `-1` and means it is disabled by default. +Setting `throttle_interval` and `throttle_retry_limit` values to any number smaller than `0`, to ensure the feature is diabled. + +You can configure these settings using either the Tyk Dashboard UI or the Tyk Dashboard API. + +### Configure via UI + +The Tyk Dashboard provides a straightforward interface to set throttling parameters on both Security Policies and Access Keys. + + + + + +The image below shows a policy with throttling. Any key using this policy will inherit the throttling settings and behaves as follows: wait 2 seconds between retries for queued requests, attempting up to 3 times before failing (so overall 6 seconds before getting another 429 error response). + +policy with throttling configured + +
+ +1. Navigate to **API Security > Policies** in the Tyk Dashboard sidebar +2. Click the **Add Policy** button +3. Under the **1. Access Rights** tab and in the **Add API Access Rule** section, select the required API +4. Scroll down to the **Global Limits and Quota** section (still under the **1. Access Rights** tab): + * To enable *Throttling*, we must configure *Rate Limiting* in the policy. + * Field **Requests (or connection attempts)** - Enter the number of requests you want to allow before rate limit is applied. + * Field **Per (seconds):** - Enter the time window in seconds during which the number of requests specified above is allowed. + * Now enable `Throttling` by setting the following values in the `Throttling` section: + * Uncheck the `Disable Throttling` checkbox + * Field **Throttle retries (or connection attempts)** - Enter the maximum number of times Tyk should attempt to retry a request after it has been queued due to exceeding a rate limit or quota. + * Field **Per (seconds):** - Enter the time interval in seconds Tyk should wait between each retry attempt for a queued request. +5. Select the **2. Configuration** tab +6. In the **Policy Name** field, enter a name +7. From the **Key expire after** dropdown, select an option +8. Click the **Create Policy** button + + + + + + +The image below shows an access key with throttling. This access key behaves as follows: wait 2 seconds between retries for queued requests, attempting up to 3 times before failing (so overall 6 seconds before getting another 429 error response). + +**Note:** Direct key configuration overrides policy settings only for that specific key. + +access key with throttling configured + +
+ +1. Navigate to **API Security > Keys** in the Tyk Dashboard sidebar +2. Click the **Create Key** button +3. Under the **1. Access Rights** tab: + * Select **Choose API** + * In the **Add API Access Rule** section, select the required API +4. Scroll down to the **Global Limits and Quota** section (still under the **1. Access Rights** tab): + * To enable *Throttling*, we must configure *Rate Limiting* in the Access Key. + * Field **Requests (or connection attempts)** - Enter the number of requests you want to allow before rate limit is applied. + * Field **Per (seconds):** - Enter the time window in seconds during which the number of requests specified above is allowed. + * Now enable `Throttling` by setting the following values in the `Throttling` section: + * Uncheck the `Disable Throttling` checkbox + * Field **Throttle retries (or connection attempts)** - Enter the maximum number of times Tyk should attempt to retry a request after it has been queued due to exceeding a rate limit or quota. + * Field **Per (seconds):** - Enter the time interval in seconds Tyk should wait between each retry attempt for a queued request. +5. Select the **2. Configuration** tab +6. In the **Alias** field, enter a name. This provides a human-readable identifier that makes tracking and managing this specific access key easier in your analytics and logs. +7. From the **Expires** dropdown, select an option +8. Click the **Create Key** button + + + + + + +### Configure via API + +These are the fields that you can set directly in the Policy object or the Access Key: + +```json +{ + // Partial policy/session object fields + "throttle_interval": 2, // Wait 2 second between retries + "throttle_retry_limit": 3, // Attempt a maximum of 3 retries + // ... more config follows +} +``` expandable + + + + + +To update the policy, do the following: +1. Retrieve the policy object using `GET /api/portal/policies/{POLICY_ID}` +2. Add or modify the `throttle_interval` and `throttle_retry_limit` fields within the policy JSON object +3. Update the policy using `PUT /api/portal/policies/{POLICY_ID}` with the modified object, or create a new one using `POST /api/portal/policies/` + +**Explanation:** +The above adds throttling to a policy. Any key using this policy will inherit the throttling settings and behaves as follows: wait 1 second between retries for queued requests, attempting up to 5 times before failing (so overall 5 seconds before getting another 429 error response). + + + + + +Note: Direct key configuration overrides policy settings only for that specific key. + +To update the access key do the following: +1. Retrieve the key's session object using `GET /api/keys/{KEY_ID}` +2. Add or modify the `throttle_interval` and `throttle_retry_limit` fields within the session object JSON +3. Update the key using `PUT /api/keys/{KEY_ID}` with the modified session object + + +**Explanation:** +The above adds throttling to a key. Any request made by the key will behave as follows: wait 1 second between retries for queued requests, attempting up to 5 times before failing (so overall 5 seconds before getting another 429 error response). + + + + + +--- +## How It Works + +```mermaid +flowchart LR + A[Client Request] --> GW(Tyk Gateway); + + subgraph Rate Limits + GW --> RL{Rate Limit OK?}; + RL -- Yes --> Q{Quota OK?}; + RL -- No --> T{Throttle Enabled?}; + Q -- Yes --> Fwd[Forward Request]; + Q -- No --> Reject[Reject Request]; + end + + subgraph Throttling Logic + T -- No --> Reject; + T -- Yes --> Queue[Queue Request]; + Queue --> Wait[Wait ThrottleInterval]; + Wait --> RetryL{Retry Limit Reached?}; + RetryL -- Yes --> Reject; + RetryL -- No --> Recheck(Re-evaluate Rate Limit Only); + %% Loop back to rate limit check only %% + Recheck --> RL; + end + + Fwd --> Backend((Upstream Service)); + Backend --> Success((Success Response)); + Success --> Client; + Reject --> Failure((Failure Response)); + Failure --> Client; +``` + +Tyk's Request Throttling intercepts API requests *after* they have exceeded a configured [Rate Limit](/api-management/rate-limit). + +Instead of immediately rejecting these requests with a `429 Too Many Requests` error (which is the default rate-limiting behaviour), the Gateway temporarily holds them in a queue. After waiting for a specified duration (`throttle_interval`), Tyk attempts to process the request again, re-checking the rate limit status. + +This retry cycle repeats until either the request can be successfully processed (if capacity becomes available) or a configured maximum number of retries (`throttle_retry_limit`) is reached. Only after exhausting all retries does Tyk return the `429` error to the client. + +Think of it like trying to access a service with a restriction on how many people can enter per minute (Rate Limit). If you arrive when the per-minute limit is full, standard behaviour is to turn you awa +y immediately. With Throttling enabled, the service instead asks you to wait briefly (the interval) and tries your entry again shortly, checking if the rate limit has freed up capacity, repeating this a f +ew times (the retry limit) before finally turning you away if access is still restricted. + +--- +## FAQ + + + + +Request Throttling in Tyk is a mechanism that allows for graceful handling of rate limit violations. Instead of immediately rejecting requests that exceed rate limits, throttling gives clients a chance to retry after a specified delay. + + + +Rate Limiting is a mechanism to restrict the number of requests a client can make in a given time period (e.g., 100 requests per minute). Request Throttling is an extension of rate limiting that provides a retry mechanism when rate limits are exceeded. Instead of immediately failing with a 429 status code, throttling allows the gateway to wait and retry the request internally. + + + +No, Request Throttling in Tyk is exclusively linked to rate limits and does not work with request quotas. When a quota is exceeded, the request is immediately rejected without any throttling or retry attempts. Throttling is only applied when rate limits are exceeded. + + + +Refer to this [documentation](#configuration-options). + + + +Request Throttling can increase response times for requests that exceed rate limits, as the gateway will wait for the specified `ThrottleInterval` between retry attempts. The maximum additional latency would be `ThrottleInterval Γ— ThrottleRetryLimit` seconds. This trade-off provides better success rates at the cost of potentially longer response times for some requests. + + + +Yes, Tyk tracks throttled requests in its health check metrics. You can monitor the `ThrottledRequestsPS` (throttled requests per second) metric to see how often requests are being throttled. Additionally, when a request is throttled, Tyk emits a `RateLimitExceeded` event that can be captured in your monitoring system. + + + +No, Request Throttling is not enabled by default. To enable throttling, you need to explicitly set `ThrottleRetryLimit` to a value greater than 0 and configure an appropriate `ThrottleInterval`. These settings can be applied through policies or directly in access keys. + + \ No newline at end of file diff --git a/api-management/response-caching.mdx b/api-management/response-caching.mdx new file mode 100644 index 00000000..caddb36c --- /dev/null +++ b/api-management/response-caching.mdx @@ -0,0 +1,754 @@ +--- +title: "Caching Responses" +'og:description': "How to manage users, teams, permissions, rbac in Tyk Dashboard" +tags: ['Caching', 'Request Optimization', 'Optimization', 'Endpoint Caching', 'Configuration', 'Cache'] +sidebarTitle: "Response Caching" +--- + +## Overview + +The Tyk Gateway can cache responses from your upstream services. + +API Clients which make subsequent requests to a cached endpoint will receive the cached response directly from the Gateway, which: + - reduces load on the upstream service + - provides a quicker response to the API Client (reduces latency) + - reduces concurrent load on the API Gateway + +Caching is best used on endpoints where responses infrequently change and are computationally expensive for the upstream service to generate. + +### Caching with Tyk + +Tyk uses Redis to store the cached responses and, as you'd expect from Tyk, there is lots of flexibility in how you configure caching so that you can optimize the performance of your system. + +There are two approaches to configure caching for an API deployed with Tyk: + + - [Basic](/api-management/response-caching#basic-caching) or [Safe Request](/api-management/response-caching#global-cache-safe-requests) caching is applied at the API level for all requests for which it is safe to do so. + - [Advanced](/api-management/response-caching#endpoint-caching) caching options can be applied at the endpoint level. + +Tyk's advanced caching options allow you to selectively cache the responses to all requests, only those from specific paths or only responses with specific status codes returned by the API. You can even cache dynamically based upon instruction from the upstream service received within the response. + +Caching is enabled by default at the Gateway level, but no caching will happen until the API Definition is configured to do so. + +### Cache Terminology and Features + +#### Cache Key +Cache keys are used to differentiate cached responses, such that slight variations in the request can generate different cache keys. This enables you to configure the cache so that different API Clients receive different cached responses when accessing the same API endpoint. + +This makes for a very granular cache, which may result in duplication of cached responses. This is preferable to the cache not being granular enough and therefore rendering it unsuitable for use, such that two API Clients receive the same cached response when this is not desired. + +The cache key is calculated using many factors: + - request HTTP method + - request URL (API path/endpoint) + - keys and values of any headers specified by `cache_by_headers` property + - hash of the request body + - API Id of the requested API + - value of the authorization header, if present, or if not, the API Client IP address + +#### Cache Value +The value stored in the cache is a base64 encoded string of the response body. When a subsequent request matches the cache key (a **cache hit**), Tyk decodes the cache value and returns this to the API Client that made the request. + +#### Indicating a Cached Response +When a request causes a cache hit, the Gateway will add a special header to indicate that the response being received is from a cache: + - `X-Tyk-Cached-Response` is added to the response header with the value `1` + +The API Client can use this to identify cached responses from non-cached responses. + +#### Global Cache (Safe Requests) +We define a safe request as any category of API request that is considered cacheable without causing any undesired side effects or security concerns. These are requests made using the HTTP methods `GET`, `HEAD` or `OPTIONS` that do not modify data and can be safely cached for performance gains (i.e. they should be idempotent and so are good candidates for caching). If these methods are not idempotent for your API, then you should not use safe request caching. + +Safe request caching at the API level is enabled by setting the `cache_all_safe_requests` option to `true`, or by checking the equivalent checkbox in the Dashboard UI. This will enable safe request caching on all endpoints for an API. + +This mode of operation is referred to as Global Caching because it is applied globally within the scope of a single API. Picking this approach will override any per-endpoint (per-path) caching configuration, so it’s not suitable if granular control is required. + +Tyk does support safe request caching at the more granular, per-endpoint level, as described [here](/api-management/response-caching#request-selective-cache-control) - but `cache_all_safe_requests` must be set to `false` in that scenario. + +#### Cache Timeout +The cache timeout (Time-To-Live or TTL) value can be configured per API and is the maximum age for which Tyk will consider a cache entry to be valid. You should use this to optimize the tradeoff between reducing calls to your upstream service and potential for changes to the upstream data. + +If the timeout has been exceeded when a request is made to a cached API, that request will be passed to the upstream and the response will (if appropriate) be used to refresh the cache. + +The timeout is configured in seconds. + +#### Cache Response Codes +You can configure Tyk to cache only responses with certain HTTP status codes (e.g. 200 OK), for example to save caching error responses. You can configure multiple status codes that will be cached for an API, but note that this applies only to APIs that return with an HTTP status code in the response. + +#### Dynamic Caching +By default Tyk maintains its response cache with a separate entry for each combination of API key (if authorization is enabled), request method and request path. Dynamic caching is a more flexible method of caching API responses based on header or body content rather than just the request method and path. This allows for more granular caching control and maintainance of separate caches for different users or request properties. + +#### Upstream Cache Control +Upstream cache control refers to caching API responses based on instructions provided by the upstream service within the response headers. This allows the upstream service to have more control over which responses are cached and for how long. + +## Basic Caching + +_On this page we describe the use of Tyk's API response cache at the API level (Global); for details on the more advanced Endpoint level cache you should refer to [this](/api-management/response-caching#endpoint-caching) page._ + +Caching is configured separately for each API according to values you set within the API definition. Subsequently, the caching scope is restricted to an API definition, rather than being applied across the portfolio of APIs deployed in the Gateway. + +If you are using the Tyk Dashboard you can set these options from the Dashboard UI, otherwise you will need to edit the raw API definition. + +### Configuring Tyk's API-level cache +Within the API Definition, the cache controls are grouped within the `cache_options` section. + +The main configuration options are: + - `enable_cache`: Set to `true` to enable caching for the API + - `cache_timeout`: Number of seconds to cache a response for, after which the next new response will be cached + - `cache_response_codes`: The HTTP status codes a response must have in order to be cached + - `cache_all_safe_requests`: Set to `true` to apply the caching rules to all requests using `GET`, `HEAD` and `OPTIONS` HTTP methods + +For more advanced use of the API-level cache we also have: + - `cache_by_headers`: used to create multiple cache entries based on the value of a [header value](#selective-caching-by-header-value) of your choice + - `enable_upstream_cache`: used to allow your [upstream service](/api-management/response-caching#upstream-cache-control-1) to identify the responses to be cached + - `cache_control_ttl_headers`: used with `enable_upstream_cache` + +#### An example of basic caching +To enable global caching for all safe requests to an API, only storing HTTP 200 responses, with a 10 second time-to-live (TTL), you would set: +``` expandable +"cache_options": { + "enable_cache": true, + "cache_timeout": 10, + "cache_all_safe_requests": true, + "cache_response_codes": [200] +} +``` + + + +If you set `cache_all_safe_requests` to true, then the cache will be global and *all* inbound requests made to the API will be evaluated by the caching middleware. This is great for simple APIs, but for most, a finer-grained control is required. This control will over-ride any per-endpoint cache configuration. + + + +#### Selective caching by header value +To create a separate cache entry for each response that has a different value in a specific HTTP header you would configure the `cache_option.cache_by_headers` option with a list of the headers to be cached. + +For example, to cache each value in the custom `Unique-User-Id` header of your API response separately you would set: +``` + "cache_options": { + "cache_by_headers": ["Unique-User-Id"] +} +``` + + + +The `cache_by_headers` configuration is not currently exposed in the Dashboard UI, so it must be enabled though either the raw API editor or the Dashboard API. + + + +### Configuring the Cache via the Dashboard +Follow these simple steps to enable and configure basic API caching via the Dashboard. + +**Steps for Configuration:** + +1. **Go to the Advanced Options** + + From the API Designer, select the **Advanced Options** tab: + + Advanced options tab location + +2. **Set the Cache Options for the Global Cache** + + Cache settings + + Here you must set: + + 1. **Enable caching** to enable the cache middleware + 2. **Cache timeout** to set the [TTL](/api-management/response-caching#cache-timeout) (in seconds) for cached requests + 3. **Cache only these status codes** to set which [response codes](/api-management/response-caching#cache-response-codes) to cache (ensure that you click **ADD** after entering each response code so that it is added to the list) + 4. **Cache all safe requests** to enable the [global cache](/api-management/response-caching#global-cache-safe-requests) + +## Endpoint Caching + +### Overview + +On this page we describe how to configure Tyk's API response cache per endpoint within an API. This gives granular control over which paths are cached and allows you to vary cache configuration across API versions. For details on the API level (Global) cache you should refer to the [global-cache](/api-management/response-caching#basic-caching) configuration page. + +When you use the API-level cache, Tyk will maintain a cache entry for each combination of request method, request path (endpoint) and API key (if authentication is enabled) for an API. The Endpoint Caching middleware gives you granular control over which paths are cached and allows you to vary cache configuration across API versions. + +For details on the API-level cache you should refer to the [API-level cache](/api-management/response-caching#basic-caching) configuration page. + +#### When to use the Endpoint Caching middleware + +##### API with multiple endpoints +When your API has more than one endpoint the upstream data could have different degrees of freshness, for example the data returned by one endpoint might refresh only once every five minutes (and so should be suitably cached) whilst another might give real-time data and so should not be cached. The endpoint cache allows you to optimize the caching of each endpoint to meet your requirements. + +##### Request based caching +If you have an API that's providing search capability (for example into a catalog of products) and want to optimize the performance for the most frequently requested search terms, you could use the endpoint cache's [request-selective](#request-selective-cache-control) capability to cache only a subset of all requests to an endpoint. + +#### How the endpoint cache works +If caching is enabled then, by default, Tyk will create separate cache entries for every endpoint (path) of your API. This may be unnecessary for your particular API, so Tyk provides a facility to cache only specific endpoint(s). + +The endpoint-level cache relies upon the API-level cache being enabled but then allows you to enable the middleware for the specific endpoints that you wish to cache. No other endpoint requests will be cached. + +For each endpoint in your API with endpoint caching middleware enabled, you can configure which response codes should be cached (for example, you might not want to cache error responses) and also the refresh interval - or timeout - for the cache entries. + + + +It's important to note that the [cache all safe requests](/api-management/response-caching#global-cache-safe-requests) feature of the API-level cache will overrule the per-endpoint configuration so you must ensure that both are not enabled for the same API. + + + +##### Request-selective cache control +For ultimate control over what Tyk caches, you can optionally configure the endpoint cache middleware to look for specific content in the request body. Tyk will then create a separate cache entry for each response where the request matches the specific combination of method, path and body content. + +You define a regex pattern and, if Tyk finds a match for this anywhere in the request body, the response will be cached. + +
+ +If you're using Tyk OAS APIs, then you can find details and examples of how to configure the Endpoint Caching middleware [here](/api-management/response-caching#using-tyk-oas-api). + +If you're using Tyk Classic APIs, then you can find details and examples of how to configure the Endpoint Caching middleware [here](/api-management/response-caching#using-classic-api). + +{/* proposed "summary box" to be shown graphically on each middleware page + ## Internal Endpoint middleware summary + - The Endpoint Cache middleware is an optional stage in Tyk's API Request processing chain, sitting between the [TBC]() and [TBC]() middleware. + - The Endpoint Cache middleware can be configured at the per-endpoint level within the API Definition and is supported by the API Designer within the Tyk Dashboard. */} + + +### Using Tyk OAS API + +The [Endpoint Caching](/api-management/response-caching#endpoint-caching) middleware allows you to perform selective caching for specific endpoints rather than for the entire API, giving you granular control over which paths are cached. + +When working with Tyk OAS APIs the middleware is configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation). You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the legacy Tyk Classic APIs, then check out the [Tyk Classic](/api-management/response-caching#using-classic-api) page. + +#### Configuring the middleware in the Tyk OAS API Definition + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. The `path` can contain wildcards in the form of any string bracketed by curly braces, for example `{user_id}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +**Configuring the endpoint cache is performed in two parts:** + +1. **Enable Tyk's caching function** + + The caching function is enabled by adding the `cache` object to the `global` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API. + + This object has the following configuration: + - `enabled`: enable the cache for the API + - `timeout`: set as the default cache refresh period for any endpoints for which you don't want to configure individual timeouts (in seconds) + +2. **Enable and configure the middleware for the specific endpoint** + + The endpoint caching middleware (`cache`) should then be added to the `operations` section of `x-tyk-api-gateway` for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + + The `cache` object has the following configuration: + - `enabled`: enable the middleware for the endpoint + - `timeout`: set to the refresh period for the cache (in seconds) + - `cacheResponseCodes`: HTTP responses codes to be cached (for example `200`) + - `cacheByRegex`: Pattern match for [selective caching by body value](/api-management/response-caching#request-selective-cache-control) + + For example: + ```json {hl_lines=["37-40", "45-51"],linenos=true, linenostart=1} expandable + { + "components": {}, + "info": { + "title": "example-endpoint-cache", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/delay/5": { + "post": { + "operationId": "delay/5post", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-endpoint-cache", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-endpoint-cache/", + "strip": true + } + }, + "global": { + "cache": { + "enabled": true, + "timeout": 60 + } + }, + "middleware": { + "operations": { + "delay/5post": { + "cache": { + "enabled": true, + "cacheResponseCodes": [ + 200 + ], + "timeout": 5 + } + } + } + } + } + } + ``` + + In this example the endpoint cache middleware has been configured to cache `HTTP 200` responses to requests to the `POST /delay/5` endpoint. The cache will refresh after 5 seconds. Note that requests to other endpoints will also be cached, with a default cache timeout of 60 seconds according to the configuration in lines 37-40. + + The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the endpoint caching. + +#### Configuring the middleware in the API Designer + +Adding endpoint caching to your API endpoints is easy when using the API Designer in the Tyk Dashboard, simply follow these steps: + +1. **Add an endpoint** + + From the **API Designer** add an endpoint to which you want to apply the middleware. + + Tyk OAS API Designer showing no endpoints created + + Adding an endpoint to an API using the Tyk OAS API Designer + + Tyk OAS API Designer showing no middleware enabled on endpoint + +2. **Select the Endpoint Cache middleware** + + Select **ADD MIDDLEWARE** and choose the **Cache** middleware from the *Add Middleware* screen. + + Adding the Endpoint Cache middleware + +3. **Configure the middleware** + + Set the timeout and HTTP response codes for the endpoint. You can remove a response code from the list by clicking on the `x` next to it. + + Configuring the endpoint cache middleware for a Tyk OAS API + + + + + Body value match or [request selective](/api-management/response-caching#request-selective-cache-control) caching is not currently exposed in the Dashboard UI, so it must be enabled though either the raw API editor or the Dashboard API. + + + + Select **UPDATE MIDDLEWARE** to apply the change to the middleware configuration. + +4. **Save the API** + + Select **SAVE API** to apply the changes to your API. + +### Using Classic API + +The [Endpoint Caching](/api-management/response-caching#endpoint-caching) middleware allows you to perform selective caching for specific endpoints rather than for the entire API, giving you granular control over which paths are cached. + +When working with Tyk Classic APIs the middleware is configured in the Tyk Classic API Definition. You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the newer Tyk OAS APIs, then check out the [Tyk OAS](/api-management/response-caching#using-tyk-oas-api) page. + +If using Tyk Operator please refer to section [configuring the middleware in the Tyk Operator](#tyk-operator). + +#### Configuring the middleware in the Tyk Classic API Definition + +When using the Tyk Classic API Definition, there are two options for endpoint caching - simple and advanced. + +The [simple](#simple-endpoint-cache) option works with the API-level cache and allows you to select which endpoints are cached, but relies upon the cache timeout (refresh) configured at the API-level. It will cache all responses received from the endpoint regardless of the HTTP response code for all [safe requests](/api-management/response-caching#global-cache-safe-requests). + +The [advanced](#advanced-endpoint-cache) option allows you to cache more selectively, giving control over the HTTP response codes to be cached, a per-endpoint cache timeout and also the possibility of caching responses only to requests containing specific data in the request body. + +##### Simple endpoint cache + +To enable the simple middleware you must add a new `cache` object to the `extended_paths` section of your API definition. The `cache` object is a list of endpoints for which you wish to cache all safe requests. + +In the API-level `cache_options` you must enable caching and configure the timeout whilst ensuring that the option to cache all safe requests is disabled. + +The `cache_options` object has the following configuration: +- `enable_cache`: set to `true` to enable caching for this API +- `cache_all_safe_requests`: set to `false` to allow selective caching per-endpoint +- `cache_timeout`: set to the refresh period for the cache (in seconds) + +For example: +```json {linenos=true, linenostart=1} +{ + "cache_options": { + "enable_cache": true, + "cache_timeout": 60, + "cache_all_safe_requests": false + }, + + "extended_paths": { + "cache": [ + { + "/widget", + "/fish" + } + ] + } +} +``` expandable + +In this example, the endpoint caching middleware has been configured to cache all safe requests to two endpoints (`/widget` and `/fish`) with a cache refresh period of 60 seconds. + +
Advanced endpoint cache
+For ultimate control over what Tyk caches, you should use the advanced configuration options for the per-endpoint cache. You can separately configure, for each HTTP method for an endpoint: +- an individual cache refresh (timeout) +- a list of HTTP response codes that should be cached +- a pattern match to cache only requests containing specific data in the [request body](/api-management/response-caching#request-selective-cache-control) + +To enable the advanced middleware you must add a new `advance_cache_config` object to the `extended_paths` section of your API definition. + +In the API-level `cache_options` you must enable caching and ensure that the option to cache all safe requests is disabled. The timeout that you set here will be used as a default for any endpoints for which you don't want to configure individual timeouts. + +The `advance_cache_config` object has the following configuration: +- `path`: the endpoint path +- `method`: the endpoint method +- `timeout`: set to the refresh period for the cache (in seconds) +- `cache_response_codes`: HTTP response codes to be cached (for example `200`) +- `cache_key_regex`: pattern match for selective caching by body value + +For example: +```json {linenos=true, linenostart=1} +{ + "cache_options": { + "enable_cache": true, + "cache_timeout": 60, + "cache_all_safe_requests": false + }, + + "extended_paths": { + "advance_cache_config": [ + { + "disabled": false, + "method": "POST", + "path": "/widget", + "cache_key_regex": "", + "cache_response_codes": [ + 200 + ], + "timeout": 10 + }, + { + "disabled": false, + "method": "GET", + "path": "/fish", + "cache_key_regex": "^shark$", + "cache_response_codes": [ + 200, 300 + ], + "timeout": 0 + } + ] + } +} +``` expandable + +In this example the endpoint caching middleware has been configured to cache requests to two endpoints (`/widget` and `/fish`) as follows: + +| endpoint | HTTP response codes to cache | cache refresh timeout | body value regex | +| :---------- | :------------------------------ | :----------------------- | :------------------ | +| `POST /widget` | 200 | 10 seconds | none | +| `GET /fish` | 200, 300 | 60 seconds (taken from `cache_options`) | `shark` | + +#### Configuring the middleware in the API Designer + +You can use the API Designer in the Tyk Dashboard to configure the endpoint caching middleware for your Tyk Classic API by following these steps. + +##### Simple endpoint cache + +To enable and configure the simple endpoint cache, follow these instructions: + +1. **Configure the API level caching options** + + From the **Advanced Options** tab configure the cache as follows: + - **Enable caching** to enable the cache middleware + - **Cache timeout** to configure the timeout (in seconds) for cached requests + - **Cache only these status codes** is a list of HTTP status codes that should be cached, remember to click **Add** after entering each code to add it to the list + - **Cache all safe requests** ensure that this is **not** selected, otherwise the responses from all endpoints for the API will be cached + + Cache Options + +2. **Add an endpoint for the path and select the plugin** + + From the **Endpoint Designer** add an endpoint that matches the path for which you want to cache responses. Select the **Cache** plugin. + + Dropdown list showing Cache plugin + +3. **Save the API** + + Use the *save* or *create* buttons to save the changes and activate the middleware. + +##### Advanced endpoint cache + +To enable and configure the advanced endpoint cache, follow these instructions: + +1. **Configure the API level caching options** + + From the **Advanced Options** tab configure the cache as follows: + - **Enable caching** to enable the cache middleware + - **Cache timeout** to configure the default timeout (in seconds) for any endpoints for which you don't want to configure individual timeouts + - **Cache only these status codes** leave this blank + - **Cache all safe requests** ensure that this is **not** selected, otherwise the responses from all endpoints for the API will be cached + + Cache Options + +2. **Add an endpoint for the path and select the plugin** + + From the **Endpoint Designer** add an endpoint that matches the path for which you want to cache responses. Select the **Advanced Cache** plugin. + + Selecting the Advanced Cache plugin for a Tyk Classic API + +3. **Configure the Advanced Cache plugin** + + Set the timeout and HTTP response codes for the endpoint. If you don't need to set a specific timeout for an endpoint you can leave this blank and Tyk will use the cache timeout configured at the API level. + + Endpoint cache configuration for Tyk Classic API + + + + + Body value match or [request selective](/api-management/response-caching#request-selective-cache-control) caching is not currently exposed in the Dashboard UI, so it must be configured through either the raw API editor or the Dashboard API. + + + +4. **Save the API** + + Use the *save* or *create* buttons to save the changes and activate the middleware. + +

Configuring the middleware in the Tyk Operator

+You can use Tyk Operator to configure the endpoint caching middleware for your Tyk Classic API by following these steps. + +##### Simple endpoint cache + +Configuring simple endpoint caching in Tyk Operator is similar to the process for a Tyk Classic API Definition. A list of endpoints for which you wish to cache safe requests should be configured within the `cache` list in the `extended_paths` section. + +In the API-level `cache_options` object, you must enable caching by setting `enable_cache` to true and configure the cache refresh period by setting a value for the `cache_timeout` in seconds. To allow selective caching per endpoint you should also set `cache_all_safe_requests`to `false`. + +```yaml {linenos=true, linenostart=1, hl_lines=["26-35"]} +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-cache +spec: + name: httpbin-cache + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin-cache + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + cache: + - /get + - /anything + cache_options: + cache_all_safe_requests: false +# cache_by_headers: [] + cache_timeout: 10 + cache_response_codes: + - 400 + enable_cache: true +``` + +##### Advanced endpoint cache + +Advanced caching with Tyk Operator is a similar process to that for configuring the [advanced caching middleware in the Tyk Classic API Definition](#tyk-classic-advanced-caching). + +To enable the advanced middleware you must add a new `advance_cache_config` object to the `extended_paths` section of your API definition. + +This allows you to configure caching per endpoint. For each endpoint, it is possible to specify the endpoint path, method, list of response codes to cache, cache timeout and a cache key regular expression. The cache key regular expression represents a pattern match to cache only requests containing specific data in the [request body](/api-management/response-caching#request-selective-cache-control) + +For example: + +```yaml {linenos=true, linenostart=1, hl_lines=["26-35"]} +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-advance-cache +spec: + name: httpbin-advance-cache + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin-advance-cache + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + advance_cache_config: + - path: /anything + method: GET + cache_key_regex: "" + cache_response_codes: [200] + cache_options: + cache_timeout: 30 + enable_cache: true +``` expandable + +In this example the endpoint caching middleware has been configured to cache requests for the `/anything` endpoint as follows: + +| endpoint | HTTP response codes to cache | cache refresh timeout | body value regex | +| :---------- | :------------------------------ | :----------------------- | :------------------ | +| `GET /anything` | 200 | 30 seconds (taken from `cache_options`) | none | + +## Upstream Cache Control + +Upstream cache control refers to the caching of API responses based on instructions provided by the upstream service. This allows the upstream service to have control over which responses are cached and for how long and can be used to perform caching of traditionally "non-safe" requests. The upstream service controls the cache using parameters in the response header. + +This approach gives the most granular control as it will also only cache responses based on the request method. + +For example, if you only want to cache requests made with the `OPTIONS` method, you can configure the upstream cache control accordingly and return cache control headers only in those responses. With this configuration, Tyk will cache only those responses, not those for other methods for the same path. + +Upstream cache control is configured on a per-API and per-endpoint basis, giving maximum flexibility. All configuration is performed within the API definition. + +### Enabling upstream cache control for an API + +To set up upstream cache control, you must configure `cache_options` in the API definition as follows: + - first enable the Tyk cache (using `enable_cache`) + - ensure that global/safe request caching is disabled (`cache_all_safe_requests` is set to `false`) + - set `enable_upstream_cache_control` to `true` + - add the endpoints to be cached to the list in `extended_paths.cache` + +For example, to enable upstream cache control for the `/ip` endpoint (path) of your API you would add the following to the API definition: + +``` +"cache_options": { + "enable_cache": true, + "cache_all_safe_requests": false, + "enable_upstream_cache_control": true, + "extended_paths": { + "cache": [ + "ip" + ] + } +} +``` expandable + +If you are using Tyk Dashboard, you can configure these settings within the Advanced Settings section of the API Designer. You should select **Enable upstream cache control** and deselect **Global cache**, then follow the steps for per-path caching. + +### Operating cache control from the upstream server + +When upstream cache control is configured, the Gateway will check the response from the upstream server for the header `x-tyk-cache-action-set`: + - if this is provided in the response header and is set to `1` or `true` then the response will be stored in the cache + - if the header is empty or absent, Tyk follows its default behavior, which typically involves not caching the request, or caching only valid response codes (`cache_response_codes`) + +The upstream server also controls the length of time that Tyk should cache the response (Time-To-Live or TTL). + +Tyk looks for the header `x-tyk-cache-action-set-ttl` in the response: + - if this is found and has a positive integer value, the Gateway will cache the response for that many seconds + - if the header is not present, Tyk falls back to the value specified in `cache_options.cache_timeout` + +By configuring these headers in the responses from your services, you can have precise control over caching behavior. + +#### Using a custom TTL header key +If you wish to use a different header value to indicate the TTL you can do so by adding the `cache_control_ttl_header` option to the API definition. + +For example, if you configure: + ``` + "cache_options": { + "cache_control_ttl_header": "x-expire" + } + ``` + +and also send `x-expire: 30` in the response header, Tyk will cache that specific response for 30 seconds. + + + +## Invalidating the Cache + +The cache for an API can be invalidated (or flushed) to force the creation of a new cache entry before the cache’s normal expiry. + +This is achieved by calling one of the dedicated cache invalidation API endpoints. There is a cache invalidation endpoint in both the Tyk Dashboard API and Tyk Gateway API; the URLs differ slightly, but they have the same effect. + +For Dashboard-managed deployments, it’s recommended to call the Dashboard API version, as this will handle the delivery of the message to all Gateways in the cluster. + +Caches are cleared on per-API basis, so the request to the invalidation endpoint must include the ID of the API in the path. + +For example, with the Tyk Gateway API: + +``` +DELETE /tyk/cache/{api-id} +``` + +and with the Tyk Dashboard API: + +``` +DELETE /api/cache/{api-id} +``` expandable + +Note that prior to Tyk version 3.0.9 and 4.0, this was not supported on MDCB Data Plane gateways. + + + +Cache invalidation is performed at the API level, so all cache entries for the API will be flushed. + + + +## Optimizing the Cache Storage + +Tyk creates the API cache in Redis, as it gives high performance and low latency. By default, the cache will use the same database that is used to store the API keys, minimizing the deployment footprint. + +For [multi-data center](/api-management/mdcb#redis) deployments, the Data Planes have a locally deployed Redis. This enables them to have a localised cache close to the traffic-serving Gateways. + +The [cache key](/api-management/response-caching#cache-key) is used as the Redis key, for quick lookups. + +For high-traffic systems that make heavy use of caching, it can make sense to use separate Redis databases for cache storage and for API keys, at the expense of increased deployment footprint. + +### Configuring a separate cache +To enable a separate cache server, you must deploy additional Redis instance(s) and apply additional configuration within your Tyk Gateway's `tyk.conf` configuration file. + +You must + - set `enable_separate_cache_store` to `true` + - provide additional Redis connection information in the `cache_storage` section + +For example: +```json +{ +"enable_separate_cache_store": true, +"cache_storage": { + "type": "redis", + "host": "", + "port": 0, + "addrs": [ + "localhost:6379" + ], + "username": "", + "password": "", + "database": 0, + "optimisation_max_idle": 3000, + "optimisation_max_active": 5000, + "enable_cluster": false + } +} +``` + +The configuration of the separate Redis Cache is the same (and uses the same underlying driver) as the regular configuration, so [Redis Cluster](/tyk-configuration-reference/redis-cluster-sentinel#configure-redis-cluster) is fully supported. If you set `enable_cluster` to `false`, you only need to set one entry in `addrs`. + + + +Prior to Tyk Gateway v2.9.3, `hosts` was used instead of `addrs`; since v2.9.3 `hosts` has been deprecated. + + + diff --git a/api-management/security-best-practices.mdx b/api-management/security-best-practices.mdx new file mode 100644 index 00000000..aaaba80a --- /dev/null +++ b/api-management/security-best-practices.mdx @@ -0,0 +1,418 @@ +--- +title: "Security Best Practices" +'og:description': "Guide on API management and security best practices, including authentication, authorization, resource protection, governance, and OWASP threat mitigation with Tyk." +sidebarTitle: "Security Best Practices" +tags: ['OWASP', 'Security', 'Top Ten', 'API Management best practice', 'API Security', 'Authentication', 'Security', 'Configuration', 'SSL', 'Certificates', 'Authentication', 'Authorization', 'API security', 'API Gateway Security'] +--- + +## Overview + +This section serves as a detailed resource for understanding key concepts and tools related to API security. It provides explanations of critical practices such as authentication, authorization, and governance, offering insights into how these concepts work and why they matter. Whether you're looking to mitigate threats identified by the [OWASP API Security Top 10](https://owasp.org/API-Security/editions/2023/en/0x00-header/) or to configure your APIs for better resilience, this page breaks down the essentials. + +Two of the most prevalent topics are [authentication](#authentication) and [authorization](#authorization), which occupy four of the top five positions. These are critical elements of API security, which verify the identity of API clients and control what they’re able to do. Alongside these are a number of other beneficial topics that are also within the remit of API management, all of which will be covered in this section. These include: + +- [Governance](#governing-apis-effectively) +- [Configuration](#configuration-best-practices) +- [Resource Consumption](#managing-api-resources) + +## Mitigating The Top 10 OWASP Threats + +The Open Web Application Security Project (OWASP) provides a top ten threat awareness document compiled by security experts. For more details on the OWASP project visit [https://www.owasp.org](https://www.owasp.org). Below are the top ten threats and how Tyk guards against them. For further details please visit our [blog](https://tyk.io/blog/res-owasp-api-security-intro/) + +##### 1 - Broken Object Level Authorization (BOLA) + +Broken Object Level Authorization (BOLA) can occur due to a lack of access control to API resources. This vulnerability allows attackers to manipulate or bypass authorization mechanisms, typically by tampering with resource identifiers to gain unauthorized access to specific resources or data. BOLA is a critical security concern as it can lead to data breaches and unauthorized actions within a system. + +It is the responsibility of the API to handle this form of attack since it can access and understand the data needed to make authorization decisions on individual objects within the application database. + +##### 2 - Broken Authentication + +Authentication is a vital aspect of API security. Failure to do so, as noted by OWASP, leads to *Broken Authentication* posing a significant risk to both API providers and data. + +Tyk provides the following features and authentication mechanisms: +- Prioritize secure methods, like [mutual TLS](/basic-config-and-security/security/mutual-tls/client-mtls#why-use-mutual-tls), over [basic authentication](/api-management/authentication/basic-authentication) wherever feasible. +- API owners can integrate external Identity Providers (IdPs) supporting methods like [OpenID Connect](/api-management/client-authentication#integrate-with-openid-connect-deprecated), [OAuth 2.0](/api-management/authentication/oauth-2#using-the-authorization-code-grant) or [JSON Web Tokens](/basic-config-and-security/security/authentication-authorization/json-web-tokens). +- [Single Sign-On](/api-management/external-service-integration#single-sign-on-sso) can be used for a centralized and trusted authentication source. API operators can choose from common authentication methods such as OAuth 2.0, LDAP, and SAML. +- [Dynamic Client Registration](/tyk-developer-portal/tyk-portal-classic/dynamic-client-registration#oauth-20-dynamic-client-registration-protocol-dcr), enables third-party authorization servers to issue client credentials via the Tyk Developer Portal. This streamlines Identity Management, eliminating the need to manage credentials across multiple systems. +- Tyk's default authentication setup disallows credentials in URLs, reducing the risk of inadvertent exposure through backend logs. +- Tyk Gateway can be configured to enforce a [minimum TLS version](/api-management/certificates#supported-tls-versions), enhancing security by blocking outdated and insecure TLS versions. + +##### 3 - Broken Object Property Level Authorization (BOPLA) + +REST APIs provide endpoints that return all properties of an object in the reponse, some of which could contain sensitive data. Conversely, GraphQL API requests allow the clients to specify which properties of an object should be retrieved. + +From a REST API perspespective, it is the responsibility of the API to ensure that the correct data is retrieved. The Gateway can provide additional security measures as follows: +- [Body transformation plugins](/api-management/traffic-transformation/request-method) can be used to remove sensitive data from the response if the API is unable to do so itself. +- [JSON Schema validation](/api-management/traffic-transformation/request-validation#request-validation-using-classic) to validate that an incoming data payload meets a defined schema. Payloads that do not adhere to the schema are rejected. + +For GraphQL APIs, the gateway can be used to define the GraphQL schemas, limiting which properties of an object are queryable. Furthermore, access can be controlled to specific properties by configuring [field-based permissions](/api-management/graphql#field-based-permissions). Subsequently, the visiblity of a schema's properties can be controlled for different consumers of the GraphQL API. + + +##### 4 - Unrestricted Resource Consumption + +APIs can become overwhelmed if the resources upon which they rely are fully consumed. In such situations, an API can no longer operate, and will no longer be able to service requests, or potentially even be unable to complete those currently in progress. + +As an APIM product, Tyk Gateway can be configured to use the following out-of-the-box functionality when handling API traffic for legitimate users: + +- [Circuit breaker](/planning-for-production/ensure-high-availability/circuit-breakers) +- [Payload size limiter](/api-management/traffic-transformation/request-size-limits) +- [Rate limiter / throttling](/api-management/rate-limit#introduction) +- [Caching](/api-management/response-caching) +- [Enforced timeout](/planning-for-production/ensure-high-availability/enforced-timeouts) +- [IP restriction](/api-management/gateway-config-tyk-classic#ip-access-control) +- [GraphQL query complexity limiting](/api-management/graphql#complexity-limiting-1) + +For Denial of Service (DoS) attacks it is recommended to use specialist 3rd party services to prevent DoS attacks from reaching your infrastructure. + +##### 5 - Broken Function Level Authorization (BFLA) + +To prevent Broken Functional Level Authorization (BFLA), requests to REST API endpoints must be authorized correctly. This involves validating client permissions against the requested resources. Requests from clients with insufficient permissions must be rejected. + +Tyk offers several measures to assist with protection from BFLA threats: + +- *Establish path-based access rights*: [Policies](/api-management/policies#what-is-a-security-policy) are predefined sets of rules which grant access to particular APIs. These can include [path-based permissions](/api-management/policies#secure-your-apis-by-method-and-path), which restrict access to particular paths and methods within an API. Clients can be assigned one or more policies which the Gateway will validate when it receives a request. +- *Access Control*: Tyk has plugins that control access to API endpoints. They are known as [allowlist](/api-management/traffic-transformation/allow-list#api-definition) and [blocklist](/api-management/traffic-transformation/block-list#api-designer) and can be configured via the Endpoint Designer of an API Definition. Both plugins grant and deny access to API paths and methods, but do so in different ways, which makes them mutually exclusive. When the allowlist plugin is used, only the marked paths and methods are allowed, all other paths and methods are blocked. This can be perceived as *deny by default* since it provides the least privileges. The reverse is true for the blocklist plugin, only the paths and methods marked as blocklist are blocked, all other paths and methods are allowed. It is recommended to use the *allowlist* approach, since it is the most restrictive, only allowing marked endpoint paths and paths. +- *CORS*: This [functionality](/api-management/gateway-config-tyk-classic#cross-origin-resource-sharing-cors) allows the Tyk Gateway to limit API access to particular browser-based consumers. + +##### 6 - Unrestricted Access To Sensitive Business Flows + +This involves attackers understanding an API's business model, identifying sensitive business processes and automating unauthorized access to these processes. This can disrupt business operations by preventing legitimate users from making purchases for example. Attackers manually locate target resources and work to bypass any existing mitigation measures. + +These business flows are application specific, being unique to the API's backend systems. Subsequently, the API owner is responsible for addressing the security issues posed by this threat. Furthermore, to discover points of exploitation and test IT security breaches, pentesting is recommended. + +The APIM can be used to protect sensitive endpoints using authentication and authorization. Tyk recommends considering splitting Admin APIs from client facing APIs. This allows authentication and authorization checks to be defined and managed by different governance models, thus establishing clear role models. + +Furthermore, the APIM can validate authentication and authorization by scope to ensure that the client has the correct credentials before the upstream API processes the request. + +##### 7 - Server Side Request Forgery (SSRF) + +Server Side Request Forgery (SSRF) is a security vulnerability in web applications where an attacker can manipulate a server to make unauthorized requests to internal or external resources, potentially leading to data leaks or remote code execution. This can allow an attacker to probe or attack other parts of the application's infrastructure, potentially compromising sensitive information and systems. + +This is application specific and is largely the responsibility of the API. However, Tyk Gateway can assist with this form of attack through [JSON schema validation](/api-management/traffic-transformation/request-validation#request-validation-using-classic) for incoming payloads. For example, a schema could contain a regular expression to reject localhost URLs. These URLs could be used by an attacker to perform port scanning for example. + +##### 8 - Security Misconfiguration + +Tyk offers several mechanisms to help protect an API from Security Misconfiguration exploits: + +- Use [response header manipulation](/api-management/traffic-transformation/response-headers) to remove or modify API sensitive information. +- Use [response body manipulation](/api-management/traffic-transformation/response-body) to remove or modify parts containing sensitive information. +- [TLS](/api-management/certificates) to ensure that clients use the right service and encrypt traffic. +- [Mutual TLS](/basic-config-and-security/security/mutual-tls/client-mtls#why-use-mutual-tls) with both the clients and API to ensure that callers with explicitly allowed client certificates can connect to the endpoints. +- [Error Templates](/api-management/gateway-events#error-templates) can be used to return a response body based on status code and content type. This can help minimize the implementation details returned to the client. +- [CORS functionality](/api-management/gateway-config-tyk-classic#cross-origin-resource-sharing-cors) allows the Tyk Gateway to limit API access to particular browser-based consumers. +- [Policy Path-Based Permissions](/api-management/policies#secure-your-apis-by-method-and-path) and the [allowlist](/api-management/traffic-transformation/allow-list#api-definition) plugin can be used to prevent clients from accessing API endpoints using non-authorized HTTP methods. For example, blocking the use of the DELETE method on an endpoint which should only accept GET requests. +- [Environment variables](/tyk-oss-gateway/configuration) can help standardize configuration across containerised deployments. +- For GraphQL APIs: +- [Schema Introspection](/api-management/graphql#introspection) ensures that the Tyk Dashboard automatically uses the schema of the upstream GraphQL API and can keep it synchronised if it changes. +- [GraphQL Schema Validation](/api-management/graphql#schema-validation) prevents invalid schemas from being saved. This catches errors such as duplicate type names and usage of unknown types. +- Third-party [Secret Storage](/tyk-configuration-reference/kv-store) to centralise configuration of sensitive data such as passwords. This data can then be dynamically referenced by Tyk configuration files, rather than being hard coded. +- Users can can write their own [custom plugins](/api-management/plugins/overview#) in a variety of languages, either directly or through gRPC calls, to implement their requirements. + +The Ops team should also take reponsibility for monitoring the APIs for errors and patching accordingly. Regular [Penetration Tests](https://en.wikipedia.org/wiki/Penetration_test) should be scheduled to ensure the security of published services. Tyk, through our Professional Services or Partners, can assist in the process. + +##### 9 - Improper Inventory Management + +Tyk offers the following features to support improper inventory management: + +- [Versioning](/api-management/api-versioning) allows newer versions of APIs to coexist with the older versions, facilitating deprecation and sunsetting. +- [Sunsetting](/api-management/api-versioning#sunsetting-api-versions) allows versions to be configured with an Expiry Time, ensuring that a version is not accessible after the expiry date. +- [Key expiry](/api-management/policies#access-key-expiry) ensures that access to an API is short lived, with a per key configurable Time to Live (TTL) for which a token remains valid before it expires. The implementation of key expiry, with a configurable Time To Live (TTL), mitigates the impact of compromised tokens by narrowing the window of vulnerability. Setting a TTL reduces the time frame during which a compromised token could be exploited, enhancing overall security. +- Tyk Developer Portal catalogs APIs and facilitates granting access to them. Integrated with a CMDB it can help keep documentation updated. +- [Tyk Analytics](/api-management/dashboard-configuration#traffic-analytics) can help identify the stagnant APIs and used stale APIs. +- [Tyk Pump](/tyk-pump) can ship metrics needed for analytics into Tyk Dashboard and other systems. +- Third-party [Secret Storage](/tyk-configuration-reference/kv-store) can be used to centralise and protect sensitive configuration data such as passwords, rather than exposing them as plain text in Tyk configuration files. + +In addition, it is best practice to consider any definition of done to include corresponding documentation updates. + +##### 10 - Unsafe Consumption Of APIs + +Attackers may identify and target the third party APIs/services used by an API. This can lead to leaked sensitive information, denial of service, injection attacks etc. + +It is the responsibility of the API to provide protection against these attacks. However, if the organization uses the Gateway as a forwarding proxy to third party APIs, then the following features could be used: + +- [JSON Schema validation](/api-management/traffic-transformation/request-validation#request-validation-using-classic) to validate that an incoming data payload meets a defined schema. Payloads that do not adhere to the schema are rejected. +- [Versioning](/api-management/api-versioning) allows newer versions of third party APIs to coexist with the older versions, facilitating deprecation and sunsetting. +- [TLS](/api-management/certificates) to ensure that clients use the right service and encrypt traffic. + + +## Managing Authentication and Authorization + +### Authentication + +Authentication is the process of identifying API clients. It’s a broad topic, with many approaches to choose from. Choosing the right approach is important, as it forms a fundamental part of the overall security strategy. The decision depends on many risk factors; users, functionality, data, accessibility and compliance, to name just a few. While there isn’t necessarily a single, correct choice, it’s usually safe to assume that some form of authentication is needed, as it’s a crucial prerequisite in performing subsequent identity-based authorization checks. + +**Implement Appropriate Authentication** + +Choose a suitable authentication approach based on the risk profile of the API. Is it publicly accessible or internal? Does it require user interaction or is it machine to machine? How sensitive is the data and functionality provided by the API? Simplistic approaches, such as [Bearer Tokens](/api-management/authentication/bearer-token), can work for low risk, basic APIs, but for higher risk or more sophisticated APIs, it may be more appropriate to use a standards-based approach such as [OAuth 2.0](/api-management/authentication/oauth-2) or [OpenID Connect](/api-management/client-authentication#integrate-with-openid-connect-deprecated). Furthermore, using an [external identity provider](/api-management/client-authentication#integrate-with-external-authorization-server-deprecated) can deliver additional benefits, such as [single sign-on](/api-management/external-service-integration#single-sign-on-sso), as well as multi-factor authentication approaches such as [biometric verification](https://www.okta.com/identity-101/biometrics-secure-authentication). + +**Handle Data Securely** + +Don’t undermine the authentication process by leaking sensitive authentication data. Use [transport layer security](/api-management/certificates) and hashing to prevent credentials from being intercepted and stolen through insecure transmission and storage. These principles also apply to upstream requests made by the gateway and upstream API to other APIs and services. + +**Enforce Good Practices** + + +Establish rules that reduce risk and enhance overall system security. Use [password policies](/api-management/user-management#password-policy) to prevent the use of weak passwords, and [TLS policies](/api-management/certificates#supported-tls-versions) to prevent the use of older TLS versions that are now deprecated and considered vulnerable. + +**Protect Sensitive Endpoints** + +Reduce susceptibility of sensitive endpoints to brute force dictionary or password stuffing attacks. The typical target for this type of attack are endpoints that use credentials, such as login and password recovery. Unfortunately, anonymous access is required for these endpoints, so authentication cannot be used to protect them, so the best approach is to hinder access by using techniques such as [rate limiting](/api-management/rate-limit#rate-limiting-layers), [captcha](https://en.wikipedia.org/wiki/CAPTCHA) and one-time URLs. + + +### Authorization +Authorization is the process of validating API client requests against the access rights they have been granted, ensuring that the requests comply with any imposed limitations. It’s the most prevalent topic on the OWASP list, with three entries covering different levels of authorization. + +Almost any part of a request can be scrutinised as part of authorization, but choosing the best approach depends on the type of API. For example, with REST APIs, the requested method and path are good candidates, but they aren’t relevant for GraphQL APIs, which should focus on the GraphQL query instead. + +Authorization can be a complex process that occurs at multiple locations throughout the request lifecycle. For example, a gateway can use access control policies to determine whether a required path is acceptable. But for decisions based on object data, such as when a client requests a particular record from the database, it’s the API that’s best positioned, as only it has access to the necessary data. For more information about the authorization process, see Authorization Levels in the appendix. + +#### Split Authorization + +Implement authorization in the best locations across the stack. For an overview of the different authorization levels across the stack please visit this [page](#managing-authorization-levels). Use the gateway to handle general API authorization related to hosts, methods, paths and properties. This leaves the API to handle the finer details of object-level authorization. In terms of OWASPs authorization categories, it can be split as follows: + +##### Object Level Authorization + +Handle with the API. It can access and understand the data needed to make authorization decisions on individual objects within its database. + +##### Object Property Level Authorization + +Handle with both the API and the gateway. The approach depends on the type of API: + +For REST APIs, it’s the API that’s primarily responsible for returning the correct data. To complement this, the gateway can use [body transforms](/api-management/traffic-transformation/response-body) to remove sensitive data from responses if the API is unable to do so itself. The gateway can also enforce object property-level restrictions using [JSON validation](/api-management/traffic-transformation/request-validation#request-validation-using-classic), for scenarios where the client is sending data to the API. + +For GraphQL APIs, use the gateway to define [GraphQL schemas](/api-management/graphql#managing-gql-schema) to limit which properties are queryable, then optionally use [field-based permissions](/api-management/graphql#field-based-permission) to also specify access rights to those properties. + +##### Function Level Authorization + +Handle with the gateway. Use [security policies](/api-management/policies), [path-based permissions](/api-management/policies#secure-your-apis-by-method-and-path), [allow lists](/api-management/traffic-transformation/allow-list#api-definition) and [block lists](/api-management/traffic-transformation/block-list#api-designer) to manage authorization of hosts and paths. + +#### Assign Least Privileges + +Design [security policies](/api-management/policies#what-is-a-security-policy) that contain the least privileges necessary for users to achieve the workflows supported by the API. By favoring specific, granular access over broad access, this enables user groups and use cases to be addressed directly, as opposed to broad policies that cover multiple use cases and expose functionality unnecessarily. + +##### Deny by Default + +Favor use of [allow lists](/api-management/traffic-transformation/allow-list#api-definition) to explicitly allow endpoints access, rather than [block lists](/api-management/traffic-transformation/block-list#api-designer) to explicitly deny. This approach prevents new API endpoints from being accessible by default, as the presence of other, allowed endpoints means that access to them is implicitly denied. + +##### Validate and Control All User Input + +Protect APIs from erroneous or malicious data by validating all input before it’s processed by the API. Bad data, whether malicious or not, can cause many problems for APIs, from basic errors and bad user experience, to data leaks and downtime. The standard mitigation approach is to validate all user input, for which there are various solutions depending on the type of API: + +For REST APIs, use [schema validation](/api-management/graphql#schema-validation) to control acceptable input data values. + +For GraphQL APIs, use [GraphQL schema](/api-management/graphql#managing-gql-schema) definitions to limit what data can be queried and mutated. Additionally, [complexity limiting](/api-management/graphql#complexity-limiting-1) can be used to block resource-intensive queries. + +#### Track Anomalies + +Use [log aggregation](/api-management/logs-metrics#exporting-logs-to-third-party-tools) and [event triggers](/api-management/gateway-events#event-categories) to push data generated by application logs and events into centralised monitoring and reporting systems. This real-time data stream can be used to highlight application issues and security-related events, such as authentication and authorization failures. + +##### Understand System State + +Perform application performance monitoring by capturing gateway [instrumentation data](/api-management/logs-metrics#statsd-instrumentation). This enables the current system state, such as requests per second and response time, to be monitored and alerted upon. + +##### Manage Cross-Origin Resource Sharing + +Use [CORS filtering](/api-management/gateway-config-tyk-classic#cross-origin-resource-sharing-cors) to control the resources accessible by browser-based clients. This is a necessity for APIs that expect to be consumed by external websites. + + +### Managing Authorization Levels + +This section provides basic examples of where different authorization levels occur in the API management stack. The accompanying diagrams use color-coding to show links between request element and the associated authorization locations and methods. + +This is how OWASP describe the attack vectors for the three authorization levels: + +**Object Level Authorization**: β€œAttackers can exploit API endpoints that are vulnerable to broken object-level authorization by manipulating the ID of an object that is sent within the request. Object IDs can be anything from sequential integers, UUIDs, or generic strings. Regardless of the data type, they are easy to identify in the request target (path or query string parameters), request headers, or even as part of the request payload.” (source: [OWASP Github](https://github.com/OWASP/API-Security/blob/9c9a808215fcbebda9f657c12f3e572371697eb2/editions/2023/en/0xa1-broken-object-level-authorization.md)) + +**Object Property Level Authorization**: β€œAPIs tend to expose endpoints that return all object’s properties. This is particularly valid for REST APIs. For other protocols such as GraphQL, it may require crafted requests to specify which properties should be returned. Identifying these additional properties that can be manipulated requires more effort, but there are a few automated tools available to assist in this task.” (source: [OWASP Github](https://github.com/OWASP/API-Security/blob/9c9a808215fcbebda9f657c12f3e572371697eb2/editions/2023/en/0xa3-broken-object-property-level-authorization.md)) + +**Function Level Authorization**: β€œExploitation requires the attacker to send legitimate API calls to an API endpoint that they should not have access to as anonymous users or regular, non-privileged users. Exposed endpoints will be easily exploited.” (source: [OWASP Github](https://github.com/OWASP/API-Security/blob/9c9a808215fcbebda9f657c12f3e572371697eb2/editions/2023/en/0xa3-broken-object-property-level-authorization.md)) + + +#### REST API - Reading Data + +Rest API - Read Data + +The client sends a `GET` request using the path `/profile/1`. This path has two parts: + +1. `/profile/`: The resource type, which is static for all requests related to profile objects. This requires function level authorization. + +2. `1`: The resource reference, which is dynamic and depends on the profile is being requested. This requires object level authorization. + +Next, the gateway handles function level authorization by checking that the static part of the path, in this case `/profile/`, is authorized for access. It does this by cross referencing the security policies connected to the API key provided in the `authorization` header. + +The gateway ignores the dynamic part of the part of the path, in this case `1`, as it doesn't have access to the necessary object-level data to make an authorization decision for this. + +Lastly, the API handles object level authorization by using custom logic. This typically involves using the value of the `authorization` header in combination with the ownership and authorization model specific to the API to determine if the client is authorized to read is requested record. + +#### REST API - Writing Data + +Rest API - Write Data + +The client sends a `POST` request using the path `/profile` and body data containing the object to write. The path `/profile` is static and requires function level authorization. The body data contains a JSON object that has two fields: + +1. `name`: A standard object field. This requires object property authorization. + +2. `id`: An object identifier field that refers to the identity of an object, so needs to be treated differently. As such, it requires both object property authorization, like name, and also object authorization. + +Next, the gateway handles function level authorization, by checking that the path, in the case `/profile`, is authorized for access. It does this by cross referencing the security policies connected to the API key provided in the `authorization` header. + +The gateway can also perform object property level authorization, by validating that the values of the body data fields, `name` and `id`, conform to a schema. + +Lastly, the API handles object level authorization by using custom logic. This typically involves using the value of the `authorization` header in combination with the ownership and authorization model specific to the API to determine if the client is authorized to write the requested data. + +#### GraphQL API - Querying Data + +Rest API - Write Data + +The client sends a `POST` request using the path `/graphql` and body data containing a GraphQL query. The path `/graphql` is static and requires function level authorization. The GraphQL query contains several elements: + +- `profile`: An object type, referring to the type of object being requested. This requires object property authorization. +- `id`: An object identifier field that refers to the identity of an object, so needs to be treated differently. As such, it requires both object property authorization, like name, and also object authorization. +- `name`: A standard object field, referring to a property of the profile object type. This requires object property authorization. + +Next, the Gateway handles function level authorization, by checking that the path, in the case `/graphql`, is authorized for access. It does this by cross referencing the security policies connected to the API key provided in the `authorization` header. Due to the nature of GraphQL using just a single endpoint, there is no need for additional path-based authorization features, only a basic security policy is required. + +Another difference between this and the REST examples is in the way that the body data is authorized: + +- All object types and fields contained in the query are checked against the API’s GraphQL schema, to ensure they are valid. In this case, the object type is `profile`, and the fields are `id` and `name`. The schema defined in the gateway configuration can differ from that in the upstream API, which enables fields to be restricted by default. +- Field-based permissions can also be used, to authorize client access of individual fields available in the schema. In this case, `id` and `name`. + +Lastly, the API handles object level authorization by using custom logic. This typically involves using the value of the `authorization` header in combination with the ownership and authorization model specific to the API to determine if the client is authorized to access the requested data. This can be more complicated for GraphQL APIs, as the data presented by the schema may actually come from several different data sources. + +## Managing API Resources + +Excessive resource consumption poses a risk to APIs. As the number of concurrent requests handled by a server increases, so too does its consumption of CPU, RAM and storage resources. Should any of these become depleted, then the quality of service offered by applications running on the server will rapidly decline, and may even lead to their complete failure. + +This issue can be caused by both legitimate consumers and malicious attackers, but they are different situations that require different solutions. For legitimate consumers, solutions should be focused on controlling API utilization through the gateway, to keep usage within agreed or desired limits. But malicious attackers require a different approach, as denial of service attacks must be blocked as far as possible from the core API infrastructure. + +**Restrict Request Flows**: Use [rate limits](/api-management/rate-limit#rate-limiting-layers) and [quotas](/api-management/request-quotas) to prevent excessive API usage. Rate limits are best used for short term control, in the range of seconds. Whereas quotas are more suited to longer terms, in the range of days, weeks or beyond. [Throttling](/api-management/request-throttling) can also be used as a type of enhanced rate limiter that queues and retries requests on the clients behalf, rather than immediately rejecting them. + +**Block Excessively Large Requests**: Place reasonable [limitations on payload sizes](/api-management/traffic-transformation/request-size-limits) to prevent oversized requests from reaching upstream servers, thereby avoiding the unnecessary consumption of resources. + +**Avoid Unnecessary Resource Usage**: Appropriate use of [caching](/api-management/response-caching) can reduce server resource consumption by simply returning cached responses instead of generating new ones. The extent to which caching can be used depends on the purpose of the endpoint, as it’s generally unsuitable for requests that modify data or responses that frequently change. Caching can be applied to [particular requests](/api-management/response-caching#endpoint-caching) or enabled for an [entire API](/api-management/response-caching#basic-caching), and can also be [controlled by the upstream API](/api-management/response-caching#upstream-cache-control-1) or [invalidated programmatically](/api-management/troubleshooting-debugging#how-to-clear--invalidate-api-cache). + +**Limit Complex Long-Running Tasks**: Use [GraphQL complexity limiting](/api-management/graphql#complexity-limiting-1) to prevent convoluted queries from being processed. Alternatively, [timeouts](/planning-for-production/ensure-high-availability/enforced-timeouts) can be used to terminate long-running requests that exceed a given time limit. + +**Protect Failing Services**: Defend struggling endpoints by using a [circuit breaker](/planning-for-production/ensure-high-availability/circuit-breakers). This feature protects endpoints by detecting error responses, then blocking requests for a short duration to allow them to recover. The same principle can be applied in a wider sense by using [uptime tests](/api-management/gateway-config-tyk-classic#uptime-tests), though this works on a host level instead, by removing failed hosts from the gateway load balancer. + +**Enforce Network-Level Security**: Problematic clients can be prevented from accessing the API by [blocking their address](/api-management/gateway-config-tyk-classic#ip-access-control). Conversely, for APIs with a known set of clients, [allow lists](/api-management/gateway-config-tyk-classic#ip-access-control) can be used to create a list of allowed addresses, thereby implicitly blocking every other address from the API. + +**Mitigate DoS Attacks**: Increase the chance of maintaining API availability during a denial of service attack by using [specialist mitigation services](https://www.cloudflare.com). These have the infrastructure capacity needed to handle [large scale distributed attacks](https://www.cloudflare.com/en-gb/learning/ddos/what-is-a-ddos-attack), with the purpose of preventing attacks from reaching the API infrastructure, thereby enabling the API to continue operating normally. + + +## Configuration Best Practices + +Modern APIs are often backed by large technology stacks composed of numerous components and libraries. Each of these is a potential weak link in the security chain, so efforts must be made to ensure that security measures are implemented throughout. The API gateway plays a critical part in an overall security strategy, by utilizing its ability to process requests in a secure manner. + +**Secure Connections** + + +Use [transport layer security](/api-management/certificates) where possible. Most importantly, on inbound connections to the gateway and outbound connection from the gateway to the upstream API and other services. TLS can also be used as a form of authentication, using [Mutual TLS](/basic-config-and-security/security/mutual-tls/client-mtls#why-use-mutual-tls). + +**Limit Functionality** + + +Use [security policies](/api-management/policies#what-is-a-security-policy) to specify which paths, methods and schemas are accessible, whilst blocking all others. + +**Mitigate Server-Side Request Forgery** + + +Restrict any URL-based input data to specific schemas, hosts and paths by using [schema validation](/api-management/graphql#schema-validation). When data is fetched server-side, it should be validated and not returned to the client in raw format. + +**Protect Secrets** + + +Prevent sensitive data, such as usernames, passwords, license keys and other secrets, from being stored as plain text in application configuration files. Use [key value secret storage](/tyk-configuration-reference/kv-store) to dynamically load sensitive data from a secure secret manager. + +**Sanitise Responses** + + +Modify or remove sensitive data from responses by using [transforms](/api-management/traffic-transformation) to alter the [response headers](/api-management/traffic-transformation/response-headers) and [body](/api-management/traffic-transformation/response-body). + + + +**Sign payloads between the Dashboard and Gateway** + + +Using payload signatures for the communication between Tyk Gateway and Tyk Dashboard is strongly recommended as an additional security measure, particularly in production environments. + +Enable payload signatures in the Gateway configuration (`tyk.conf` or environment variable) by setting `allow_insecure_configs` to `false` and then provide the public key (certificate) to the Gateway in the `public_key_path`. + +You'll need to provide the private key to the Dashboard using the `private_key_path` option in the appropriate configuration (`tyk_analytics.conf` or environment variable). This will allow your Dashboard to sign all of its payloads using the private key. + +You can easily create a public / private keypair with: + +```{.copyWrapper} +# private key +openssl genrsa -out privkey.pem 2048 + +# public key +openssl rsa -in privkey.pem -pubout -out pubkey.pem +``` + +Make sure to keep your private key safe! + +
+ +## Governing APIs Effectively + +APIs need to be managed and governed just like any other resource, otherwise organizations risk losing track of their API estate and becoming unaware of potentially vulnerable APIs running within their infrastructure. This risk is magnified as the number of teams, environments and APIs increases. Use API management as part of overarching business processes to control how APIs are accessed, managed and deployed. + +**Restrict Version Availability**: Enforce the expiry of [API versions](/api-management/api-versioning) that are planned for deprecation, by setting a sunset date, beyond which they will not be accessible. + +**Enforce Key Expiry**: In many situations it’s best to issue API keys that have a short, finite lifetime, especially when serving anonymous, external consumers. Set [expiry dates](/api-management/policies#access-key-expiry) for API keys, or use ephemeral credentials with complementary authentication techniques that support key renewal, such as [OAuth 2.0 refresh tokens](/api-management/authentication/oauth-2#using-refresh-tokens) and [dynamic client registration](/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/dynamic-client-registration). Then, should an API key fall into the wrong hands, there’s a chance that it has already expired. + +**Use Standardized Specifications**: Use the [OpenAPI Specification](https://en.wikipedia.org/wiki/OpenAPI_Specification) standard to design APIs. These specification documents act as a source of truth that can generate [API configuration](/api-management/gateway-config-tyk-oas) and [portal documentation](/tyk-apis/tyk-portal-api/portal-documentation#create-documentation). + +**Understand API Usage**: Use [API analytics](/api-management/dashboard-configuration#traffic-analytics) to report on usage. This captured data generates useful, actionable insights across a variety of metrics, such as API popularity, performance and trends. + +**Control API Distribution**: Use [sharding](/api-management/multiple-environments#what-is-api-sharding-) to control availability of APIs across multi-gateway, multi-environment deployments. This ensures that specific APIs are only available through specific gateways, which helps to prevent undesirable situations, such as internal APIs being published to externally accessible gateways, or test API configurations reaching the production environment. +
+ +## Securing APIs with Tyk + +Securing your APIs is one of the primary uses of Tyk API management solution. Out of the box, the Gateway offers a lot of functionality for securing your APIs and the Gateway itself. + +This section outlines all of the security configurations and components that are available to you when securing your Tyk stack. + +This section outlines some of the key security concepts that Tyk uses and that you should be familiar with before setting up and using a Tyk stack to secure your API. + +**Key Hashing** + + +See [Key Hashing](/api-management/policies#access-key-hashing) for details on how Tyk obfuscates keys in Redis. + +**TLS and SSL** + + +Tyk supports TLS connections and Mutual TLS. All TLS connections also support HTTP/2. Tyk also supports Let's Encrypt. See [TLS and SSL](/api-management/certificates) for more details. + +**Trusted Certificates** + + +As part of using Mutual TLS, you can create a list of [trusted certificates](/basic-config-and-security/security/mutual-tls/client-mtls#how-does-mutual-tls-work). + +**Certificate Pinning** + + +Introduced in Tyk Gateway 2.6.0, [certificate pinning](/api-management/upstream-authentication/mtls#certificate-pinning) is a feature which allows you to allow only specified public keys used to generate certificates, so you will be protected in case an upstream certificate is compromised. + +**API Security** + +Tyk supports various ways to secure your APIs, including: + +* Bearer Tokens +* HMAC +* JSON Web Tokens (JWT) +* Multi Chained Authentication +* OAuth 2.0 +* OpenID Connect + +See [Authentication and Authorization](/api-management/client-authentication) for more details. + +**Security Policies** + + +A Tyk security policy incorporates several security options that can be applied to an API key. These include [Partioned Policies](/api-management/policies#partitioned-policies) and securing by [Method and Path](/api-management/policies#secure-your-apis-by-method-and-path). + +See [Security Policies](/api-management/policies) for more details. diff --git a/api-management/security-features.mdx b/api-management/security-features.mdx new file mode 100644 index 00000000..7411fc1b --- /dev/null +++ b/api-management/security-features.mdx @@ -0,0 +1,86 @@ +--- +title: "Security Features" +'og:description': "Guide on API management and security best practices, including authentication, authorization, resource protection, governance, and OWASP threat mitigation with Tyk." +sidebarTitle: "Security Features" +tags: ['Security', 'security features', 'CORS', 'API Security', 'Cross-Origin Resource Sharing', 'Security', 'Configuration'] +--- + +## Cross-Origin Resource Sharing (CORS) + +CORS (Cross-Origin Resource Sharing) is a security feature that controls how web pages from one domain (origin) can make requests to resources hosted on a different domain. With Tyk Gateway, it is possible to enable and configure CORS per-API so that users can make browser-based requests. + +The `CORS` section is added to an API definition as listed in the examples below for Tyk Gateway and Tyk Operator. + +### Examples + + + +```json expandable +"CORS": { + "enable": true, + "allowed_origins": [ + "http://foo.com" + ], + "allowed_methods": [], + "allowed_headers": [], + "exposed_headers": [], + "allow_credentials": false, + "max_age": 24, + "options_passthrough": false, + "debug": false +} +``` + + +```yaml {linenos=true, linenostart=1, hl_lines=["14-24"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-cors-sample +spec: + name: httpbin-cors-sample + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /cors + strip_listen_path: true + CORS: + enable: true + allowed_origins: + - "http://foo.com" + allowed_methods: null + allowed_headers: null + exposed_headers: null + allow_credentials: false + max_age: 24 + options_passthrough: false + debug: false +``` + + + +--- + +### Configuration + +The CORS middleware has the following options: + +* `CORS.allowed_origins`: A list of origin domains to allow access from. Wildcards are also supported, e.g. `http://*.foo.com`. Default value is `["*"]` + +* `CORS.allowed_methods`: A list of methods to allow access via. Default value is `["GET", "POST", "HEAD"]` + +* `CORS.allowed_headers`: A list of headers that are allowed within a request. Default value is `["Origin", "Accept", "Content-Type", "X-Requested-With"]` + +* `CORS.exposed_headers`: A list of headers that are exposed back in the response. + +* `CORS.allow_credentials`: Whether credentials (cookies) should be allowed. + +* `CORS.max_age`: Maximum age of credentials. + +* `CORS.options_passthrough`: allow CORS OPTIONS preflight request to be proxied directly to upstream, without authentication and rest of checks. This means that pre-flight requests generated by web-clients such as SwaggerUI or +the Tyk Portal documentation system will be able to test the API using trial keys. If your service handles CORS natively, then enable this option. + +* `debug`: If set to `true`, this option produces log files for the CORS middleware. + diff --git a/api-management/stream-config.mdx b/api-management/stream-config.mdx new file mode 100644 index 00000000..208682fa --- /dev/null +++ b/api-management/stream-config.mdx @@ -0,0 +1,4666 @@ +--- +title: "Tyk Streams Configuration" +'og:description': "How to configure Tyk Streams" +tags: ['Broker', 'Input', 'Output', 'HTTP Client', 'HTTP Server', 'Processors', 'Scanners', 'CSV', 'Lines', 'Regular Expression', 'Switch', 'Avro', 'Kafka'] +sidebarTitle: "Tyk Streams Reference" +--- + +## Overview + +Tyk streams configuration is specified using YAML. The configuration consists of several main sections: *input*, *pipeline*, *output* and optionally *logger*. + +### Input + +The input section defines the publisher source of the data stream. Tyk Streams supports various input types such as Kafka, HTTP, MQTT etc. Each input type has specific configuration parameters. + +```yaml expandable +input: + kafka: + addresses: + - localhost:9092 + topics: + - example_topic + consumer_group: example_group + client_id: example_client +``` + +### Pipeline + +The pipeline section defines the processing steps applied to the data. It includes processors for filtering, mapping, enriching and transforming the data. Processors can be chained together. + +```yaml expandable +pipeline: + processors: + - mapping: | + root = this + root.foo = this.bar.uppercase() + - json_schema: + schema_path: "./schemas/example_schema.json" +``` + +### Output + +The output section specifies the destination of the processed data. Similar to inputs, Tyk Streams supports various output types like Kafka, HTTP etc. + +```yaml expandable +output: + kafka: + addresses: + - localhost:9092 + topic: output_topic + client_id: example_output_client +``` + +### Logger (Optional) + +The logger section is used to configure logging options, such as log level and output format. + +```yaml +logger: + level: INFO + format: json +``` + +## Inputs + +### Overview + +An input is a source of data piped through an array of optional [processors](/api-management/stream-config#overview-3): + +```yaml expandable +input: + label: my_kafka_input + + kafka: + addresses: [ localhost:9092 ] + topics: [ foo, bar ] + consumer_group: foogroup + + # Optional list of processing steps + processors: + - avro: + operator: to_json +``` + +#### Brokering + +Only one input is configured at the root of a Tyk Streams config. However, the root input can be a [broker](/api-management/stream-config#broker) which combines multiple inputs and merges the streams: + +```yaml expandable +input: + broker: + inputs: + - kafka: + addresses: [ localhost:9092 ] + topics: [ foo, bar ] + consumer_group: foogroup + + - http_client: + url: https://localhost:8085 + verb: GET + stream: + enabled: true +``` + +#### Labels + +Inputs have an optional field `label` that can uniquely identify them in observability data such as logs. + +{/* TODO + +When know if Tyk Streams will support metrics then link to metrics + +Inputs have an optional field `label` that can uniquely identify them in observability data such as metrics and logs. This can be useful when running configs with multiple inputs, otherwise their metrics labels will be generated based on their composition. For more information check out the [metrics documentation][metrics.about]. */} + +### Broker + +Allows you to combine multiple inputs into a single stream of data, where each input will be read in parallel. + +#### Common + +```yml expandable +# Common config fields, showing default values +input: + label: "" + broker: + inputs: [] # No default (required) + batching: + count: 0 + byte_size: 0 + period: "" + check: "" +``` + +## Advanced + +```yml expandable +# All config fields, showing default values +input: + label: "" + broker: + copies: 1 + inputs: [] # No default (required) + batching: + count: 0 + byte_size: 0 + period: "" + check: "" + processors: [] # No default (optional) +``` + +A broker type is configured with its own list of input configurations and a field to specify how many copies of the list of inputs should be created. + +Adding more input types allows you to combine streams from multiple sources into one. For example, reading from both RabbitMQ and Kafka: + +```yaml expandable +input: + broker: + copies: 1 + inputs: + - amqp_0_9: + urls: + - amqp://guest:guest@localhost:5672/ + consumer_tag: tyk-consumer + queue: tyk-queue + + # Optional list of input specific processing steps + processors: + - mapping: | + root.message = this + root.meta.link_count = this.links.length() + root.user.age = this.user.age.number() + + - kafka: + addresses: + - localhost:9092 + client_id: tyk_kafka_input + consumer_group: tyk_consumer_group + topics: [ tyk_stream:0 ] +``` + +If the number of copies is greater than zero the list will be copied that number of times. For example, if your inputs were of type foo and bar, with 'copies' set to '2', you would end up with two 'foo' inputs and two 'bar' inputs. + +##### Batching + +It's possible to configure a [batch policy](/api-management/stream-config#batch-policy) with a broker using the `batching` fields. When doing this the feeds from all child inputs are combined. Some inputs do not support broker based batching and specify this in their documentation. + +##### Processors + +It is possible to configure processors at the broker level, where they will be applied to *all* child inputs, as well as on the individual child inputs. If you have processors at both the broker level *and* on child inputs then the broker processors will be applied *after* the child nodes processors. + +#### Fields + +##### copies + +Whatever is specified within `inputs` will be created this many times. + + +Type: `int` +Default: `1` + +##### inputs + +A list of inputs to create. + + +Type: `array` + +##### batching + +Allows you to configure a [batching policy](/api-management/stream-config#batch-policy). + + +Type: `object` + +```yml expandable +# Examples + +batching: + byte_size: 5000 + count: 0 + period: 1s + +batching: + count: 10 + period: 1s + +batching: + check: this.contains("END BATCH") + count: 0 + period: 1m +``` + +##### batching.count + +A number of messages at which the batch should be flushed. If `0` disables count based batching. + + +Type: `int` +Default: `0` + +##### batching.byte_size + +An amount of bytes at which the batch should be flushed. If `0` disables size based batching. + + +Type: `int` +Default: `0` + +##### batching.period + +A period in which an incomplete batch should be flushed regardless of its size. + + +Type: `string` +Default: `""` + +```yml +# Examples + +period: 1s + +period: 1m + +period: 500ms +``` + +{/* TODO: when bloblang is supported +##### batching.check + +A Bloblang query that should return a boolean value indicating whether a message should end a batch. + + +Type: `string` +Default: `""` + +```yml +# Examples + +check: this.type == "end_of_transaction" +``` */} + +##### batching.processors + +A list of processors to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op. + + +Type: `array` + +```yml expandable +# Examples + +processors: + - archive: + format: concatenate + +processors: + - archive: + format: lines + +processors: + - archive: + format: json_array +``` + +### Http Client + +Connects to a server and continuously performs requests for a single message. + +#### Common + +```yml expandable +# Common config fields, showing default values +input: + label: "" + http_client: + url: "" # No default (required) + verb: GET + headers: {} + timeout: 5s + payload: "" # No default (optional) + stream: + enabled: false + reconnect: true + auto_replay_nacks: true +``` + +#### Advanced + +```yml expandable +# All config fields, showing default values +input: + label: "" + http_client: + url: "" # No default (required) + verb: GET + headers: {} + metadata: + include_prefixes: [] + include_patterns: [] + dump_request_log_level: "" + oauth: + enabled: false + consumer_key: "" + consumer_secret: "" + access_token: "" + access_token_secret: "" + oauth2: + enabled: false + client_key: "" + client_secret: "" + token_url: "" + scopes: [] + endpoint_params: {} + basic_auth: + enabled: false + username: "" + password: "" + jwt: + enabled: false + private_key_file: "" + signing_method: "" + claims: {} + headers: {} + tls: + enabled: false + skip_cert_verify: false + enable_renegotiation: false + root_cas: "" + root_cas_file: "" + client_certs: [] + extract_headers: + include_prefixes: [] + include_patterns: [] + timeout: 5s + retry_period: 1s + max_retry_backoff: 300s + retries: 3 + backoff_on: + - 429 + drop_on: [] + successful_on: [] + proxy_url: "" # No default (optional) + payload: "" # No default (optional) + drop_empty_bodies: true + stream: + enabled: false + reconnect: true + auto_replay_nacks: true +``` + +##### Streaming + +If you enable streaming then Tyk Streams will consume the body of the response as a continuous stream of data. This allows you to consume APIs that provide long lived streamed data feeds (such as Twitter). + +##### Pagination + +This input supports interpolation functions in the `url` and `headers` fields where data from the previous successfully consumed message (if there was one) can be referenced. This can be used in order to support basic levels of pagination. + +#### Examples + +##### Basic Pagination + +Interpolation functions within the `url` and `headers` fields can be used to reference the previously consumed message, which allows simple pagination. + +```yaml expandable +input: + http_client: + url: >- + http://api.example.com/search?query=allmyfoos&start_time=${! ( + (timestamp_unix()-300).ts_format("2006-01-02T15:04:05Z","UTC").escape_url_query() + ) }${! ("&next_token="+this.meta.next_token.not_null()) | "" } + verb: GET +``` + +{/* Update example when Tyk secrets Stream release has been performed + +```yaml expandable +input: + http_client: + url: >- + http://api.example.com/search?query=allmyfoos&start_time=${! ( + (timestamp_unix()-300).ts_format("2006-01-02T15:04:05Z","UTC").escape_url_query() + ) }${! ("&next_token="+this.meta.next_token.not_null()) | "" } + verb: GET + rate_limit: foo_searches + # oauth2: + # enabled: true + # token_url: https://api.example.com/oauth2/token + # client_key: "${EXAMPLE_KEY}" + # client_secret: "${EXAMPLE_SECRET}" + +rate_limit_resources: + - label: foo_searches + local: + count: 1 + interval: 30s +``` */} + +#### Fields + +##### url + +The URL to connect to. + + +Type: `string` + +##### verb + +A verb to connect with + + +Type: `string` +Default: `"GET"` + +```yml +# Examples + +verb: POST + +verb: GET + +verb: DELETE +``` + +##### headers + +A map of headers to add to the request. +{/* TODO: when interpolation supported: +This field supports interpolation functions. */} + +Type: `object` +Default: `{}` + +```yml +# Examples + +headers: + Content-Type: application/octet-stream + traceparent: ${! tracing_span().traceparent } +``` + +##### metadata + +Specify optional matching rules to determine which metadata keys should be added to the HTTP request as headers. + + +Type: `object` + +##### metadata.include_prefixes + +Provide a list of explicit metadata key prefixes to match against. + + +Type: `array` +Default: `[]` + +```yml expandable +# Examples + +include_prefixes: + - foo_ + - bar_ + +include_prefixes: + - kafka_ + +include_prefixes: + - content- +``` + +##### metadata.include_patterns + +Provide a list of explicit metadata key regular expression (re2) patterns to match against. + + +Type: `array` +Default: `[]` + +```yml +# Examples + +include_patterns: + - .* + +include_patterns: + - _timestamp_unix$ +``` + +##### dump_request_log_level + +Optionally set a level at which the request and response payload of each request made will be logged. + + +Type: `string` +Default: `""` +Options: `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR`, `FATAL`, ``. + +##### oauth + +Allows you to specify open authentication via OAuth version 1. + + +Type: `object` + +##### oauth.enabled + +Whether to use OAuth version 1 in requests. + + +Type: `bool` +Default: `false` + +##### oauth.consumer_key + +A value used to identify the client to the service provider. + + +Type: `string` +Default: `""` + +##### oauth.consumer_secret + +A secret used to establish ownership of the consumer key. + + +Type: `string` +Default: `""` + +##### oauth.access_token + +A value used to gain access to the protected resources on behalf of the user. + + +Type: `string` +Default: `""` + +##### oauth.access_token_secret + +A secret provided in order to establish ownership of a given access token. + + +Type: `string` +Default: `""` + +##### oauth2 + +Allows you to specify open authentication via OAuth version 2 using the client credentials token flow. + + +Type: `object` + +##### oauth2.enabled + +Whether to use OAuth version 2 in requests. + + +Type: `bool` +Default: `false` + +##### oauth2.client_key + +A value used to identify the client to the token provider. + + +Type: `string` +Default: `""` + +##### oauth2.client_secret + +A secret used to establish ownership of the client key. + + +Type: `string` +Default: `""` + +##### oauth2.token_url + +The URL of the token provider. + + +Type: `string` +Default: `""` + +##### oauth2.scopes + +A list of optional requested permissions. + + +Type: `array` +Default: `[]` + +##### oauth2.endpoint_params + +A list of optional endpoint parameters, values should be arrays of strings. + + +Type: `object` +Default: `{}` + +```yml expandable +# Examples + +endpoint_params: + bar: + - woof + foo: + - meow + - quack +``` + +##### basic_auth + +Allows you to specify basic authentication. + + +Type: `object` + +##### basic_auth.enabled + +Whether to use basic authentication in requests. + + +Type: `bool` +Default: `false` + +##### basic_auth.username + +A username to authenticate as. + + +Type: `string` +Default: `""` + +##### basic_auth.password + +A password to authenticate with. + + +Type: `string` +Default: `""` + +##### jwt + +Allows you to specify JWT authentication. + + +Type: `object` + +##### jwt.enabled + +Whether to use JWT authentication in requests. + + +Type: `bool` +Default: `false` + +##### jwt.private_key_file + +A file with the PEM encoded via PKCS1 or PKCS8 as private key. + + +Type: `string` +Default: `""` + +##### jwt.signing_method + +A method used to sign the token such as RS256, RS384, RS512 or EdDSA. + + +Type: `string` +Default: `""` + +##### jwt.claims + +A value used to identify the claims that issued the JWT. + + +Type: `object` +Default: `{}` + +##### jwt.headers + +Add optional key/value headers to the JWT. + + +Type: `object` +Default: `{}` + +##### tls + +Custom TLS settings can be used to override system defaults. + + +Type: `object` + +##### tls.enabled + +Whether custom TLS settings are enabled. + + +Type: `bool` +Default: `false` + +##### tls.skip_cert_verify + +Whether to skip server side certificate verification. + + +Type: `bool` +Default: `false` + +##### tls.enable_renegotiation + +Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. + + +Type: `bool` +Default: `false` + +##### tls.root_cas + +An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. + + +Type: `string` +Default: `""` + +```yml +# Examples + +root_cas: |- + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- +``` + +##### tls.root_cas_file + +An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. + + +Type: `string` +Default: `""` + +```yml +# Examples + +root_cas_file: ./root_cas.pem +``` + +##### tls.client_certs + +A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. + + +Type: `array` +Default: `[]` + +```yml expandable +# Examples + +client_certs: + - cert: foo + key: bar + +client_certs: + - cert_file: ./example.pem + key_file: ./example.key +``` + +##### tls.client_certs[].cert + +A plain text certificate to use. + + +Type: `string` +Default: `""` + +##### tls.client_certs[].key + +A plain text certificate key to use. + + +Type: `string` +Default: `""` + +##### tls.client_certs[].cert_file + +The path of a certificate to use. + + +Type: `string` +Default: `""` + +##### tls.client_certs[].key_file + +The path of a certificate key to use. + + +Type: `string` +Default: `""` + +##### tls.client_certs[].password + +A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format. Warning: Since it does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext. + +Type: `string` +Default: `""` + +```yml +# Examples + +password: foo +``` + +##### extract_headers + +Specify which response headers should be added to resulting messages as metadata. Header keys are lowercased before matching, so ensure that your patterns target lowercased versions of the header keys that you expect. + + +Type: `object` + +##### extract_headers.include_prefixes + +Provide a list of explicit metadata key prefixes to match against. + + +Type: `array` +Default: `[]` + +```yml expandable +# Examples + +include_prefixes: + - foo_ + - bar_ + +include_prefixes: + - kafka_ + +include_prefixes: + - content- +``` + +##### extract_headers.include_patterns + +Provide a list of explicit metadata key regular expression (re2) patterns to match against. + + +Type: `array` +Default: `[]` + +```yml +# Examples + +include_patterns: + - .* + +include_patterns: + - _timestamp_unix$ +``` + +##### timeout + +A static timeout to apply to requests. + + +Type: `string` +Default: `"5s"` + +##### retry_period + +The base period to wait between failed requests. + + +Type: `string` +Default: `"1s"` + +##### max_retry_backoff + +The maximum period to wait between failed requests. + + +Type: `string` +Default: `"300s"` + +##### retries + +The maximum number of retry attempts to make. + + +Type: `int` +Default: `3` + +##### backoff_on + +A list of status codes whereby the request should be considered to have failed and retries should be attempted, but the period between them should be increased gradually. + + +Type: `array` +Default: `[429]` + +##### drop_on + +A list of status codes whereby the request should be considered to have failed but retries should not be attempted. This is useful for preventing wasted retries for requests that will never succeed. Note that with these status codes the *request* is dropped, but *message* that caused the request will not be dropped. + + +Type: `array` +Default: `[]` + +##### successful_on + +A list of status codes whereby the attempt should be considered successful, this is useful for dropping requests that return non-2XX codes indicating that the message has been dealt with, such as a 303 See Other or a 409 Conflict. All 2XX codes are considered successful unless they are present within `backoff_on` or `drop_on`, regardless of this field. + + +Type: `array` +Default: `[]` + +##### proxy_url + +An optional HTTP proxy URL. + + +Type: `string` + +##### payload + +An optional payload to deliver for each request. +{/* TODO: when interpolation supported: +This field supports interpolation functions. */} + + +Type: `string` + +##### drop_empty_bodies + +Whether empty payloads received from the target server should be dropped. + + +Type: `bool` +Default: `true` + +##### stream + +Allows you to set streaming mode, where requests are kept open and messages are processed line-by-line. + + +Type: `object` + +##### stream.enabled + +Enables streaming mode. + + +Type: `bool` +Default: `false` + +##### stream.reconnect + +Sets whether to re-establish the connection once it is lost. + + +Type: `bool` +Default: `true` + + +##### auto_replay_nacks + +Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. + + +Type: `bool` +Default: `true` + +### HTTP Server + +Receive messages POSTed over HTTP(S). HTTP 2.0 is supported when using TLS, which is enabled when key and cert files are specified. + +#### Common + +```yml expandable +# Common config fields, showing default values +input: + label: "" + http_server: + address: "" + path: /post + ws_path: /post/ws + allowed_verbs: + - POST + timeout: 5s +``` + +#### Advanced + +```yml expandable +# All config fields, showing default values +input: + label: "" + http_server: + address: "" + path: /post + ws_path: /post/ws + ws_welcome_message: "" + allowed_verbs: + - POST + timeout: 5s + cert_file: "" + key_file: "" + cors: + enabled: false + allowed_origins: [] + sync_response: + status: "200" + headers: + Content-Type: application/octet-stream + metadata_headers: + include_prefixes: [] + include_patterns: [] +``` + +{/* TODO add link to service wide HTTP server If the `address` config field is left blank the [service-wide HTTP server](/docs/components/http/about) will be used. */} + +{/* TODO add rate limit The field `rate_limit` allows you to specify an optional [`rate_limit` resource](/docs/components/rate_limits/about), which will be applied to each HTTP request made and each websocket payload received. + +When the rate limit is breached HTTP requests will have a 429 response returned with a Retry-After header. Websocket payloads will be dropped and an optional response payload will be sent as per `ws_rate_limit_message`. */} + +##### Responses + +{/* TODO describe how to use synchronous responses when avail: + +It's possible to return a response for each message received using synchronous responses. When doing so you can customise headers with the `sync_response` field `headers`, which can also use function interpolation in the value based on the response message contents. */} + + +##### Endpoints + +The following fields specify endpoints that are registered for sending messages, and support path parameters of the form `/{foo}`, which are added to ingested messages as metadata. A path ending in `/` will match against all extensions of that path: + +###### path (defaults to `/post`) + +This endpoint expects POST requests where the entire request body is consumed as a single message. + +If the request contains a multipart `content-type` header as per [rfc1341](https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html) then the multiple parts are consumed as a batch of messages, where each body part is a message of the batch. + +###### ws_path (defaults to `/post/ws`) + +Creates a websocket connection, where payloads received on the socket are passed through the pipeline as a batch of one message. + +Please note that components within a Tyk Streams config will register their respective endpoints in a non-deterministic order. This means that establishing precedence of endpoints that are registered via multiple `http_server` inputs or outputs (either within brokers or from cohabiting streams) is not possible in a predictable way. + +This ambiguity makes it difficult to ensure that paths which are both a subset of a path registered by a separate component, and end in a slash (`/`) and will therefore match against all extensions of that path, do not prevent the more specific path from matching against requests. + +It is therefore recommended that you ensure paths of separate components do not collide unless they are explicitly non-competing. + +For example, if you were to deploy two separate `http_server` inputs, one with a path `/foo/` and the other with a path `/foo/bar`, it would not be possible to ensure that the path `/foo/` does not swallow requests made to `/foo/bar`. + +You may specify an optional `ws_welcome_message`, which is a static payload to be sent to all clients once a websocket connection is first established. + +##### Metadata + +This input adds the following metadata fields to each message: + +``` text expandable +- http_server_user_agent +- http_server_request_path +- http_server_verb +- http_server_remote_ip +- All headers (only first values are taken) +- All query parameters +- All path parameters +- All cookies +``` + +If HTTPS is enabled, the following fields are added as well: +``` text +- http_server_tls_version +- http_server_tls_subject +- http_server_tls_cipher_suite +``` + +{/* TODO: when interpolaion supported +You can access these metadata fields using interpolation functions. */} + +#### Examples + + +##### Path Switching + +This example shows an `http_server` input that captures all requests and processes them by switching on that path: + +```yaml expandable +input: + http_server: + path: / + allowed_verbs: [ GET, POST ] + sync_response: + headers: + Content-Type: application/json + + processors: + - switch: + - check: '@http_server_request_path == "/foo"' + processors: + - mapping: | + root.title = "You Got Fooed!" + root.result = content().string().uppercase() + + - check: '@http_server_request_path == "/bar"' + processors: + - mapping: 'root.title = "Bar Is Slow"' + - sleep: # Simulate a slow endpoint + duration: 1s +``` + +##### Mock OAuth 2.0 Server + +This example shows an `http_server` input that mocks an OAuth 2.0 Client Credentials flow server at the endpoint `/oauth2_test`: + +```yaml expandable +input: + http_server: + path: /oauth2_test + allowed_verbs: [ GET, POST ] + sync_response: + headers: + Content-Type: application/json + + processors: + - log: + message: "Received request" + level: INFO + fields_mapping: | + root = @ + root.body = content().string() + + - mapping: | + root.access_token = "MTQ0NjJkZmQ5OTM2NDE1ZTZjNGZmZjI3" + root.token_type = "Bearer" + root.expires_in = 3600 + + - sync_response: {} + - mapping: 'root = deleted()' +``` + +#### Fields + +##### address + +An alternative address to host from. If left empty the service wide address is used. + + +Type: `string` +Default: `""` + +##### path + +The endpoint path to listen for POST requests. + + +Type: `string` +Default: `"/post"` + +##### ws_path + +The endpoint path to create websocket connections from. + + +Type: `string` +Default: `"/post/ws"` + +##### ws_welcome_message + +An optional message to deliver to fresh websocket connections. + + +Type: `string` +Default: `""` + +##### allowed_verbs + +An array of verbs that are allowed for the `path` endpoint. + + +Type: `array` +Default: `["POST"]` +Requires version 3.33.0 or newer + +##### timeout + +Timeout for requests. If a consumed messages takes longer than this to be delivered the connection is closed, but the message may still be delivered. + + +Type: `string` +Default: `"5s"` + +{/* TODO add rate limit ##### rate_limit + +An optional [rate limit](/docs/components/rate_limits/about) to throttle requests by. */} + + +Type: `string` +Default: `""` + +##### cert_file + +Enable TLS by specifying a certificate and key file. Only valid with a custom `address`. + + +Type: `string` +Default: `""` + +##### key_file + +Enable TLS by specifying a certificate and key file. Only valid with a custom `address`. + + +Type: `string` +Default: `""` + +##### cors + +Adds Cross-Origin Resource Sharing headers. Only valid with a custom `address`. + + +Type: `object` +Requires version 3.63.0 or newer + +##### cors.enabled + +Whether to allow CORS requests. + + +Type: `bool` +Default: `false` + +##### cors.allowed_origins + +An explicit list of origins that are allowed for CORS requests. + + +Type: `array` +Default: `[]` + +##### sync_response + +{/* TODO add links to synchronous responses */} +Customize messages returned via synchronous responses. + + +Type: `object` + +##### sync_response.status + +Specify the status code to return with synchronous responses. This is a string value, which allows you to customize it based on resulting payloads and their metadata. +{/* TODO: when inerpolation supported: +This field supports interpolation functions. */} + + +Type: `string` +Default: `"200"` + +```yml +# Examples + +status: ${! json("status") } + +status: ${! meta("status") } +``` + +##### sync_response.headers + +Specify headers to return with synchronous responses. +{/* TODO: when interpolation supported: +This field supports interpolation functions. */} + + +Type: `object` +Default: `{"Content-Type":"application/octet-stream"}` + +##### sync_response.metadata_headers + +Specify criteria for which metadata values are added to the response as headers. + + +Type: `object` + +##### sync_response.metadata_headers.include_prefixes + +Provide a list of explicit metadata key prefixes to match against. + + +Type: `array` +Default: `[]` + +```yml expandable +# Examples + +include_prefixes: + - foo_ + - bar_ + +include_prefixes: + - kafka_ + +include_prefixes: + - content- +``` + +##### sync_response.metadata_headers.include_patterns + +Provide a list of explicit metadata key regular expression (re2) patterns to match against. + + +Type: `array` +Default: `[]` + +```yml +# Examples + +include_patterns: + - .* + +include_patterns: + - _timestamp_unix$ +``` + +### Kafka + +Connects to Kafka brokers and consumes one or more topics. + +#### Common + +```yml expandable +# Common config fields, showing default values +input: + label: "" + kafka: + addresses: [] # No default (required) + topics: [] # No default (required) + target_version: 2.1.0 # No default (optional) + consumer_group: "" + checkpoint_limit: 1024 + auto_replay_nacks: true +``` + +#### Advanced + +```yml expandable +# All config fields, showing default values +input: + label: "" + kafka: + addresses: [] # No default (required) + topics: [] # No default (required) + target_version: 2.1.0 # No default (optional) + tls: + enabled: false + skip_cert_verify: false + enable_renegotiation: false + root_cas: "" + root_cas_file: "" + client_certs: [] + sasl: + mechanism: none + user: "" + password: "" + access_token: "" + token_cache: "" + token_key: "" + consumer_group: "" + client_id: tyk + rack_id: "" + start_from_oldest: true + checkpoint_limit: 1024 + auto_replay_nacks: true + commit_period: 1s + max_processing_period: 100ms + extract_tracing_map: root = @ # No default (optional) + group: + session_timeout: 10s + heartbeat_interval: 3s + rebalance_timeout: 60s + fetch_buffer_cap: 256 + multi_header: false + batching: + count: 0 + byte_size: 0 + period: "" + check: "" + processors: [] # No default (optional) +``` + +Offsets are managed within Kafka under the specified consumer group, and partitions for each topic are automatically balanced across members of the consumer group. + +The Kafka input allows parallel processing of messages from different topic partitions, and messages of the same topic partition are processed with a maximum parallelism determined by the field [checkpoint_limit](#checkpoint_limit). + +In order to enforce ordered processing of partition messages set the [checkpoint_limit](#checkpoint_limit) to `1` and this will force partitions to be processed in lock-step, where a message will only be processed once the prior message is delivered. + +Batching messages before processing can be enabled using the [batching](#batching) field, and this batching is performed per-partition such that messages of a batch will always originate from the same partition. This batching mechanism is capable of creating batches of greater size than the [checkpoint_limit](#checkpoint_limit), in which case the next batch will only be created upon delivery of the current one. + +##### Metadata + +This input adds the following metadata fields to each message: + +``` text expandable +- kafka_key +- kafka_topic +- kafka_partition +- kafka_offset +- kafka_lag +- kafka_timestamp_unix +- kafka_tombstone_message +- All existing message headers (version 0.11+) +``` + +The field `kafka_lag` is the calculated difference between the high water mark offset of the partition at the time of ingestion and the current message offset. + +{/* TODO: when interpolation supported +You can access these metadata fields using function interpolation. */} + +##### Ordering + +By default messages of a topic partition can be processed in parallel, up to a limit determined by the field `checkpoint_limit`. However, if strict ordered processing is required then this value must be set to 1 in order to process shard messages in lock-step. When doing so it is recommended that you perform batching at this component for performance as it will not be possible to batch lock-stepped messages at the output level. + +##### Troubleshooting + +- I'm seeing logs that report `Failed to connect to kafka: kafka: client has run out of available brokers to talk to (Is your cluster reachable?)`, but the brokers are definitely reachable. + +Unfortunately this error message will appear for a wide range of connection problems even when the broker endpoint can be reached. Double check your authentication configuration and also ensure that you have [enabled TLS](#tlsenabled) if applicable. + +#### Fields + +##### addresses + +A list of broker addresses to connect to. If an item of the list contains commas it will be expanded into multiple addresses. + + +Type: `array` + +```yml expandable +# Examples + +addresses: + - localhost:9092 + +addresses: + - localhost:9041,localhost:9042 + +addresses: + - localhost:9041 + - localhost:9042 +``` + +##### topics + +A list of topics to consume from. Multiple comma separated topics can be listed in a single element. Partitions are automatically distributed across consumers of a topic. Alternatively, it's possible to specify explicit partitions to consume from with a colon after the topic name, e.g. `foo:0` would consume the partition 0 of the topic foo. This syntax supports ranges, e.g. `foo:0-10` would consume partitions 0 through to 10 inclusive. + + +Type: `array` +Requires version 3.33.0 or newer + +```yml expandable +# Examples + +topics: + - foo + - bar + +topics: + - foo,bar + +topics: + - foo:0 + - bar:1 + - bar:3 + +topics: + - foo:0,bar:1,bar:3 + +topics: + - foo:0-5 +``` + +##### target_version + +The version of the Kafka protocol to use. This limits the capabilities used by the client and should ideally match the version of your brokers. Defaults to the oldest supported stable version. + + +Type: `string` + +```yml +# Examples + +target_version: 2.1.0 + +target_version: 3.1.0 +``` + +##### tls + +Custom TLS settings can be used to override system defaults. + + +Type: `object` + +##### tls.enabled + +Whether custom TLS settings are enabled. + + +Type: `bool` +Default: `false` + +##### tls.skip_cert_verify + +Whether to skip server side certificate verification. + + +Type: `bool` +Default: `false` + +##### tls.enable_renegotiation + +Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. + + +Type: `bool` +Default: `false` +Requires version 3.45.0 or newer + +##### tls.root_cas + +An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. + +{/* TO DO add secrets link :::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: */} + + +Type: `string` +Default: `""` + +```yml +# Examples + +root_cas: |- + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- +``` + +##### tls.root_cas_file + +An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. + + +Type: `string` +Default: `""` + +```yml +# Examples + +root_cas_file: ./root_cas.pem +``` + +##### tls.client_certs + +A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. + + +Type: `array` +Default: `[]` + +```yml expandable +# Examples + +client_certs: + - cert: foo + key: bar + +client_certs: + - cert_file: ./example.pem + key_file: ./example.key +``` + +##### tls.client_certs[].cert + +A plain text certificate to use. + + +Type: `string` +Default: `""` + +##### tls.client_certs[].key + +A plain text certificate key to use. + +{/* TODO add secrets link :::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: */} + + +Type: `string` +Default: `""` + +##### tls.client_certs[].cert_file + +The path of a certificate to use. + + +Type: `string` +Default: `""` + +##### tls.client_certs[].key_file + +The path of a certificate key to use. + + +Type: `string` +Default: `""` + +##### tls.client_certs[].password + +A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format. Warning: Since it does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext. + + +Type: `string` +Default: `""` + +```yml +# Example + +password: foo +``` + +{/* When Tyk streams with secrets released include this in above example => password: ${KEY_PASSWORD} */} + +##### sasl + +Enables SASL authentication. + + +Type: `object` + +##### sasl.mechanism + +The SASL authentication mechanism, if left empty SASL authentication is not used. + + +Type: `string` +Default: `"none"` + +| Option | Summary | +| :--- | :--- | +| `OAUTHBEARER` | OAuth Bearer based authentication. | +| `PLAIN` | Plain text authentication. NOTE: When using plain text auth it is extremely likely that you'll also need to [enable TLS](#tlsenabled). | +| `SCRAM-SHA-256` | Authentication using the SCRAM-SHA-256 mechanism. | +| `SCRAM-SHA-512` | Authentication using the SCRAM-SHA-512 mechanism. | +| `none` | Default, no SASL authentication. | + + +##### sasl.user + +A PLAIN username. It is recommended that you use environment variables to populate this field. + + +Type: `string` +Default: `""` + +```yml +# Examples + +user: ${USER} +``` + +##### sasl.password + +A PLAIN password. It is recommended that you use environment variables to populate this field. +{/* TODO add secret link :::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: */} + + +Type: `string` +Default: `""` + +```yml +# Examples + +password: ${PASSWORD} +``` + +##### sasl.access_token + +A static OAUTHBEARER access token + + +Type: `string` +Default: `""` + +{/* TODO add ##### sasl.token_cache + +Instead of using a static `access_token` allows you to query a [`cache`](/docs/components/caches/about) resource to fetch OAUTHBEARER tokens from */} + + +Type: `string` +Default: `""` + +##### sasl.token_key + +Required when using a `token_cache`, the key to query the cache with for tokens. + + +Type: `string` +Default: `""` + +##### consumer_group + +An identifier for the consumer group of the connection. This field can be explicitly made empty in order to disable stored offsets for the consumed topic partitions. + + +Type: `string` +Default: `""` + +##### client_id + +An identifier for the client connection. + + +Type: `string` +Default: `"tyk"` + +##### rack_id + +A rack identifier for this client. + + +Type: `string` +Default: `""` + +##### start_from_oldest + +Determines whether to consume from the oldest available offset, otherwise messages are consumed from the latest offset. The setting is applied when creating a new consumer group or the saved offset no longer exists. + + +Type: `bool` +Default: `true` + +##### checkpoint_limit + +The maximum number of messages of the same topic and partition that can be processed at a given time. Increasing this limit enables parallel processing and batching at the output level to work on individual partitions. Any given offset will not be committed unless all messages under that offset are delivered in order to preserve at least once delivery guarantees. + + +Type: `int` +Default: `1024` +Requires version 3.33.0 or newer + +##### auto_replay_nacks + +Whether messages that are rejected (nacked) at the output level should be automatically replayed indefinitely, eventually resulting in back pressure if the cause of the rejections is persistent. If set to `false` these messages will instead be deleted. Disabling auto replays can greatly improve memory efficiency of high throughput streams as the original shape of the data can be discarded immediately upon consumption and mutation. + + +Type: `bool` +Default: `true` + +##### commit_period + +The period of time between each commit of the current partition offsets. Offsets are always committed during shutdown. + + +Type: `string` +Default: `"1s"` + +##### max_processing_period + +A maximum estimate for the time taken to process a message, this is used for tuning consumer group synchronization. + + +Type: `string` +Default: `"100ms"` + +{/* TODO: when bloblang is supported +##### extract_tracing_map + +A Bloblang mapping that attempts to extract an object containing tracing propagation information, which will then be used as the root tracing span for the message. The specification of the extracted fields must match the format used by the service wide tracer. + + +Type: `string` +Requires version 3.45.0 or newer + +```yml +# Examples + +extract_tracing_map: root = @ + +extract_tracing_map: root = this.meta.span +``` */} + +##### group + +Tuning parameters for consumer group synchronization. + + +Type: `object` + +##### group.session_timeout + +A period after which a consumer of the group is kicked after no heartbeats. + + +Type: `string` +Default: `"10s"` + +##### group.heartbeat_interval + +A period in which heartbeats should be sent out. + + +Type: `string` +Default: `"3s"` + +##### group.rebalance_timeout + +A period after which rebalancing is abandoned if unresolved. + + +Type: `string` +Default: `"60s"` + +##### fetch_buffer_cap + +The maximum number of unprocessed messages to fetch at a given time. + + +Type: `int` +Default: `256` + +##### multi_header + +Decode headers into lists to allow handling of multiple values with the same key + + +Type: `bool` +Default: `false` + +##### batching + +Allows you to configure a [batching policy](/api-management/stream-config#batch-policy). + +Type: `object` + +```yml expandable +# Examples + +batching: + byte_size: 5000 + count: 0 + period: 1s + +batching: + count: 10 + period: 1s + +batching: + check: this.contains("END BATCH") + count: 0 + period: 1m +``` + +##### batching.count + +A number of messages at which the batch should be flushed. If `0` disables count based batching. + + +Type: `int` +Default: `0` + +##### batching.byte_size + +An amount of bytes at which the batch should be flushed. If `0` disables size based batching. + + +Type: `int` +Default: `0` + +##### batching.period + +A period in which an incomplete batch should be flushed regardless of its size. + + +Type: `string` +Default: `""` + +```yml +# Examples + +period: 1s + +period: 1m + +period: 500ms +``` + +{/* TODO: when bloblang is supported +##### batching.check + +A Bloblang query that should return a boolean value indicating whether a message should end a batch. + +Type: `string` +Default: `""` + +```yml +# Examples + +check: this.type == "end_of_transaction" +``` */} + +##### batching.processors + +A list of processors to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op. + + +Type: `array` + +```yml expandable +# Examples + +processors: + - archive: + format: concatenate + +processors: + - archive: + format: lines + +processors: + - archive: + format: json_array +``` + +## Outputs + +### Overview + +An output is a sink where we wish to send our consumed data after applying an optional array of [processors](/api-management/stream-config#overview-3). Only one output is configured at the root of a Tyk Streams config. However, the output can be a [broker](/api-management/stream-config#broker-1) which combines multiple outputs under a chosen brokering pattern. + +An output config section looks like this: + +```yaml expandable +outout: + label: my_kafka_output + + kafka: + addresses: [ localhost:9092 ] + topic: "foobar" + + # Optional list of processing steps + processors: + - avro: + operator: from_json +``` + +#### Labels + +Outputs have an optional field `label` that can uniquely identify them in observability data such as logs. + +{/* TODO replace with this paragraph when determine if product supports metrics + +Outputs have an optional field `label` that can uniquely identify them in observability data such as metrics and logs. This can be useful when running configs with multiple outputs, otherwise their metrics labels will be generated based on their composition. For more information check out the [metrics documentation][metrics.about]. */} + +### Broker + +Allows you to route messages to multiple child outputs using a range of brokering [patterns](#patterns). + +#### Common + +```yml expandable +# Common config fields, showing default values +output: + label: "" + broker: + pattern: fan_out + outputs: [] # No default (required) + batching: + count: 0 + byte_size: 0 + period: "" + check: "" +``` + +#### Advanced + +```yml expandable +# All config fields, showing default values +output: + label: "" + broker: + copies: 1 + pattern: fan_out + outputs: [] # No default (required) + batching: + count: 0 + byte_size: 0 + period: "" + check: "" + processors: [] # No default (optional) +``` + +Processors can be listed to apply across individual outputs or all outputs: + +```yaml expandable +output: + broker: + pattern: fan_out + outputs: + - resource: foo + - resource: bar + # Processors only applied to messages sent to bar. + processors: + - resource: bar_processor + + # Processors applied to messages sent to all brokered outputs. + processors: + - resource: general_processor +``` + +#### Fields + +##### copies + +The number of copies of each configured output to spawn. + + +Type: `int` +Default: `1` + +##### pattern + +The brokering pattern to use. + + +Type: `string` +Default: `"fan_out"` +Options: `fan_out`, `fan_out_fail_fast`, `fan_out_sequential`, `fan_out_sequential_fail_fast`, `round_robin`, `greedy`. + +##### outputs + +A list of child outputs to broker. + + +Type: `array` + +##### batching + +Allows you to configure a [batching policy](/api-management/stream-config#batch-policy). + + +Type: `object` + +```yml expandable +# Examples + +batching: + byte_size: 5000 + count: 0 + period: 1s + +batching: + count: 10 + period: 1s + +batching: + check: this.contains("END BATCH") + count: 0 + period: 1m +``` + +##### batching.count + +A number of messages at which the batch should be flushed. If `0` disables count based batching. + + +Type: `int` +Default: `0` + +##### batching.byte_size + +An amount of bytes at which the batch should be flushed. If `0` disables size based batching. + + +Type: `int` +Default: `0` + +##### batching.period + +A period in which an incomplete batch should be flushed regardless of its size. + + +Type: `string` +Default: `""` + +```yml +# Examples + +period: 1s + +period: 1m + +period: 500ms +``` + +{/* TODO: when bloblang is supported +##### batching.check + +A Bloblang query that should return a boolean value indicating whether a message should end a batch. + + +Type: `string` +Default: `""` + +```yml +# Examples + +check: this.type == "end_of_transaction" +``` */} + +##### batching.processors + +A list of processors to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op. + + +Type: `array` + +```yml expandable +# Examples + +processors: + - archive: + format: concatenate + +processors: + - archive: + format: lines + +processors: + - archive: + format: json_array +``` + +#### Patterns + +The broker pattern determines the way in which messages are allocated and can be chosen from the following: + +##### fan_out + +With the fan out pattern all outputs will be sent every message that passes through Tyk Streams in parallel. + +If an output applies back pressure it will block all subsequent messages, and if an output fails to send a message it will be retried continuously until completion or service shut down. This mechanism is in place in order to prevent one bad output from causing a larger retry loop that results in a good output from receiving unbounded message duplicates. + +##### fan_out_fail_fast + +The same as the `fan_out` pattern, except that output failures will not be automatically retried. This pattern should be used with caution as busy retry loops could result in unlimited duplicates being introduced into the non-failure outputs. + +##### fan_out_sequential + +Similar to the fan out pattern except outputs are written to sequentially, meaning an output is only written to once the preceding output has confirmed receipt of the same message. + +If an output applies back pressure it will block all subsequent messages, and if an output fails to send a message it will be retried continuously until completion or service shut down. This mechanism is in place in order to prevent one bad output from causing a larger retry loop that results in a good output from receiving unbounded message duplicates. + +##### fan_out_sequential_fail_fast + +The same as the `fan_out_sequential` pattern, except that output failures will not be automatically retried. This pattern should be used with caution as busy retry loops could result in unlimited duplicates being introduced into the non-failure outputs. + +##### round_robin + +With the round robin pattern each message will be assigned a single output following their order. If an output applies back pressure it will block all subsequent messages. If an output fails to send a message then the message will be re-attempted with the next input, and so on. + +##### greedy + +The greedy pattern results in higher output throughput at the cost of potentially disproportionate message allocations to those outputs. Each message is sent to a single output, which is determined by allowing outputs to claim messages as soon as they are able to process them. This results in certain faster outputs potentially processing more messages at the cost of slower outputs. + + +### HTTP Client + +Sends messages to an HTTP server. + +#### Common + +```yml expandable +# Common config fields, showing default values +output: + label: "" + http_client: + url: "" # No default (required) + verb: POST + headers: {} + timeout: 5s + max_in_flight: 64 + batching: + count: 0 + byte_size: 0 + period: "" + check: "" +``` + +#### Advanced + +```yml expandable +# All config fields, showing default values +output: + label: "" + http_client: + url: "" # No default (required) + verb: POST + headers: {} + metadata: + include_prefixes: [] + include_patterns: [] + dump_request_log_level: "" + oauth: + enabled: false + consumer_key: "" + consumer_secret: "" + access_token: "" + access_token_secret: "" + oauth2: + enabled: false + client_key: "" + client_secret: "" + token_url: "" + scopes: [] + endpoint_params: {} + basic_auth: + enabled: false + username: "" + password: "" + jwt: + enabled: false + private_key_file: "" + signing_method: "" + claims: {} + headers: {} + tls: + enabled: false + skip_cert_verify: false + enable_renegotiation: false + root_cas: "" + root_cas_file: "" + client_certs: [] + extract_headers: + include_prefixes: [] + include_patterns: [] + timeout: 5s + retry_period: 1s + max_retry_backoff: 300s + retries: 3 + backoff_on: + - 429 + drop_on: [] + successful_on: [] + proxy_url: "" # No default (optional) + batch_as_multipart: false + propagate_response: false + max_in_flight: 64 + batching: + count: 0 + byte_size: 0 + period: "" + check: "" + processors: [] # No default (optional) + multipart: [] +``` + +When the number of retries expires the output will reject the message, the behavior after this will depend on the pipeline but usually this simply means the send is attempted again until successful whilst applying back pressure. + +{/* TODO: when interpolation supported +The URL and header values of this type can be dynamically set using function interpolations. */} + +The body of the HTTP request is the raw contents of the message payload. If the message has multiple parts (is a batch) the request will be sent according to [RFC1341](https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html). This behavior can be disabled by setting the field [batch_as_multipart](#batch_as_multipart) to `false`. + +##### Propagating Responses + +It's possible to propagate the response from each HTTP request back to the input source by setting `propagate_response` to `true`. Only inputs that support synchronous responses are able to make use of these propagated responses. + +#### Performance + +This output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`. + +This output benefits from sending messages as a [batch](/api-management/stream-config#batching-6) for improved performance. Batches can be formed at both the input and output level. + +#### Fields + +##### url + +The URL to connect to. +{/* TODO: when interpolation supported +This field supports interpolation functions. */} + +Type: `string` + +##### verb + +A verb to connect with + + +Type: `string` +Default: `"POST"` + +```yml +# Examples + +verb: POST + +verb: GET + +verb: DELETE +``` + +##### headers + +A map of headers to add to the request. +{/* TODO: when interpolation supported +This field supports interpolation functions. */} + + +Type: `object` +Default: `{}` + +```yml +# Examples + +headers: + Content-Type: application/octet-stream + traceparent: ${! tracing_span().traceparent } +``` + +##### metadata + +Specify optional matching rules to determine which metadata keys should be added to the HTTP request as headers. + + +Type: `object` + +##### metadata.include_prefixes + +Provide a list of explicit metadata key prefixes to match against. + + +Type: `array` +Default: `[]` + +```yml expandable +# Examples + +include_prefixes: + - foo_ + - bar_ + +include_prefixes: + - kafka_ + +include_prefixes: + - content- +``` + +##### metadata.include_patterns + +Provide a list of explicit metadata key regular expression (re2) patterns to match against. + + +Type: `array` +Default: `[]` + +```yml +# Examples + +include_patterns: + - .* + +include_patterns: + - _timestamp_unix$ +``` + +##### dump_request_log_level + +Optionally set a level at which the request and response payload of each request made will be logged. + + +Type: `string` +Default: `""` +Options: `TRACE`, `DEBUG`, `INFO`, `WARN`, `ERROR`, `FATAL`, ``. + +##### oauth + +Allows you to specify open authentication via OAuth version 1. + + +Type: `object` + +##### oauth.enabled + +Whether to use OAuth version 1 in requests. + + +Type: `bool` +Default: `false` + +##### oauth.consumer_key + +A value used to identify the client to the service provider. + + +Type: `string` +Default: `""` + +##### oauth.consumer_secret + +A secret used to establish ownership of the consumer key. + + +Type: `string` +Default: `""` + +##### oauth.access_token + +A value used to gain access to the protected resources on behalf of the user. + + +Type: `string` +Default: `""` + +##### oauth.access_token_secret + +A secret provided in order to establish ownership of a given access token. + + +Type: `string` +Default: `""` + +##### oauth2 + +Allows you to specify open authentication via OAuth version 2 using the client credentials token flow. + + +Type: `object` + +##### oauth2.enabled + +Whether to use OAuth version 2 in requests. + + +Type: `bool` +Default: `false` + +##### oauth2.client_key + +A value used to identify the client to the token provider. + + +Type: `string` +Default: `""` + +##### oauth2.client_secret + +A secret used to establish ownership of the client key. + + +Type: `string` +Default: `""` + +##### oauth2.token_url + +The URL of the token provider. + + +Type: `string` +Default: `""` + +##### oauth2.scopes + +A list of optional requested permissions. + + +Type: `array` +Default: `[]` + +##### oauth2.endpoint_params + +A list of optional endpoint parameters, values should be arrays of strings. + + +Type: `object` +Default: `{}` + +```yml expandable +# Examples + +endpoint_params: + bar: + - woof + foo: + - meow + - quack +``` + +##### basic_auth + +Allows you to specify basic authentication. + + +Type: `object` + +##### basic_auth.enabled + +Whether to use basic authentication in requests. + + +Type: `bool` +Default: `false` + +##### basic_auth.username + +A username to authenticate as. + + +Type: `string` +Default: `""` + +##### basic_auth.password + +A password to authenticate with. + + +Type: `string` +Default: `""` + +##### jwt + +Allows you to specify JWT authentication. + + +Type: `object` + +##### jwt.enabled + +Whether to use JWT authentication in requests. + + +Type: `bool` +Default: `false` + +##### jwt.private_key_file + +A file with the PEM encoded via PKCS1 or PKCS8 as private key. + + +Type: `string` +Default: `""` + +##### jwt.signing_method + +A method used to sign the token such as RS256, RS384, RS512 or EdDSA. + + +Type: `string` +Default: `""` + +##### jwt.claims + +A value used to identify the claims that issued the JWT. + + +Type: `object` +Default: `{}` + +##### jwt.headers + +Add optional key/value headers to the JWT. + + +Type: `object` +Default: `{}` + +##### tls + +Custom TLS settings can be used to override system defaults. + + +Type: `object` + +##### tls.enabled + +Whether custom TLS settings are enabled. + + +Type: `bool` +Default: `false` + +##### tls.skip_cert_verify + +Whether to skip server side certificate verification. + + +Type: `bool` +Default: `false` + +##### tls.enable_renegotiation + +Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. + + +Type: `bool` +Default: `false` + +##### tls.root_cas + +An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. + + +Type: `string` +Default: `""` + +```yml +# Examples + +root_cas: |- + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- +``` + +##### tls.root_cas_file + +An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. + + +Type: `string` +Default: `""` + +```yml +# Examples + +root_cas_file: ./root_cas.pem +``` + +##### tls.client_certs + +A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. + + +Type: `array` +Default: `[]` + +```yml expandable +# Examples + +client_certs: + - cert: foo + key: bar + +client_certs: + - cert_file: ./example.pem + key_file: ./example.key +``` + +##### tls.client_certs[].cert + +A plain text certificate to use. + + +Type: `string` +Default: `""` + +##### tls.client_certs[].key + +A plain text certificate key to use. + + +Type: `string` +Default: `""` + +##### tls.client_certs[].cert_file + +The path of a certificate to use. + + +Type: `string` +Default: `""` + +##### tls.client_certs[].key_file + +The path of a certificate key to use. + + +Type: `string` +Default: `""` + +##### tls.client_certs[].password + +A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format. Warning: Since it does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext. + + +Type: `string` +Default: `""` + +```yml +# Examples + +password: foo +``` + +##### extract_headers + +Specify which response headers should be added to resulting synchronous response messages as metadata. Header keys are lowercased before matching, so ensure that your patterns target lowercased versions of the header keys that you expect. This field is not applicable unless `propagate_response` is set to `true`. + + +Type: `object` + +##### extract_headers.include_prefixes + +Provide a list of explicit metadata key prefixes to match against. + + +Type: `array` +Default: `[]` + +```yml expandable +# Examples + +include_prefixes: + - foo_ + - bar_ + +include_prefixes: + - kafka_ + +include_prefixes: + - content- +``` + +##### extract_headers.include_patterns + +Provide a list of explicit metadata key regular expression (re2) patterns to match against. + + +Type: `array` +Default: `[]` + +```yml +# Examples + +include_patterns: + - .* + +include_patterns: + - _timestamp_unix$ +``` + +##### timeout + +A static timeout to apply to requests. + + +Type: `string` +Default: `"5s"` + +##### retry_period + +The base period to wait between failed requests. + + +Type: `string` +Default: `"1s"` + +##### max_retry_backoff + +The maximum period to wait between failed requests. + + +Type: `string` +Default: `"300s"` + +##### retries + +The maximum number of retry attempts to make. + + +Type: `int` +Default: `3` + +##### backoff_on + +A list of status codes whereby the request should be considered to have failed and retries should be attempted, but the period between them should be increased gradually. + + +Type: `array` +Default: `[429]` + +##### drop_on + +A list of status codes whereby the request should be considered to have failed but retries should not be attempted. This is useful for preventing wasted retries for requests that will never succeed. Note that with these status codes the _request_ is dropped, but _message_ that caused the request will not be dropped. + + +Type: `array` +Default: `[]` + +##### successful_on + +A list of status codes whereby the attempt should be considered successful, this is useful for dropping requests that return non-2XX codes indicating that the message has been dealt with, such as a 303 See Other or a 409 Conflict. All 2XX codes are considered successful unless they are present within `backoff_on` or `drop_on`, regardless of this field. + + +Type: `array` +Default: `[]` + +##### proxy_url + +An optional HTTP proxy URL. + + +Type: `string` + +##### batch_as_multipart + +Send message batches as a single request using [RFC1341](https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html). If disabled messages in batches will be sent as individual requests. + + +Type: `bool` +Default: `false` + +##### propagate_response + +Whether responses from the server should be propagated back to the input. + + +Type: `bool` +Default: `false` + +##### max_in_flight + +The maximum number of parallel message batches to have in flight at any given time. + + +Type: `int` +Default: `64` + +##### batching + +Allows you to configure a [batching policy](/api-management/stream-config#batching-6). + + +Type: `object` + +```yml expandable +# Examples + +batching: + byte_size: 5000 + count: 0 + period: 1s + +batching: + count: 10 + period: 1s + +batching: + check: this.contains("END BATCH") + count: 0 + period: 1m +``` + +##### batching.count + +A number of messages at which the batch should be flushed. If `0` disables count based batching. + + +Type: `int` +Default: `0` + +##### batching.byte_size + +An amount of bytes at which the batch should be flushed. If `0` disables size based batching. + + +Type: `int` +Default: `0` + +##### batching.period + +A period in which an incomplete batch should be flushed regardless of its size. + + +Type: `string` +Default: `""` + +```yml +# Examples + +period: 1s + +period: 1m + +period: 500ms +``` + +{/* TODO: when bloblang supported +##### batching.check + +A Bloblang query that should return a boolean value indicating whether a message should end a batch. + + +Type: `string` +Default: `""` + +```yml +# Examples + +check: this.type == "end_of_transaction" +``` */} + +##### batching.processors + +A list of processors to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op. + + +Type: `array` + +```yml expandable +# Examples + +processors: + - archive: + format: concatenate + +processors: + - archive: + format: lines + +processors: + - archive: + format: json_array +``` + +##### multipart + +Create explicit multipart HTTP requests by specifying an array of parts to add to the request, each part specified consists of content headers and a data field that can be populated dynamically. If this field is populated it will override the default request creation behavior. + + +Type: `array` +Default: `[]` + +##### multipart[].content_type + +The content type of the individual message part. +{/* TODO: when interpolation supported +This field supports interpolation functions. */} + + +Type: `string` +Default: `""` + +```yml +# Examples + +content_type: application/bin +``` + +##### multipart[].content_disposition + +The content disposition of the individual message part. +{/* TODO: when interpolation supported +This field supports interpolation functions. */} + + +Type: `string` +Default: `""` + +```yml +# Examples + +content_disposition: form-data; name="bin"; filename='${! @AttachmentName } +``` + +##### multipart[].body + +The body of the individual message part. +{/* TODO: when interpolation supported +This field supports interpolation functions. */} + + +Type: `string` +Default: `""` + +```yml +# Examples + +body: ${! this.data.part1 } +``` + +### HTTP Server + +Sets up an HTTP server that will send messages over HTTP(S) GET requests. HTTP 2.0 is supported when using TLS, which is enabled when key and cert files are specified. + +#### Common + +```yml expandable +# Common config fields, showing default values +output: + label: "" + http_server: + address: "" + path: /get + stream_path: /get/stream + ws_path: /get/ws + allowed_verbs: + - GET +``` + +#### Advanced + +```yml expandable +# All config fields, showing default values +output: + label: "" + http_server: + address: "" + path: /get + stream_path: /get/stream + ws_path: /get/ws + allowed_verbs: + - GET + timeout: 5s + cert_file: "" + key_file: "" + cors: + enabled: false + allowed_origins: [] +``` + +Sets up an HTTP server that will send messages over HTTP(S) GET requests. + +{/* TODO add link here If the `address` config field is left blank the [service-wide HTTP server](/docs/components/http/about) will be used. */} + +Three endpoints will be registered at the paths specified by the fields `path`, `stream_path` and `ws_path`. Which allow you to consume a single message batch, a continuous stream of line delimited messages, or a websocket of messages for each request respectively. + +When messages are batched the `path` endpoint encodes the batch according to [RFC1341](https://www.w3.org/Protocols/rfc1341/7_2_Multipart.html). + +{/* TODO add link here - This behavior can be overridden by [archiving your batches](/docs/configuration/batching#post-batch-processing). */} + +Please note, messages are considered delivered as soon as the data is written to the client. There is no concept of at least once delivery on this output. + +Please note that components within a Tyk config will register their respective endpoints in a non-deterministic order. This means that establishing precedence of endpoints that are registered via multiple `http_server` inputs or outputs (either within brokers or from cohabiting streams) is not possible in a predictable way. + +This ambiguity makes it difficult to ensure that paths which are both a subset of a path registered by a separate component, and end in a slash (`/`) and will therefore match against all extensions of that path, do not prevent the more specific path from matching against requests. + +It is therefore recommended that you ensure paths of separate components do not collide unless they are explicitly non-competing. + +For example, if you were to deploy two separate `http_server` inputs, one with a path `/foo/` and the other with a path `/foo/bar`, it would not be possible to ensure that the path `/foo/` does not swallow requests made to `/foo/bar`. + + +#### Fields + +##### address + +An alternative address to host from. If left empty the service wide address is used. + + +Type: `string` +Default: `""` + +##### path + +The path from which discrete messages can be consumed. + + +Type: `string` +Default: `"/get"` + +##### stream_path + +The path from which a continuous stream of messages can be consumed. + + +Type: `string` +Default: `"/get/stream"` + +##### ws_path + +The path from which websocket connections can be established. + + +Type: `string` +Default: `"/get/ws"` + +##### allowed_verbs + +An array of verbs that are allowed for the `path` and `stream_path` HTTP endpoint. + + +Type: `array` +Default: `["GET"]` + +##### timeout + +The maximum time to wait before a blocking, inactive connection is dropped (only applies to the `path` endpoint). + + +Type: `string` +Default: `"5s"` + +##### cert_file + +Enable TLS by specifying a certificate and key file. Only valid with a custom `address`. + + +Type: `string` +Default: `""` + +##### key_file + +Enable TLS by specifying a certificate and key file. Only valid with a custom `address`. + + +Type: `string` +Default: `""` + +##### cors + +Adds Cross-Origin Resource Sharing headers. Only valid with a custom `address`. + + +Type: `object` + +##### cors.enabled + +Whether to allow CORS requests. + + +Type: `bool` +Default: `false` + +##### cors.allowed_origins + +An explicit list of origins that are allowed for CORS requests. + + +Type: `array` +Default: `[]` + + + +### Kafka + +The kafka output type writes a batch of messages to Kafka brokers and waits for acknowledgment before propagating it back to the input. + +#### Common + +```yml expandable +# Common config fields, showing default values +output: + label: "" + kafka: + addresses: [] # No default (required) + topic: "" # No default (required) + target_version: 2.1.0 # No default (optional) + key: "" + partitioner: fnv1a_hash + compression: none + static_headers: {} # No default (optional) + metadata: + exclude_prefixes: [] + max_in_flight: 64 + batching: + count: 0 + byte_size: 0 + period: "" + check: "" +``` + +#### Advanced + +```yml expandable +# All config fields, showing default values +output: + label: "" + kafka: + addresses: [] # No default (required) + tls: + enabled: false + skip_cert_verify: false + enable_renegotiation: false + root_cas: "" + root_cas_file: "" + client_certs: [] + sasl: + mechanism: none + user: "" + password: "" + access_token: "" + token_cache: "" + token_key: "" + topic: "" # No default (required) + client_id: tyk + target_version: 2.1.0 # No default (optional) + rack_id: "" + key: "" + partitioner: fnv1a_hash + partition: "" + custom_topic_creation: + enabled: false + partitions: -1 + replication_factor: -1 + compression: none + static_headers: {} # No default (optional) + metadata: + exclude_prefixes: [] + inject_tracing_map: meta = @.merge(this) # No default (optional) + max_in_flight: 64 + idempotent_write: false + ack_replicas: false + max_msg_bytes: 1000000 + timeout: 5s + retry_as_batch: false + batching: + count: 0 + byte_size: 0 + period: "" + check: "" + processors: [] # No default (optional) + max_retries: 0 + backoff: + initial_interval: 3s + max_interval: 10s + max_elapsed_time: 30s +``` + +The config field `ack_replicas` determines whether we wait for acknowledgment from all replicas or just a single broker. + +{/* Add links to bloblang queries : Both the `key` and `topic` fields can be dynamically set using function interpolations described [here](/docs/configuration/interpolation#bloblang-queries). */} + +Metadata will be added to each message sent as headers (version 0.11+), but can be restricted using the field [metadata](#metadata). + +##### Strict Ordering and Retries + +When strict ordering is required for messages written to topic partitions it is important to ensure that both the field `max_in_flight` is set to `1` and that the field `retry_as_batch` is set to `true`. + +You must also ensure that failed batches are never rerouted back to the same output. This can be done by setting the field `max_retries` to `0` and `backoff.max_elapsed_time` to empty, which will apply back pressure indefinitely until the batch is sent successfully. + +{/* TODO: Add link to fallback broker */} +However, this also means that manual intervention will eventually be required in cases where the batch cannot be sent due to configuration problems such as an incorrect `max_msg_bytes` estimate. A less strict but automated alternative would be to route failed batches to a dead letter queue using a `fallback` broker, but this would allow subsequent batches to be delivered in the meantime whilst those failed batches are dealt with. + +##### Troubleshooting + +- I'm seeing logs that report `Failed to connect to kafka: kafka: client has run out of available brokers to talk to (Is your cluster reachable?)`, but the brokers are definitely reachable. + +Unfortunately this error message will appear for a wide range of connection problems even when the broker endpoint can be reached. Double check your authentication configuration and also ensure that you have [enabled TLS](#tlsenabled) if applicable. + +#### Performance + +This output benefits from sending multiple messages in flight in parallel for improved performance. You can tune the max number of in flight messages (or message batches) with the field `max_in_flight`. + +This output benefits from sending messages as a [batch](/api-management/stream-config#batching-6) for improved performance. Batches can be formed at both the input and output level. + +#### Fields + +##### addresses + +A list of broker addresses to connect to. If an item of the list contains commas it will be expanded into multiple addresses. + + +Type: `array` + +```yml expandable +# Examples + +addresses: + - localhost:9092 + +addresses: + - localhost:9041,localhost:9042 + +addresses: + - localhost:9041 + - localhost:9042 +``` + +##### tls + +Custom TLS settings can be used to override system defaults. + + +Type: `object` + +##### tls.enabled + +Whether custom TLS settings are enabled. + + +Type: `bool` +Default: `false` + +##### tls.skip_cert_verify + +Whether to skip server side certificate verification. + + +Type: `bool` +Default: `false` + +##### tls.enable_renegotiation + +Whether to allow the remote server to repeatedly request renegotiation. Enable this option if you're seeing the error message `local error: tls: no renegotiation`. + + +Type: `bool` +Default: `false` + +##### tls.root_cas + +An optional root certificate authority to use. This is a string, representing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. +{/* TODO add secrets link :::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: */} + + +Type: `string` +Default: `""` + +```yml +# Examples + +root_cas: |- + -----BEGIN CERTIFICATE----- + ... + -----END CERTIFICATE----- +``` + +##### tls.root_cas_file + +An optional path of a root certificate authority file to use. This is a file, often with a .pem extension, containing a certificate chain from the parent trusted root certificate, to possible intermediate signing certificates, to the host certificate. + + +Type: `string` +Default: `""` + +```yml +# Examples + +root_cas_file: ./root_cas.pem +``` + +##### tls.client_certs + +A list of client certificates to use. For each certificate either the fields `cert` and `key`, or `cert_file` and `key_file` should be specified, but not both. + + +Type: `array` +Default: `[]` + +```yml expandable +# Examples + +client_certs: + - cert: foo + key: bar + +client_certs: + - cert_file: ./example.pem + key_file: ./example.key +``` + +##### tls.client_certs[].cert + +A plain text certificate to use. + + +Type: `string` +Default: `""` + +##### tls.client_certs[].key + +A plain text certificate key to use. +{/* TODO: add secrets link :::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: */} + + +Type: `string` +Default: `""` + +##### tls.client_certs[].cert_file + +The path of a certificate to use. + + +Type: `string` +Default: `""` + +##### tls.client_certs[].key_file + +The path of a certificate key to use. + + +Type: `string` +Default: `""` + +##### tls.client_certs[].password + +A plain text password for when the private key is password encrypted in PKCS#1 or PKCS#8 format. The obsolete `pbeWithMD5AndDES-CBC` algorithm is not supported for the PKCS#8 format. Warning: Since it does not authenticate the ciphertext, it is vulnerable to padding oracle attacks that can let an attacker recover the plaintext. + + +Type: `string` +Default: `""` + +```yml +# Example + +password: foo +``` + +{/* When Tyk streams with secrets released include this in above example => password: ${KEY_PASSWORD} */} + +##### sasl + +Enables SASL authentication. + + +Type: `object` + +##### sasl.mechanism + +The SASL authentication mechanism, if left empty SASL authentication is not used. + + +Type: `string` +Default: `"none"` + +| Option | Summary | +| :--- | :--- | +| `OAUTHBEARER` | OAuth Bearer based authentication. | +| `PLAIN` | Plain text authentication. NOTE: When using plain text auth it is extremely likely that you'll also need to [enable TLS](#tlsenabled). | +| `SCRAM-SHA-256` | Authentication using the SCRAM-SHA-256 mechanism. | +| `SCRAM-SHA-512` | Authentication using the SCRAM-SHA-512 mechanism. | +| `none` | Default, no SASL authentication. | + + +##### sasl.user + +A PLAIN username. It is recommended that you use environment variables to populate this field. + + +Type: `string` +Default: `""` + +```yml +# Examples + +user: ${USER} +``` + +##### sasl.password + +A PLAIN password. It is recommended that you use environment variables to populate this field. +{/* TODO add secret link :::warning Secret +This field contains sensitive information that usually shouldn't be added to a config directly, read our [secrets page for more info](/docs/configuration/secrets). +::: */} + + +Type: `string` +Default: `""` + +```yml +# Examples + +password: ${PASSWORD} +``` + +##### sasl.access_token + +A static OAUTHBEARER access token + + +Type: `string` +Default: `""` + +##### sasl.token_cache + +Instead of using a static `access_token` allows you to query a `cache` resource to fetch OAUTHBEARER tokens from +{/* TODO: add cache resource link */} + +Type: `string` +Default: `""` + +##### sasl.token_key + +Required when using a `token_cache`, the key to query the cache with for tokens. + + +Type: `string` +Default: `""` + +##### topic + +The topic to publish messages to. +{/* TODO: when interpolation supported +This field supports interpolation functions. */} + + +Type: `string` + +##### client_id + +An identifier for the client connection. + + +Type: `string` +Default: `"tyk"` + +##### target_version + +The version of the Kafka protocol to use. This limits the capabilities used by the client and should ideally match the version of your brokers. Defaults to the oldest supported stable version. + + +Type: `string` + +```yml +# Examples + +target_version: 2.1.0 + +target_version: 3.1.0 +``` + +##### rack_id + +A rack identifier for this client. + + +Type: `string` +Default: `""` + +##### key + +The key to publish messages with. +{/* TODO: when interpolation supported +This field supports interpolation functions. */} + + +Type: `string` +Default: `""` + +##### partitioner + +The partitioning algorithm to use. + + +Type: `string` +Default: `"fnv1a_hash"` +Options: `fnv1a_hash`, `murmur2_hash`, `random`, `round_robin`, `manual`. + +##### partition + +The manually-specified partition to publish messages to, relevant only when the field `partitioner` is set to `manual`. Must be able to parse as a 32-bit integer. +{/* TODO: when interpolation supported +This field supports interpolation functions. */} + + +Type: `string` +Default: `""` + +##### custom_topic_creation + +If enabled, topics will be created with the specified number of partitions and replication factor if they do not already exist. + + +Type: `object` + +##### custom_topic_creation.enabled + +Whether to enable custom topic creation. + + +Type: `bool` +Default: `false` + +##### custom_topic_creation.partitions + +The number of partitions to create for new topics. Leave at -1 to use the broker configured default. Must `be >= 1`. + + +Type: `int` +Default: `-1` + +##### custom_topic_creation.replication_factor + +The replication factor to use for new topics. Leave at -1 to use the broker configured default. Must be an odd number, and less then or equal to the number of brokers. + + +Type: `int` +Default: `-1` + +##### compression + +The compression algorithm to use. + + +Type: `string` +Default: `"none"` +Options: `none`, `snappy`, `lz4`, `gzip`, `zstd`. + +##### static_headers + +An optional map of static headers that should be added to messages in addition to metadata. + + +Type: `object` + +```yml +# Examples + +static_headers: + first-static-header: value-1 + second-static-header: value-2 +``` + +##### metadata + +Specify criteria for which metadata values are sent with messages as headers. + + +Type: `object` + +##### metadata.exclude_prefixes + +Provide a list of explicit metadata key prefixes to be excluded when adding metadata to sent messages. + + +Type: `array` +Default: `[]` + +{/* TODO: when bloblang is supported +##### inject_tracing_map + +A Bloblang mapping used to inject an object containing tracing propagation information into outbound messages. The specification of the injected fields will match the format used by the service wide tracer. + + +Type: `string` +Requires version 3.45.0 or newer + +```yml +# Examples + +inject_tracing_map: meta = @.merge(this) + +inject_tracing_map: root.meta.span = this +``` */} + +##### max_in_flight + +The maximum number of messages to have in flight at a given time. Increase this to improve throughput. + + +Type: `int` +Default: `64` + +##### idempotent_write + +Enable the idempotent write producer option. This requires the `IDEMPOTENT_WRITE` permission on `CLUSTER` and can be disabled if this permission is not available. + + +Type: `bool` +Default: `false` + +##### ack_replicas + +Ensure that messages have been copied across all replicas before acknowledging receipt. + + +Type: `bool` +Default: `false` + +##### max_msg_bytes + +The maximum size in bytes of messages sent to the target topic. + + +Type: `int` +Default: `1000000` + +##### timeout + +The maximum period of time to wait for message sends before abandoning the request and retrying. + + +Type: `string` +Default: `"5s"` + +##### retry_as_batch + +When enabled forces an entire batch of messages to be retried if any individual message fails on a send, otherwise only the individual messages that failed are retried. Disabling this helps to reduce message duplicates during intermittent errors, but also makes it impossible to guarantee strict ordering of messages. + + +Type: `bool` +Default: `false` + +##### batching + +Allows you to configure a [batching policy](/api-management/stream-config#batch-policy). + + +Type: `object` + +```yml expandable +# Examples + +batching: + byte_size: 5000 + count: 0 + period: 1s + +batching: + count: 10 + period: 1s + +batching: + check: this.contains("END BATCH") + count: 0 + period: 1m +``` + +##### batching.count + +A number of messages at which the batch should be flushed. If `0` disables count based batching. + + +Type: `int` +Default: `0` + +##### batching.byte_size + +An amount of bytes at which the batch should be flushed. If `0` disables size based batching. + + +Type: `int` +Default: `0` + +##### batching.period + +A period in which an incomplete batch should be flushed regardless of its size. + + +Type: `string` +Default: `""` + +```yml +# Examples + +period: 1s + +period: 1m + +period: 500ms +``` + +{/* TODO: when bloblang is supported +##### batching.check + +A Bloblang query that should return a boolean value indicating whether a message should end a batch. + + +Type: `string` +Default: `""` + +```yml +# Examples + +check: this.type == "end_of_transaction" +``` */} + +##### batching.processors + +{/* TODO: add list of processors link */} + +A list of processors to apply to a batch as it is flushed. This allows you to aggregate and archive the batch however you see fit. Please note that all resulting messages are flushed as a single batch, therefore splitting the batch into smaller batches using these processors is a no-op. + + +Type: `array` + +```yml expandable +# Examples + +processors: + - archive: + format: concatenate + +processors: + - archive: + format: lines + +processors: + - archive: + format: json_array +``` + +##### max_retries + +The maximum number of retries before giving up on the request. If set to zero there is no discrete limit. + + +Type: `int` +Default: `0` + +##### backoff + +Control time intervals between retry attempts. + + +Type: `object` + +##### backoff.initial_interval + +The initial period to wait between retry attempts. + + +Type: `string` +Default: `"3s"` + +```yml +# Examples + +initial_interval: 50ms + +initial_interval: 1s +``` + +##### backoff.max_interval + +The maximum period to wait between retry attempts + + +Type: `string` +Default: `"10s"` + +```yml +# Examples + +max_interval: 5s + +max_interval: 1m +``` + +##### backoff.max_elapsed_time + +The maximum overall period of time to spend on retry attempts before the request is aborted. Setting this value to a zeroed duration (such as `0s`) will result in unbounded retries. + + +Type: `string` +Default: `"30s"` + +```yml +# Examples + +max_elapsed_time: 1m + +max_elapsed_time: 1h +``` + +## Processors + +### Overview + +Tyk Streams processors are functions applied to messages passing through a pipeline. + +Processors are set via config, and depending on where in the config they are placed they will be run either immediately after a specific input (set in the input section), on all messages (set in the pipeline section) or before a specific output (set in the output section). Most processors apply to all messages and can be placed in the pipeline section: + +```yaml expandable +pipeline: + threads: 1 + processors: + - label: my_avro + avro: + operator: "to_json" + encoding: textual +``` + +The `threads` field in the pipeline section determines how many parallel processing threads are created. You can read more about parallel processing in the [pipeline guide](/api-management/stream-config#processing-pipelines). + +#### Labels + +{/* TODO: Replace paragraph below in subsequent iteration when know if metrics supported from product + +Processors have an optional field `label` that can uniquely identify them in observability data such as metrics and logs. This can be useful when running configs with multiple nested processors, otherwise their metrics labels will be generated based on their composition. For more information check out the [metrics documentation]. */} + +Processors have an optional field `label` that can uniquely identify them in observability data such as logs. + +### Avro + +```yml expandable +# Config fields, with default values +label: "" +avro: + operator: "" # No default (required) + encoding: textual + schema: "" + schema_path: "" +``` + + + +**Note** + +If you are consuming or generating messages using a schema registry service then it is likely this processor will fail as those services require messages to be prefixed with the identifier of the schema version being used. + + + +#### Operators + +##### to_json + +Converts Avro documents into a JSON structure. This makes it easier to +manipulate the contents of the document within Tyk Streams. The encoding field +specifies how the source documents are encoded. + + +##### from_json + +Attempts to convert JSON documents into Avro documents according to the +specified encoding. + +#### Fields + +##### operator + +The [operator](#operators) to execute + + +Type: `string` +Options: `to_json`, `from_json`. + +##### encoding + +An Avro encoding format to use for conversions to and from a schema. + + +Type: `string` +Default: `"textual"` +Options: `textual`, `binary`, `single`. + +##### schema + +A full Avro schema to use. + + +Type: `string` +Default: `""` + +##### schema_path + +The path of a schema document to apply. Use either this or the `schema` field. + + +Type: `string` +Default: `""` + +```yml +# Examples + +schema_path: file://path/to/spec.avsc + +schema_path: http://localhost:8081/path/to/spec/versions/1 +``` + +## Tracers + +### Overview + +A tracer type represents a destination for Tyk Streams to send tracing events to such as [Jaeger](https://www.jaegertracing.io/). + +When a tracer is configured all messages will be allocated a root span during ingestion that represents their journey through a Streams pipeline. Many Streams processors create spans, and so tracing is a great way to analyse the pathways of individual messages as they progress through a Streams instance. + +Some inputs, such as `http_server` and `http_client`, are capable of extracting a root span from the source of the message (HTTP headers). This is +a work in progress and should eventually expand so that all inputs have a way of doing so. + +Other inputs, such as `kafka` can be configured to extract a root span by using the `extract_tracing_map` field. + +A tracer config section looks like this: + +```yaml +tracer: + jaeger: + agent_address: localhost:6831 + sampler_type: const + sampler_param: 1 +``` + + + +**Note** + +Although the configuration spec of this component is stable the format of spans, tags and logs created by Streams is subject to change as it is tuned for improvement. + + + +### Jaeger + +```yml expandable +# Common config fields, showing default values +tracer: + jaeger: + agent_address: "" + collector_url: "" + sampler_type: const + flush_interval: "" # No default (optional) +``` + +#### Advanced + +```yml expandable +# All config fields, showing default values +tracer: + jaeger: + agent_address: "" + collector_url: "" + sampler_type: const + sampler_param: 1 + tags: {} + flush_interval: "" # No default (optional) +``` + +Send tracing events to a [Jaeger](https://www.jaegertracing.io/) agent or collector. + +#### Fields + +##### agent_address + +The address of a Jaeger agent to send tracing events to. + +Type: `string` +Default: `""` + +```yml +# Examples + +agent_address: jaeger-agent:6831 +``` + +##### collector_url + +The URL of a Jaeger collector to send tracing events to. If set, this will override `agent_address`. + +Type: `string` +Default: `""` + +```yml +# Examples + +collector_url: https://jaeger-collector:14268/api/traces +``` + +##### sampler_type + +The sampler type to use. + +Type: `string` +Default: `"const"` + +| Option | Summary | +| :--- | :--- | +| `const` | Sample a percentage of traces. 1 or more means all traces are sampled, 0 means no traces are sampled and anything in between means a percentage of traces are sampled. Tuning the sampling rate is recommended for high-volume production workloads. | + +##### sampler_param + +A parameter to use for sampling. This field is unused for some sampling types. + +Type: `float` +Default: `1` + +##### tags + +A map of tags to add to tracing spans. + +Type: `object` +Default: `{}` + +##### flush_interval + +The period of time between each flush of tracing spans. + +Type: `string` + +### OpenTelemetry Collector + +```yml expandable +# Common config fields, showing default values +tracer: + open_telemetry_collector: + http: [] # No default (required) + grpc: [] # No default (required) + sampling: + enabled: false + ratio: 0.85 # No default (optional) +``` + +#### Advanced + +```yml expandable +# All config fields, showing default values +tracer: + open_telemetry_collector: + http: [] # No default (required) + grpc: [] # No default (required) + tags: {} + sampling: + enabled: false + ratio: 0.85 # No default (optional) +``` + + + +**Note** + +This component is experimental and therefore subject to change or removal outside of major version releases. + + + +Send tracing events to an [Open Telemetry collector](https://opentelemetry.io/docs/collector/). + +#### Fields + +##### http + +A list of http collectors. + +Type: `array` + +##### http[].address + +The endpoint of a collector to send tracing events to. + +Type: `string` + +```yml +# Examples + +address: localhost:4318 +``` + +##### http[].secure + +Connect to the collector over HTTPS + +Type: `bool` +Default: `false` + +##### grpc + +A list of grpc collectors. + +Type: `array` + +##### grpc[].address + +The endpoint of a collector to send tracing events to. + +Type: `string` + +```yml +# Examples + +address: localhost:4317 +``` + +##### grpc[].secure + +Connect to the collector with client transport security + +Type: `bool` +Default: `false` + +##### tags + +A map of tags to add to all tracing spans. + +Type: `object` +Default: `{}` + +##### sampling + +Settings for trace sampling. Sampling is recommended for high-volume production workloads. + +Type: `object` + +##### sampling.enabled + +Whether to enable sampling. + +Type: `bool` +Default: `false` + +##### sampling.ratio + +Sets the ratio of traces to sample. + +Type: `float` + +```yml +# Examples + +ratio: 0.85 + +ratio: 0.5 +``` + +## Metrics + +### Overview + +Streams emits lots of metrics in order to expose how components configured within your pipeline are behaving. You can configure exactly where these metrics end up with the config field `metrics`, which describes a metrics format and destination. For example, if you wished to push them via the Prometheus protocol you could use this configuration: + +```yaml +metrics: + prometheus: + push_interval: 1s + push_job_name: in + push_url: http://localhost:9091 +``` + +### Metric Names + +Metrics are emitted with a prefix that can be configured with the field `prefix`. The default prefix is `bento`. The following metrics are emitted with the respective types: + +#### Gauges + +- `{prefix}_input_count` Number of inputs currently active. +- `{prefix}_output_count` Number of outputs currently active. +- `{prefix}_processor_count` Number of processors currently active. +- `{prefix}_cache_count` Number of caches currently active. +- `{prefix}_condition_count` Number of conditions currently active. +- `{prefix}_input_connection_up` 1 if a particular input is connected, 0 if it is not. +- `{prefix}_output_connection_up` 1 if a particular output is connected, 0 if it is not. +- `{prefix}_input_running` 1 if a particular input is running, 0 if it is not. +- `{prefix}_output_running` 1 if a particular output is running, 0 if it is not. +- `{prefix}_processor_running` 1 if a particular processor is running, 0 if it is not. +- `{prefix}_cache_running` 1 if a particular cache is running, 0 if it is not. +- `{prefix}_condition_running` 1 if a particular condition is running, 0 if it is not. +- `{prefix}_buffer_running` 1 if a particular buffer is running, 0 if it is not. +- `{prefix}_buffer_available` The number of messages that can be read from a buffer. +- `{prefix}_input_retry` The number of active retry attempts for a particular input. +- `{prefix}_output_retry` The number of active retry attempts for a particular output. +- `{prefix}_processor_retry` The number of active retry attempts for a particular processor. +- `{prefix}_cache_retry` The number of active retry attempts for a particular cache. +- `{prefix}_condition_retry` The number of active retry attempts for a particular condition. +- `{prefix}_buffer_retry` The number of active retry attempts for a particular buffer. +- `{prefix}_threads_active` The number of processing threads currently active. + +#### Counters + +- `{prefix}_input_received` Count of messages received by a particular input. +- `{prefix}_input_batch_received` Count of batches received by a particular input. +- `{prefix}_output_sent` Count of messages sent by a particular output. +- `{prefix}_output_batch_sent` Count of batches sent by a particular output. +- `{prefix}_processor_processed` Count of messages processed by a particular processor. +- `{prefix}_processor_batch_processed` Count of batches processed by a particular processor. +- `{prefix}_processor_dropped` Count of messages dropped by a particular processor. +- `{prefix}_processor_batch_dropped` Count of batches dropped by a particular processor. +- `{prefix}_processor_error` Count of errors returned by a particular processor. +- `{prefix}_processor_batch_error` Count of batch errors returned by a particular processor. +- `{prefix}_cache_hit` Count of cache key lookups that found a value. +- `{prefix}_cache_miss` Count of cache key lookups that did not find a value. +- `{prefix}_cache_added` Count of new cache entries. +- `{prefix}_cache_err` Count of errors that occurred during a cache operation. +- `{prefix}_condition_hit` Count of condition checks that passed. +- `{prefix}_condition_miss` Count of condition checks that failed. +- `{prefix}_condition_error` Count of errors that occurred during a condition check. +- `{prefix}_buffer_added` Count of messages added to a particular buffer. +- `{prefix}_buffer_batch_added` Count of batches added to a particular buffer. +- `{prefix}_buffer_read` Count of messages read from a particular buffer. +- `{prefix}_buffer_batch_read` Count of batches read from a particular buffer. +- `{prefix}_buffer_ack` Count of messages removed from a particular buffer. +- `{prefix}_buffer_batch_ack` Count of batches removed from a particular buffer. +- `{prefix}_buffer_nack` Count of messages that failed to be removed from a particular buffer. +- `{prefix}_buffer_batch_nack` Count of batches that failed to be removed from a particular buffer. +- `{prefix}_buffer_err` Count of errors that occurred during a buffer operation. +- `{prefix}_buffer_batch_err` Count of batch errors that occurred during a buffer operation. +- `{prefix}_input_error` Count of errors that occurred during an input operation. +- `{prefix}_input_batch_error` Count of batch errors that occurred during an input operation. +- `{prefix}_output_error` Count of errors that occurred during an output operation. +- `{prefix}_output_batch_error` Count of batch errors that occurred during an output operation. +- `{prefix}_resource_cache_error` Count of errors that occurred during a resource cache operation. +- `{prefix}_resource_condition_error` Count of errors that occurred during a resource condition operation. +- `{prefix}_resource_input_error` Count of errors that occurred during a resource input operation. +- `{prefix}_resource_processor_error` Count of errors that occurred during a resource processor operation. +- `{prefix}_resource_output_error` Count of errors that occurred during a resource output operation. +- `{prefix}_resource_rate_limit_error` Count of errors that occurred during a resource rate limit operation. + +#### Timers + +- `{prefix}_input_latency` Latency of a particular input. +- `{prefix}_input_batch_latency` Latency of a particular input at the batch level. +- `{prefix}_output_latency` Latency of a particular output. +- `{prefix}_output_batch_latency` Latency of a particular output at the batch level. +- `{prefix}_processor_latency` Latency of a particular processor. +- `{prefix}_processor_batch_latency` Latency of a particular processor at the batch level. +- `{prefix}_condition_latency` Latency of a particular condition. +- `{prefix}_condition_batch_latency` Latency of a particular condition at the batch level. +- `{prefix}_cache_latency` Latency of a particular cache. +- `{prefix}_buffer_latency` Latency of a particular buffer. +- `{prefix}_buffer_batch_latency` Latency of a particular buffer at the batch level. + +### Metric Labels + +All metrics are emitted with the following labels: + +- `path` The path of the component within the config. +- `label` A custom label for the component, which is optional and falls back to the component type. + +### Prometheus + +```yml expandable +# Common config fields, showing default values +metrics: + prometheus: + prefix: tyk + push_interval: "" + push_job_name: kafka_out + push_url: "" +``` + +#### Advanced + +```yml expandable +# All config fields, showing default values +metrics: + prometheus: + prefix: tyk + push_interval: "" + push_job_name: my_stream + push_url: "" + push_basic_auth: + enabled: false + username: "" + password: "" + file_path: "" + use_histogram_timing: false + histogram_buckets: [0.000001, 0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0] +``` + +Send metrics to a Prometheus push gateway, or expose them via HTTP endpoints. + +#### Fields + +##### prefix + +A string prefix for all metrics. + +Type: `string` +Default: `"bento"` + +##### push_interval + +The interval between pushing metrics to the push gateway. + +Type: `string` +Default: `""` + +```yml +# Examples + +push_interval: 1s + +push_interval: 1m +``` + +##### push_job_name + +A job name to attach to metrics pushed to the push gateway. + +Type: `string` +Default: `"bento_push"` + +##### push_url + +The URL to push metrics to. + +Type: `string` +Default: `""` + +```yml +# Examples + +push_url: http://localhost:9091 +``` + +##### push_basic_auth + +Basic authentication configuration for the push gateway. + +Type: `object` + +##### push_basic_auth.enabled + +Whether to use basic authentication when pushing metrics. + +Type: `bool` +Default: `false` + +##### push_basic_auth.username + +The username to authenticate with. + +Type: `string` +Default: `""` + +##### push_basic_auth.password + +The password to authenticate with. + +Type: `string` +Default: `""` + +##### file_path + +The file path to write metrics to. + +Type: `string` +Default: `""` + +```yml +# Examples + +file_path: /tmp/metrics.txt +``` + +##### use_histogram_timing + +Whether to use histogram metrics for timing values. When set to false, summary metrics are used instead. + +Type: `bool` +Default: `false` + +##### histogram_buckets + +A list of duration buckets to track when use_histogram_timing is set to true. + +Type: `array` +Default: `[0.000001, 0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0]` + +## Common Configuration + +### Batching + +Tyk Streams is able to join sources and sinks with sometimes conflicting batching behaviours without sacrificing its strong delivery guarantees. Therefore, batching within Tyk Streams is a mechanism that serves multiple purposes: + +1. [Performance (throughput)](#performance) +2. [Compatibility (mixing multi and single part message protocols)](#compatibility) + +#### Performance + +For most users the only benefit of batching messages is improving throughput over your output protocol. For some protocols this can happen in the background and requires no configuration from you. However, if an output has a `batching` configuration block this means it benefits from batching and requires you to specify how you'd like your batches to be formed by configuring a [batching policy](#batch-policy): + +```yaml expandable +output: + kafka: + addresses: [ todo:9092 ] + topic: tyk_stream + + # Either send batches when they reach 10 messages or when 100ms has passed + # since the last batch. + batching: + count: 10 + period: 100ms +``` + +However, a small number of inputs such as [kafka](/api-management/stream-config#kafka) must be consumed sequentially (in this case by partition) and therefore benefit from specifying your batch policy at the input level instead: + +```yaml expandable +input: + kafka: + addresses: [ todo:9092 ] + topics: [ tyk_input_stream ] + batching: + count: 10 + period: 100ms + +output: + kafka: + addresses: [ todo:9092 ] + topic: tyk_stream +``` + +Inputs that behave this way are documented as such and have a `batching` configuration block. + +Sometimes you may prefer to create your batches before processing, in which case if your input doesn't already support [a batch policy](#batch-policy) you can instead use a [broker](/api-management/stream-config#broker), which also allows you to combine inputs with a single batch policy: + +```yaml expandable +input: + broker: + inputs: + - resource: foo + - resource: bar + batching: + count: 50 + period: 500ms +``` + +This also works the same with [output brokers](/api-management/stream-config#broker-1). + +#### Compatibility + +Tyk Streams is able to read and write over protocols that support multiple part messages, and all payloads travelling through Tyk Streams are represented as a multiple part message. Therefore, all components within Tyk Streams are able to work with multiple parts in a message as standard. + +When messages reach an output that *doesn't* support multiple parts the message is broken down into an individual message per part, and then one of two behaviours happen depending on the output. If the output supports batch sending messages then the collection of messages are sent as a single batch. Otherwise, Tyk Streams falls back to sending the messages sequentially in multiple, individual requests. + +This behaviour means that not only can multiple part message protocols be easily matched with single part protocols, but also the concept of multiple part messages and message batches are interchangeable within Tyk Streams. + +#### Batch Policy + +When an input or output component has a config field `batching` that means it supports a batch policy. This is a mechanism that allows you to configure exactly how your batching should work on messages before they are routed to the input or output it's associated with. Batches are considered complete and will be flushed downstream when either of the following conditions are met: + + +- The `byte_size` field is non-zero and the total size of the batch in bytes matches or exceeds it (disregarding metadata.) +- The `count` field is non-zero and the total number of messages in the batch matches or exceeds it. +- The `period` field is non-empty and the time since the last batch exceeds its value. + +This allows you to combine conditions: + +```yaml expandable +output: + kafka: + addresses: [ todo:9092 ] + topic: tyk_stream + + # Either send batches when they reach 10 messages or when 100ms has passed + # since the last batch. + batching: + count: 10 + period: 100ms +``` + + + +A batch policy has the capability to *create* batches, but not to break them down. + + + +If your configured pipeline is processing messages that are batched *before* they reach the batch policy then they may circumvent the conditions you've specified here, resulting in sizes you aren't expecting. + +### Field Paths + +Many components within Tyk Streams allow you to target certain fields using a JSON dot path. The syntax of a path within Tyk Streams is similar to [JSON Pointers](https://tools.ietf.org/html/rfc6901), except with dot separators instead of slashes (and no leading dot.) When a path is used to set a value any path segment that does not yet exist in the structure is created as an object. + +For example, if we had the following JSON structure: + +```json +{ + "foo": { + "bar": 21 + } +} +``` + +The query path `foo.bar` would return `21`. + +The characters `~` (%x7E) and `.` (%x2E) have special meaning in Tyk Streams paths. Therefore `~` needs to be encoded as `~0` and `.` needs to be encoded as `~1` when these characters appear within a key. + +For example, if we had the following JSON structure: + +```json expandable +{ + "foo.foo": { + "bar~bo": { + "": { + "baz": 22 + } + } + } +} +``` + +The query path `foo~1foo.bar~0bo..baz` would return `22`. + +#### Arrays + +When Tyk Streams encounters an array whilst traversing a JSON structure it requires the next path segment to be either an integer of an existing index, or, depending on whether the path is used to query or set the target value, the character `*` or `-` respectively. + +For example, if we had the following JSON structure: + +```json +{ + "foo": [ + 0, 1, { "bar": 23 } + ] +} +``` + +The query path `foo.2.bar` would return `23`. + +##### Querying + +When a query reaches an array the character `*` indicates that the query should return the value of the remaining path from each element of the array (within an array.) + +##### Setting + +When an array is reached the character `-` indicates that a new element should be appended to the end of the existing elements, if this character is not the final segment of the path then an object is created. + +### Processing Pipelines + +Within a Tyk Streams configuration, in between `input` and `output`, is a `pipeline` section. This section describes an array of processors that are to be applied to *all* messages, and are not bound to any particular input or output. + +If you have processors that are heavy on CPU and aren't specific to a certain input or output they are best suited for the pipeline section. It is advantageous to use the pipeline section as it allows you to set an explicit number of parallel threads of execution: + +```yaml expandable +input: + resource: foo + +pipeline: + threads: 4 + processors: + - avro: + operator: "to_json" + +output: + resource: bar +``` + +If the field `threads` is set to `-1` (the default) it will automatically match the number of logical CPUs available. By default almost all Tyk Streams sources will utilize as many processing threads as have been configured, which makes horizontal scaling easy. diff --git a/api-management/streams-end-to-end-example.mdx b/api-management/streams-end-to-end-example.mdx new file mode 100644 index 00000000..ebac63dc --- /dev/null +++ b/api-management/streams-end-to-end-example.mdx @@ -0,0 +1,230 @@ +--- +title: "Tyk Streams End-to-End Example" +'og:description': "A comprehensive end-to-end example of Tyk Streams implementation" +tags: ['Tyk Streams', 'Event-Driven APIs', 'Kafka', 'WebSockets', 'SSE', 'Correlation IDs'] +sidebarTitle: "Tyk Streams End-to-End Example" +--- + +
+ +## Why Tyk Streams? + +Tyk Streams adds a **declarative event layer** on top of the Tyk Gateway, letting you expose or consume broker topics (Kafka, NATS, RabbitMQ…) through normal HTTP channelsβ€”REST, WebSocket, Server-Sent Eventsβ€”without glue code. + +You can manage stream definitions in three interchangeable ways: + +| Method | When to use | +| :-------- | :------------- | +| **Tyk Dashboard UI** | Rapid prototyping and PoCs | +| **OpenAPI + `x-tyk-streaming`** | β€œEverything-as-code”, safe for Git | +| **Tyk Operator (Kubernetes CRD)** | GitOps & CI/CD pipelines | + +--- + +## Requirements + +* **Tyk Gateway β‰₯ 5.8** with **Streams** feature enabled +* **Apache Kafka** reachable on `localhost:9093` +* *(Optional)* **Prometheus** and **Jaeger** if you enable the commented observability blocks + +--- + +## Architecture + +The demo shows a classic pattern: a user request becomes an event on a bus, a worker processes it asynchronously, and the result is delivered back to the same userβ€”without leaking data across tenants. + +```mermaid +sequenceDiagram + autonumber + participant Client + participant GatewayIn as Gateway
in stream + participant KafkaJobs as Kafka
topic **jobs** + participant Worker as Worker
Worker stream + participant KafkaCompleted as Kafka
topic **completed** + participant GatewayOut as Gateway
out stream + + %% synchronous request from client + Client ->> GatewayIn: POST /push-event **(sync)** + GatewayIn -->> Client: 200 OK + echo **(sync)** + + %% asynchronous event flow + GatewayIn -->> KafkaJobs: publish event **(async)** + KafkaJobs -->> Worker: consume job **(async)** + Worker -->> KafkaCompleted: publish result **(async)** + KafkaCompleted -->> GatewayOut: consume result **(async)** + + %% synchronous delivery back to the same user + GatewayOut ->> Client: GET /get-event / WS / SSE **(sync)** +``` + +### Stream-per-responsibility pattern + +| Stream | Role | Input | Output | +| :-------- | :------ | :------- | :-------- | +| **`in`** | Edge entrypoint: accepts HTTP, enriches payload (`user_id`, `job_id`), publishes to **`jobs`** and echoes to caller | HTTP | Kafka + sync response | +| **`Worker`** | Background micro-service: listens to **`jobs`**, attaches `result: "bar"`, publishes to **`completed`** | Kafka | Kafka | +| **`out`** | Edge exit point: listens to **`completed`**, drops messages not owned by caller, delivers via REST/WS/SSE | Kafka | HTTP | + +--- + +## Processor mapping (built-in scripting) + +Streams pipelines include **processors**. The *mapping* processor embeds [Bloblang](https://www.benthos.dev/docs/guides/bloblang/about/) so you can transform or filter messages inline: + +* **Enrich** – `in` adds `user_id` & `job_id` +* **Augment** – `Worker` adds a static field `{ "result": "bar" }` +* **Filter** – `out` calls `deleted()` for non-matching users + +Dynamic placeholders (`$tyk_context.…`) can reference query params, headers, JWT claims, or any other context variableβ€”usable anywhere in the Streams config. + +--- + +## Observability (optional) + +Uncomment the `metrics:` and `tracer:` blocks to push per-stream Prometheus metrics and Jaeger traces. Tags like `stream: Worker` make end-to-end tracing trivial. + +--- + +## Full OpenAPI definition + +Copy/paste into `streams-demo.yaml`, import via Dashboard UI, or apply with Tyk Operator: + +```yaml expandable +info: + title: streams-demo + version: 1.0.0 +openapi: 3.0.3 +servers: + - url: http://tyk-gateway:8282/stream-demo/ +x-tyk-streaming: + streams: + Worker: + input: + kafka: + addresses: + - localhost:9093 + consumer_group: worker + topics: + - jobs + output: + kafka: + addresses: + - localhost:9093 + topic: completed + pipeline: + processors: + - mapping: | + root = this.merge({ "result": "bar" }) +# metrics: +# prometheus: +# push_interval: 1s +# push_job_name: Worker +# push_url: http://localhost:9091 +# tracer: +# jaeger: +# collector_url: http://localhost:14268/api/traces +# tags: +# stream: Worker + + in: + input: + http_server: + path: /push-event + ws_path: /ws-out + output: + broker: + outputs: + - kafka: + addresses: + - localhost:9093 + topic: jobs + - sync_response: {} + pipeline: + processors: + - mapping: | + root = this + root.user_id = "$tyk_context.request_data_user" # or $tyk_context.jwt.claims.sub + root.job_id = uuid_v4() +# tracer: +# jaeger: +# collector_url: http://localhost:14268/api/traces +# tags: +# stream: in +# metrics: +# prometheus: +# push_interval: 1s +# push_job_name: in +# push_url: http://localhost:9091 + + out: + input: + kafka: + addresses: + - localhost:9093 + consumer_group: $tyk_context.request_data_user + topics: + - completed + output: + http_server: + path: /get-event + ws_path: /ws-in + pipeline: + processors: + - mapping: | + root = if this.user_id != "$tyk_context.request_data_user" { + deleted() + } +# tracer: +# jaeger: +# collector_url: http://localhost:14268/api/traces +# tags: +# stream: out +# metrics: +# prometheus: +# push_interval: 1s +# push_job_name: out +# push_url: http://localhost:9091 +security: [] +paths: {} +components: + securitySchemes: {} +x-tyk-api-gateway: + info: + name: stream-demo + state: + active: true + internal: false + middleware: + global: + contextVariables: + enabled: true + trafficLogs: + enabled: true + server: + listenPath: + strip: true + value: /stream-demo/ + upstream: + proxy: + enabled: false + url: "" +``` + +--- + +## Running the demo + +1. **Start Kafka** (e.g. docker-compose). +2. **Launch Tyk Gateway 5.8+** with the YAML above. +3. **Send an event** + ```bash + curl -X POST "http://localhost:8282/stream-demo/push-event?user=alice" \ + -H "Content-Type: application/json" \ + -d '{"message":"hello world"}' + ``` +4. **Receive the result** (only *alice*’s jobs) + ```bash + curl "http://localhost:8282/stream-demo/get-event?user=alice" + ``` +5. **Switch transport** – connect via websocket `wscat -c http://127.0.0.1:8282/stream-demo/ws-in\?user\=alice` +6. *(Optional)* **Enable metrics & tracing** – uncomment blocks, restart Gateway, explore in Grafana & Jaeger. diff --git a/api-management/traffic-transformation.mdx b/api-management/traffic-transformation.mdx new file mode 100644 index 00000000..1e1cc968 --- /dev/null +++ b/api-management/traffic-transformation.mdx @@ -0,0 +1,106 @@ +--- +title: "Transform Traffic by using Tyk Middleware" +tags: ['Overview', 'Allow List', 'Block List', 'Ignore Authentication', 'Internal Endpoint', 'Request Method ', 'Request Body ', 'Request Headers ', 'Response Body', 'Response Headers', 'Request Validation', 'Mock Response', 'Virtual Endpoints', 'Go Templates', 'JQ Transforms', 'Request Context Variables'] +sidebarTitle: "Overview" +--- + +## Overview + +When you configure an API on Tyk, the Gateway will proxy all requests received at the listen path that you have defined through to the upstream (target) URL configured in the API definition. Responses from the upstream are likewise proxied on to the originating client. Requests and responses are processed through a powerful [chain of middleware](/api-management/traffic-transformation#request-middleware-chain) that perform security and processing functions. + +Within that chain are a highly configurable set of optional middleware that can, on a per-endpint basis: +- apply processing to [API requests](#middleware-applied-to-the-api-request) before they are proxied to the upstream service +- apply customization to the [API response](#middleware-applied-to-the-api-response) prior to it being proxied back to the client + +Tyk also supports a powerful custom plugin feature that enables you to add custom processing at different stages in the processing chains. For more details on custom plugins please see the [dedicated guide](/api-management/plugins/overview#). + +### Middleware applied to the API Request + +The following standard middleware can optionally be applied to API requests on a per-endpoint basis. + +#### Allow list + +The [Allow List](/api-management/traffic-transformation/allow-list) middleware is a feature designed to restrict access to only specific API endpoints. It rejects requests to endpoints not specifically "allowed", returning `HTTP 403 Forbidden`. This enhances the security of the API by preventing unauthorized access to endpoints that are not explicitly permitted. + +Enabling the allow list will cause the entire API to become blocked other than for endpoints that have this middleware enabled. This is great if you wish to have very strict access rules for your services, limiting access to specific published endpoints. + +#### Block list + +The [Block List](/api-management/traffic-transformation/block-list) middleware is a feature designed to prevent access to specific API endpoints. Tyk Gateway rejects all requests made to endpoints with the block list enabled, returning `HTTP 403 Forbidden`. + +#### Cache + +Tyk's [API-level cache](/api-management/response-caching#basic-caching) does not discriminate between endpoints and will usually be configured to cache all safe requests. You can use the granular [Endpoint Cache](/api-management/response-caching#endpoint-caching) to ensure finer control over which API responses are cached by Tyk. + +#### Circuit Breaker + +The [Circuit Breaker](/planning-for-production/ensure-high-availability/circuit-breakers) is a protective mechanism that helps to maintain system stability by preventing repeated failures and overloading of services that are erroring. When a network or service failure occurs, the circuit breaker prevents further calls to that service, allowing the affected service time to recover while ensuring that the overall system remains functional. + +#### Do Not Track Endpoint + +If [traffic logging](/api-management/logs-metrics#api-traffic-logs) is enabled for your Tyk Gateway, then it will create transaction logs for all API requests (and responses) to deployed APIs. You can use the [Do-Not-Track](/api-management/traffic-transformation/do-not-track) middleware to suppress creation of transaction records for specific endpoints. + +#### Enforced Timeout + +Tyk’s [Enforced Timeout](/planning-for-production/ensure-high-availability/circuit-breakers) middleware can be used to apply a maximum time that the Gateway will wait for a response before it terminates (or times out) the request. This helps to maintain system stability and prevents unresponsive or long-running tasks from affecting the overall performance of the system. + +#### Ignore Authentication + +Adding the [Ignore Authentication](/api-management/traffic-transformation/ignore-authentication) middleware means that Tyk Gateway will not perform authentication checks on requests to that endpoint. This plugin can be very useful if you have a specific endpoint (such as a ping) that you don't need to secure. + +#### Internal Endpoint + +The [Internal Endpoint](/api-management/traffic-transformation/internal-endpoint) middleware instructs Tyk Gateway not to expose the endpoint externally. Tyk Gateway will then ignore external requests to that endpoint while continuing to process internal requests from other APIs; this is often used with the [internal looping](/advanced-configuration/transform-traffic/looping) functionality. + +#### Method Transformation + +The [Method Transformation](/api-management/traffic-transformation/request-method) middleware allows you to change the HTTP method of a request. + +#### Mock Response + +A [Mock Response](/api-management/traffic-transformation/mock-response) is a simulated API response that can be returned by the API gateway without actually sending the request to the backend API. Mock responses are an integral feature for API development, enabling developers to emulate API behavior without the need for upstream execution. + +#### Request Body Transform + +The [Request Body Transform](/api-management/traffic-transformation/request-body) middleware allows you to perform modification to the body (payload) of the API request to ensure that it meets the requirements of your upstream service. + +#### Request Header Transform + +The [Request Header Transform](/api-management/traffic-transformation/request-headers) middleware allows you to modify the header information provided in the request before it leaves the Gateway and is passed to your upstream API. + +#### Request Size Limit + +Tyk Gateway offers a flexible tiered system of limiting request sizes ranging from globally applied limits across all APIs deployed on the gateway down to specific size limits for individual API endpoints. The [Request Size Limit](/api-management/traffic-transformation/request-size-limits) middleware provides the most granular control over request size by enabling you to set different limits for individual endpoints. + +#### Request Validation + +Tyk’s [Request Validation](/api-management/traffic-transformation/request-validation) middleware provides a way to validate the presence, correctness and conformity of HTTP requests to make sure they meet the expected format required by the upstream API endpoints. + +When working with Tyk OAS APIs, the request validation covers both headers and body (payload); with the older Tyk Classic API style we can validate only the request body (payload). + +#### Track Endpoint + +If you do not want to include all endpoints in your [Activity by Endpoint](/api-management/dashboard-configuration#activity-by-endpoint) statistics in Tyk Dashboard, you can enable this middleware for the endpoints to be included. + +#### URL Rewrite + +[URL Rewriting](/transform-traffic/url-rewriting#url-rewrite-middleware) in Tyk is a powerful feature that enables the modification of incoming API request paths to match the expected endpoint format of your backend services. This allows you to translate an outbound API interface to the internal structure of your services. It is a key capability used in [internal looping](/advanced-configuration/transform-traffic/looping) + +#### Virtual Endpoint + +Tyk’s [Virtual Endpoints](/api-management/traffic-transformation/virtual-endpoints) is a programmable middleware component that allows you to perform complex interactions with your upstream service(s) that cannot be handled by one of the other middleware components. + +### Middleware applied to the API Response + +The following transformations can be applied to the response recieved from the upstream to ensure that it contains the correct data and format expected by your clients. + +#### Response Body Transform + +The [Response Body Transform](/api-management/traffic-transformation/response-body) middleware allows you to perform modification to the body (payload) of the response received from the upstream service to ensure that it meets the expectations of the client. + +#### Response Header Transform + +The [Response Header Transform](/api-management/traffic-transformation/response-headers) middleware allows you to modify the header information provided in the response before it leaves the Gateway and is passed to the client. +### Request Middleware Chain + +Middleware execution flow diff --git a/api-management/traffic-transformation/allow-list.mdx b/api-management/traffic-transformation/allow-list.mdx new file mode 100644 index 00000000..fd1618c6 --- /dev/null +++ b/api-management/traffic-transformation/allow-list.mdx @@ -0,0 +1,319 @@ +--- +title: "Allow List" +'og:description': "How to configure Allow List traffic transformation middleware in Tyk" +tags: ['Traffic Transformation', 'Allow List'] +sidebarTitle: "Allow List" +--- + +

Overview

+The Allow List middleware is a feature designed to restrict access to only specific API endpoints. It rejects requests to endpoints not specifically "allowed", returning `HTTP 403 Forbidden`. This enhances the security of the API by preventing unauthorized access to endpoints that are not explicitly permitted. + +Note that this is not the same as Tyk's [IP allow list](/api-management/gateway-config-tyk-classic#ip-access-control) feature, which is used to restrict access to APIs based upon the IP of the requestor. + +### Use Cases + +#### Restricting access to private endpoints + +If you have a service that exposes endpoints or supports methods that you do not want to be available to clients, you should use the allow list to perform strict restriction to a subset of methods and paths. If the allow list is not enabled, requests to endpoints that are not explicitly defined in Tyk will be proxied to the upstream service and may lead to unexpected behavior. + +### Working + +Tyk Gateway does not actually maintain a list of allowed endpoints but rather works on the model whereby if the *allow list* middleware is added to an endpoint then this will automatically block all other endpoints. + +Tyk Gateway will subsequently return `HTTP 403 Forbidden` to any requested endpoint that doesn't have the *allow list* middleware enabled, even if the endpoint is defined and configured in the API definition. + +
+ + +If you enable the allow list feature by adding the middleware to any endpoint, ensure that you also add the middleware to any other endpoint for which you wish to accept requests. + + + +#### Case sensitivity + +By default the allow list is case-sensitive, so for example if you have defined the endpoint `GET /userID` in your API definition then only calls to `GET /userID` will be allowed: calls to `GET /UserID` or `GET /userid` will be rejected. You can configure the middleware to be case-insensitive at the endpoint level. + +You can also set case sensitivity for the entire [gateway](/tyk-oss-gateway/configuration#ignore_endpoint_case) in the Gateway configuration file `tyk.conf`. If case insensitivity is configured at the gateway level, this will override the endpoint-level setting. + +#### Endpoint parsing + +When using the allow list middleware, we recommend that you familiarize yourself with Tyk's [URL matching](/getting-started/key-concepts/url-matching) options. + +
+ + +Tyk recommends that you use [exact](/getting-started/key-concepts/url-matching#exact-match) matching for maximum security, though prefix and wildcard strategies might also apply for your particular deployment or use case. + + + +
+ +If you're using Tyk OAS APIs, then you can find details and examples of how to configure the allow list middleware [here](#allow-list-using-tyk-oas). + +If you're using Tyk Classic APIs, then you can find details and examples of how to configure the allow list middleware [here](#allow-list-using-classic). + +{/* proposed "summary box" to be shown graphically on each middleware page + # Allow List middleware summary + - The Allow List is an optional stage in Tyk's API Request processing chain, sitting between the [TBC]() and [TBC]() middleware. + - The Allow List can be configured at the per-endpoint level within the API Definition and is supported by the API Designer within the Tyk Dashboard. */} + + +

Using Tyk OAS

+The [allow list](/api-management/traffic-transformation/allow-list) is a feature designed to restrict access to only specific API endpoints. It rejects requests to endpoints not specifically "allowed", returning `HTTP 403 Forbidden`. This enhances the security of the API by preventing unauthorized access to endpoints that are not explicitly permitted. + +When working with Tyk OAS APIs the middleware is configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation). You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the legacy Tyk Classic APIs, then check out the [Tyk Classic](#allow-list-using-classic) page. + + +### API Definition + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. Endpoint `paths` entries (and the associated `operationId`) can contain wildcards in the form of any string bracketed by curly braces, for example `/status/{code}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +The allow list middleware (`allow`) can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + +The `allow` object has the following configuration: + +- `enabled`: enable the middleware for the endpoint +- `ignoreCase`: if set to `true` then the path matching will be case insensitive + +For example: + +```json {hl_lines=["47-50", "53-56"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-allow-list", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/anything": { + "get": { + "operationId": "anythingget", + "responses": { + "200": { + "description": "" + } + } + }, + "put": { + "operationId": "anythingput", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-allow-list", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-allow-list/", + "strip": true + } + }, + "middleware": { + "operations": { + "anythingget": { + "allow": { + "enabled": true, + "ignoreCase": true + } + }, + "anythingput": { + "allow": { + "enabled": true, + "ignoreCase": true + } + } + } + } + } +} +``` + +In this example the allow list middleware has been configured for requests to the `GET /anything` and `PUT /anything` endpoints. Requests to any other endpoints will be rejected with `HTTP 403 Forbidden`, unless they also have the allow list middleware enabled. +Note that the allow list has been configured to be case insensitive, so calls to `GET /Anything` will be allowed +Note also that the endpoint path has not been terminated with `$`. Requests to, for example, `GET /anything/foobar` will be allowed as the [regular expression pattern match](#endpoint-parsing) will recognize this as `GET /anything`. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the allow list feature. + +### API Designer + +Adding the allow list to your API endpoints is easy is easy when using the API Designer in the Tyk Dashboard, simply follow these steps: + +1. **Add an endpoint** + + From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + + Tyk OAS API Designer showing no endpoints created + + Adding an endpoint to an API using the Tyk OAS API Designer + + Tyk OAS API Designer showing no middleware enabled on endpoint + +2. **Select the Allow List middleware** + + Select **ADD MIDDLEWARE** and choose the **Allow List** middleware from the *Add Middleware* screen. + + Adding the Allow List middleware + +3. **Optionally configure case-insensitivity** + + If you want to disable case-sensitivity for the allow list, then you must select **EDIT** on the Allow List icon. + + Allow List middleware added to endpoint - click through to edit the config + + This takes you to the middleware configuration screen where you can alter the case sensitivity setting. + Configuring case sensitivity for the Allow List + + Select **UPDATE MIDDLEWARE** to apply the change to the middleware configuration. + +4. **Save the API** + + Select **SAVE API** to apply the changes to your API. + +

Using Classic

+The [allow list](/api-management/traffic-transformation/allow-list) is a feature designed to restrict access to only specific API endpoints. It rejects requests to endpoints not specifically "allowed", returning `HTTP 403 Forbidden`. This enhances the security of the API by preventing unauthorized access to endpoints that are not explicitly permitted. + +When working with Tyk Classic APIs the middleware is configured in the Tyk Classic API Definition. You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the newer Tyk OAS APIs, then check out the [Tyk OAS](#allow-list-using-tyk-oas) page. + +### API Definition + +To enable and configure the allow list you must add a new `white_list` object to the `extended_paths` section of your API definition. + + + +Historically, Tyk followed the out-dated whitelist/blacklist naming convention. We are working to remove this terminology from the product and documentation, however this configuration object currently retains the old name. + + + +The `white_list` object has the following configuration: + +- `path`: the endpoint path +- `method`: this should be blank +- `ignore_case`: if set to `true` then the path matching will be case insensitive +- `method_actions`: a shared object used to configure the [mock response](/api-management/traffic-transformation/mock-response#configuring-mock-response-using-tyk-dashboard-ui) middleware + +The `method_actions` object should be configured as follows, with an entry created for each allowed method on the path: + +- `action`: this should be set to `no_action` +- `code`: this should be set to `200` +- `headers` : this should be blank + +For example: + +```json {linenos=true, linenostart=1} expandable +{ + "extended_paths": { + "white_list": [ + { + "disabled": false, + "path": "/status/200", + "method": "", + "ignore_case": false, + "method_actions": { + "GET": { + "action": "no_action", + "code": 200, + "headers": {} + }, + "PUT": { + "action": "no_action", + "code": 200, + "headers": {} + } + } + } + ] + } +} +``` + +In this example the allow list middleware has been configured for HTTP `GET` and `PUT` requests to the `/status/200` endpoint. Requests to any other endpoints will be rejected with `HTTP 403 Forbidden`, unless they also have the allow list middleware enabled. +Note that the allow list has been configured to be case sensitive, so calls to `GET /Status/200` will also be rejected. +Note also that the endpoint path has not been terminated with `$`. Requests to, for example, `GET /status/200/foobar` will be allowed as the [regular expression pattern match](#endpoint-parsing) will recognize this as `GET /status/200`. + +Consult section [configuring the Allow List in Tyk Operator](#tyk-operator) for details on how to configure allow lists for endpoints using Tyk Operator. + +### API Designer + +You can use the API Designer in the Tyk Dashboard to configure the allow list middleware for your Tyk Classic API by following these steps. + +1. **Add an endpoint for the path and select the plugin** + + From the **Endpoint Designer**, add an endpoint that matches the path for which you want to allow access. Select the **Whitelist** plugin. + +2. **Configure the allow list** + + Once you have selected the middleware for the endpoint, the only additional feature that you need to configure is whether to make the middleware case insensitive by selecting **Ignore Case**. + + Allowlist options + +3. **Save the API** + + Use the *save* or *create* buttons to save the changes and activate the allow list middleware. + +### Tyk Operator + +Similar to the configuration of a Tyk Classic API Definition you must add a new `white_list` object to the `extended_paths` section of your API definition. Furthermore, the `use_extended_paths` configuration parameter should be set to `true`. + + + +Historically, Tyk followed the out-dated whitelist/blacklist naming convention. We are working to remove this terminology from the product and documentation, however this configuration object currently retains the old name. + + + +```yaml {linenos=true,linenostart=1,hl_lines=["26-34"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-whitelist +spec: + name: httpbin-whitelist + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org/ + listen_path: /httpbin + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + white_list: + - ignore_case: true + method_actions: + GET: + action: "no_action" + code: 200 + data: "" + headers: {} + path: "/get" +``` + +In this example the allow list middleware has been configured for `HTTP GET` requests to the `/get` endpoint. Requests to any other endpoints will be rejected with `HTTP 403 Forbidden`, unless they also have the allow list middleware enabled. Note that the allow list has been configured to case insensitive, so calls to `GET /Get` will also be accepted. Note also that the endpoint path has not been terminated with `$`. Requests to, for example, `GET /get/foobar` will be allowed as the [regular expression pattern match](#endpoint-parsing) will recognize this as `GET /get`. + + diff --git a/api-management/traffic-transformation/block-list.mdx b/api-management/traffic-transformation/block-list.mdx new file mode 100644 index 00000000..e9860bfd --- /dev/null +++ b/api-management/traffic-transformation/block-list.mdx @@ -0,0 +1,308 @@ +--- +title: "Block List" +'og:description': "How to configure Block List traffic transformation middleware in Tyk" +tags: ['Traffic Transformation', 'Block List'] +sidebarTitle: "Block List" +--- + +

Overview

+The Block List middleware is a feature designed to block access to specific API endpoints. Tyk Gateway rejects all requests made to endpoints with the block list enabled, returning `HTTP 403 Forbidden`. + +Note that this is not the same as Tyk's [IP block list](/api-management/gateway-config-tyk-classic#ip-access-control) feature, which is used to restrict access to APIs based upon the IP of the requestor. + +### Use Cases + +#### Prevent access to deprecated resources + +If you are versioning your API and deprecating an endpoint then, instead of having to remove the functionality from your upstream service's API you can simply block access to it using the block list middleware. + +### Working + +Tyk Gateway does not actually maintain a list of blocked endpoints but rather works on the model whereby if the *block list* middleware is added to an endpoint then any request to that endpoint will be rejected, returning `HTTP 403 Forbidden`. + +#### Case sensitivity + +By default the block list is case-sensitive, so for example if you have defined the endpoint `GET /userID` in your API definition then only calls to `GET /userID` will be blocked: calls to `GET /UserID` or `GET /userid` will be allowed. You can configure the middleware to be case-insensitive at the endpoint level. + +You can also set case sensitivity for the entire [gateway](/tyk-oss-gateway/configuration#ignore_endpoint_case) in the Gateway configuration file `tyk.conf`. If case insensitivity is configured at the gateway level, this will override the endpoint-level setting. + +#### Endpoint parsing + +When using the block list middleware, we recommend that you familiarize yourself with Tyk's [URL matching](/getting-started/key-concepts/url-matching) options. + +
+ + +Tyk recommends that you use [exact](/getting-started/key-concepts/url-matching#exact-match) matching for maximum security, though prefix and wildcard strategies might also apply for your particular deployment or use case. + + + +
+ +If you're using Tyk OAS APIs, then you can find details and examples of how to configure the block list middleware [here](#block-list-using-tyk-oas). + +If you're using Tyk Classic APIs, then you can find details and examples of how to configure the block list middleware [here](#block-list-using-classic). + +{/* proposed "summary box" to be shown graphically on each middleware page + # Block List middleware summary + - The Block List is an optional stage in Tyk's API Request processing chain, sitting between the [TBC]() and [TBC]() middleware. + - The Block List can be configured at the per-endpoint level within the API Definition and is supported by the API Designer within the Tyk Dashboard. */} + +

Using Tyk OAS

+The [block list](/api-management/traffic-transformation/block-list) is a feature designed to block access to specific API endpoints. Tyk Gateway rejects all requests made to endpoints with the block list enabled, returning `HTTP 403 Forbidden`. + +When working with Tyk OAS APIs the middleware is configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation). You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the legacy Tyk Classic APIs, then check out the [Tyk Classic](#block-list-using-classic) page. + +### API Definition + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. The `path` can contain wildcards in the form of any string bracketed by curly braces, for example `{user_id}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +The block list middleware (`block`) can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + +The `block` object has the following configuration: +- `enabled`: enable the middleware for the endpoint +- `ignoreCase`: if set to `true` then the path matching will be case insensitive + +For example: +```json {hl_lines=["47-50", "53-56"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-block-list", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/anything": { + "get": { + "operationId": "anythingget", + "responses": { + "200": { + "description": "" + } + } + }, + "put": { + "operationId": "anythingput", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-block-list", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-block-list/", + "strip": true + } + }, + "middleware": { + "operations": { + "anythingget": { + "block": { + "enabled": true, + "ignoreCase": true + } + }, + "anythingput": { + "block": { + "enabled": true, + "ignoreCase": true + } + } + } + } + } +} +``` + +In this example the block list middleware has been configured for requests to the `GET /anything` and `PUT /anything` endpoints. Requests to these endpoints will be rejected with `HTTP 403 Forbidden`. +Note that the block list has been configured to be case insensitive, so calls to `GET /Anything` will also be blocked. +Note also that the endpoint path has not been terminated with `$`. Requests to, for example, `GET /anything/foobar` will be rejected as the [regular expression pattern match](#endpoint-parsing) will recognize this as `GET /anything`. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the block list feature. + +### API Designer + +Adding the block list to your API endpoints is easy is easy when using the API Designer in the Tyk Dashboard, simply follow these steps: + +1. **Add an endpoint** + + From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + + Tyk OAS API Designer showing no endpoints created + + Adding an endpoint to an API using the Tyk OAS API Designer + + Tyk OAS API Designer showing no middleware enabled on endpoint + +2. **Select the Block List middleware** + + Select **ADD MIDDLEWARE** and choose the **Block List** middleware from the *Add Middleware* screen. + + Adding the Block List middleware + +3. **Optionally configure case-insensitivity** + + If you want to disable case-sensitivity for the block list, then you must select **EDIT** on the Block List icon. + + Block List middleware added to endpoint - click through to edit the config + + This takes you to the middleware configuration screen where you can alter the case sensitivity setting. + Configuring case sensitivity for the Block List + + Select **UPDATE MIDDLEWARE** to apply the change to the middleware configuration. + +4. **Save the API** + + Select **SAVE API** to apply the changes to your API. + +

Using Classic

+The [block list](/api-management/traffic-transformation/block-list) is a feature designed to block access to specific API endpoints. Tyk Gateway rejects all requests made to endpoints with the block list enabled, returning `HTTP 403 Forbidden`. + +When working with Tyk Classic APIs the middleware is configured in the Tyk Classic API Definition. You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the newer Tyk OAS APIs, then check out the [Tyk OAS](#block-list-using-tyk-oas) page. + +If you're using Tyk Operator then check out the [configuring the block list in Tyk Operator](#tyk-operator) section below. + +### API Definition + +To enable and configure the block list you must add a new `black_list` object to the `extended_paths` section of your API definition. + + + +Historically, Tyk followed the out-dated whitelist/blacklist naming convention. We are working to remove this terminology from the product and documentation, however this configuration object currently retains the old name. + + + +The `black_list` object has the following configuration: +- `path`: the endpoint path +- `method`: this should be blank +- `ignore_case`: if set to `true` then the path matching will be case insensitive +- `method_actions`: a shared object used to configure the [mock response](/api-management/traffic-transformation/mock-response#when-is-it-useful) middleware + +The `method_actions` object should be configured as follows, with an entry created for each blocked method on the path: +- `action`: this should be set to `no_action` +- `code`: this should be set to `200` +- `headers` : this should be blank + +For example: +```json {linenos=true, linenostart=1} expandable +{ + "extended_paths": { + "black_list": [ + { + "disabled": false, + "path": "/status/200", + "method": "", + "ignore_case": false, + "method_actions": { + "GET": { + "action": "no_action", + "code": 200, + "headers": {} + } + "PUT": { + "action": "no_action", + "code": 200, + "headers": {} + } + } + } + ] + } +} +``` + +In this example the block list middleware has been configured for HTTP `GET` and `PUT` requests to the `/status/200` endpoint. Requests to these endpoints will be rejected with `HTTP 403 Forbidden`. +Note that the block list has been configured to be case sensitive, so calls to `GET /Status/200` will not be rejected. +Note also that the endpoint path has not been terminated with `$`. Requests to, for example, `GET /status/200/foobar` will be rejected as the [regular expression pattern match](#endpoint-parsing) will recognize this as `GET /status/200`. + +Consult section [configuring the Allow List in Tyk Operator](#tyk-operator) for details on how to configure allow lists for endpoints using Tyk Operator. + +### API Designer + +You can use the API Designer in the Tyk Dashboard to configure the block list middleware for your Tyk Classic API by following these steps. + +1. **Add an endpoint for the path and select the plugin** + + From the **Endpoint Designer** add an endpoint that matches the path for which you want to prevent access. Select the **Blacklist** plugin. + +2. **Configure the block list** + + Once you have selected the middleware for the endpoint, the only additional feature that you need to configure is whether to make the middleware case insensitive by selecting **Ignore Case**. + + Blocklist options + +3. **Save the API** + + Use the *save* or *create* buttons to save the changes and activate the middleware. + +### Tyk Operator + +Similar to the configuration of a Tyk Classic API Definition you must add a new `black_list` object to the `extended_paths` section of your API definition. Furthermore, the `use_extended_paths` configuration parameter should be set to `true`. + + + +Historically, Tyk followed the out-dated whitelist/blacklist naming convention. We are working to remove this terminology from the product and documentation, however this configuration object currently retains the old name. + + + +```yaml {linenos=true, linenostart=1, hl_lines=["26-34"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-blacklist +spec: + name: httpbin-blacklist + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org/ + listen_path: /httpbin + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + black_list: + - ignore_case: true + method_actions: + GET: + action: "no_action" + code: 200 + data: "" + headers: {} + path: "/get" +``` + +In this example the block list middleware has been configured for HTTP `GET` requests to the `/get` endpoint. Requests to this endpoint will be rejected with `HTTP 403 Forbidden`. +Note that the block list has been configured to be case insensitive, so calls to `GET /Get` will not be rejected. +Note also that the endpoint path has not been terminated with `$`. Requests to, for example, `GET /get/foobar` will be rejected as the [regular expression pattern match](#endpoint-parsing) will recognize this as `GET /get`. + + + diff --git a/api-management/traffic-transformation/do-not-track.mdx b/api-management/traffic-transformation/do-not-track.mdx new file mode 100644 index 00000000..c75832ea --- /dev/null +++ b/api-management/traffic-transformation/do-not-track.mdx @@ -0,0 +1,262 @@ +--- +title: "Do Not Track" +'og:description': "How to configure Do Not Track traffic transformation middleware in Tyk" +tags: ['Traffic Transformation', 'Do Not Track'] +sidebarTitle: "Do Not Track" +--- + +

Overview

+When [transaction logging](/api-management/logs-metrics#api-traffic-logs) is enabled in the Tyk Gateway, a transaction record will be generated for every request made to an API endpoint deployed on the gateway. You can suppress the generation of transaction records for any API by enabling the do-not-track middleware. This provides granular control over request tracking. + +### Use Cases + +#### Compliance and privacy + +Disabling tracking on endpoints that handle personal or sensitive information is crucial for adhering to privacy laws such as GDPR or HIPAA. This action prevents the storage and logging of sensitive data, ensuring compliance and safeguarding user privacy. + +#### Optimizing performance + +For endpoints experiencing high traffic, disabling tracking can mitigate the impact on the analytics processing pipeline and storage systems. Disabling tracking on endpoints used primarily for health checks or load balancing can prevent the analytics data from being cluttered with information that offers little insight. These optimizations help to maintain system responsiveness and efficiency by reducing unnecessary data load and help to ensure that analytics efforts are concentrated on more meaningful data. + +#### Cost Management + +In scenarios where analytics data storage and processing incur significant costs, particularly in cloud-based deployments, disabling tracking for non-essential endpoints can be a cost-effective strategy. This approach allows for focusing resources on capturing valuable data from critical endpoints. + +### Working + +When transaction logging is enabled, the gateway will automatically generate a transaction record for every request made to deployed APIs. + +You can enable the do-not-track middleware on whichever endpoints for which you do not want to generate logs. This will instruct the Gateway not to generate any transaction records for those endpoints or APIs. As no record of these transactions will be generated by the Gateway, there will be nothing created in Redis and hence nothing for the pumps to transfer to the persistent storage and these endpoints will not show traffic in the Dashboard's analytics screens. + + + +When working with Tyk Classic APIs, you can disable tracking at the API or endpoint-level. When working with Tyk OAS APIs, you can currently disable tracking only at the more granular endpoint-level. + + + +
+ +If you're using Tyk OAS APIs, then you can find details and examples of how to configure the do-not-track middleware [here](/api-management/traffic-transformation/do-not-track#do-not-track-using-tyk-oas). + +If you're using Tyk Classic APIs, then you can find details and examples of how to configure the do-not-track middleware [here](#do-not-track-using-classic). + +{/* proposed "summary box" to be shown graphically on each middleware page + # Do-Not-Track middleware summary + - The Do-Not-Track middleware is an optional stage in Tyk's API Request processing chain sitting between the [TBC]() and [TBC]() middleware. + - The Do-Not-Track middleware can be configured at the per-endpoint level within the API Definition and is supported by the API Designer within the Tyk Dashboard. */} + + +

Using Tyk OAS

+The [Do-Not-Track](#do-not-track-overview) middleware provides the facility to disable generation of transaction records (which are used to track requests to your APIs). When working with Tyk OAS APIs, you can currently disable tracking only at the endpoint-level. + +When working with Tyk OAS APIs the middleware is configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation) either manually within the `.json` file or from the API Designer in the Tyk Dashboard. + +If you're using the legacy Tyk Classic APIs, then check out the [Tyk Classic](#do-not-track-using-classic) page. + +### API Definition + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. The `path` can contain wildcards in the form of any string bracketed by curly braces, for example `{user_id}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +The do-not-track middleware (`doNotTrackEndpoint`) can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + +The `doNotTrackEndpoint` object has the following configuration: +- `enabled`: enable the middleware for the endpoint + +For example: +```json {hl_lines=["39-41"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-do-not-track", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/anything": { + "get": { + "operationId": "anythingget", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-do-not-track", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-do-not-track/", + "strip": true + } + }, + "middleware": { + "operations": { + "anythingget": { + "doNotTrackEndpoint": { + "enabled": true + } + } + } + } + } +} +``` + +In this example the do-not-track middleware has been configured for requests to the `GET /anything` endpoint. Any such calls will not generate transaction records from the Gateway and so will not appear in the analytics. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the do-not-track middleware. + +### API Designer + +Adding do-not-track to your API endpoints is easy when using the API Designer in the Tyk Dashboard, simply follow these steps: + +1. **Add an endpoint** + + From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + + Tyk OAS API Designer showing no endpoints created + + Adding an endpoint to an API using the Tyk OAS API Designer + + Tyk OAS API Designer showing no middleware enabled on endpoint + +2. **Select the Do Not Track Endpoint middleware** + + Select **ADD MIDDLEWARE** and choose the **Do Not Track Endpoint** middleware from the *Add Middleware* screen. + + Adding the Do Not Track middleware + +3. **Save the API** + + Select **SAVE API** to apply the changes to your API. + +

Using Classic

+The [Do-Not-Track](#do-not-track-overview) middleware provides the facility to disable generation of transaction records (which are used to track requests) at the API or endpoint level. + +When working with Tyk Classic APIs the middleware is configured in the Tyk Classic API Definition either manually within the `.json` file or from the API Designer in the Tyk Dashboard. + +If you're using the newer Tyk OAS APIs, then check out the [Tyk OAS](/api-management/traffic-transformation/do-not-track#do-not-track-using-tyk-oas) page. + +If you're using Tyk Operator then check out the [configuring the middleware in Tyk Operator](#tyk-operator) section below. + +### API Definition + +You can prevent tracking for all endpoints of an API by configuring the `do_not_track` field in the root of your API definition. +- `true`: no transaction logs will be generated for requests to the API +- `false`: transaction logs will be generated for requests to the API + +If you want to be more granular and disable tracking only for selected endpoints, then you must add a new `do_not_track_endpoints` object to the `extended_paths` section of your API definition. + +The `do_not_track_endpoints` object has the following configuration: +- `path`: the endpoint path +- `method`: the endpoint HTTP method + +The `path` can contain wildcards in the form of any string bracketed by curly braces, for example `{user_id}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +For example: +```json {linenos=true, linenostart=1} expandable +{ + "extended_paths": { + "do_not_track_endpoints": [ + { + "disabled": false, + "path": "/anything", + "method": "GET" + } + ] + } +} +``` + +In this example the do-not-track middleware has been configured for requests to the `GET /anything` endpoint. Any such calls will not generate transaction records from the Gateway and so will not appear in the analytics. + +### API Designer + +You can use the API Designer in the Tyk Dashboard to configure the per-endpoint Do-Not-Track middleware for your Tyk Classic API by following these steps. Note that the API-level middleware can only be configured from the Raw Definition screen. + +1. **Add an endpoint for the path and select the plugin** + + From the **Endpoint Designer** add an endpoint that matches the path for which you do not want to generate records. Select the **Do not track endpoint** plugin. + + Select the middleware + +2. **Save the API** + + Use the *save* or *create* buttons to save the changes and activate the middleware. + +### Tyk Operator + +The process for configuring the middleware in Tyk Operator is similar to that explained in configuring the middleware in the Tyk Classic API Definition. + +It is possible to prevent tracking for all endpoints of an API by configuring the `do_not_track` field in the root of your API definition as follows: + +- `true`: no transaction logs will be generated for requests to the API +- `false`: transaction logs will be generated for requests to the API + +```yaml {linenos=true, linenostart=1, hl_lines=["10"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-do-not-track +spec: + name: httpbin-do-not-track + use_keyless: true + protocol: http + active: true + do_not_track: true + proxy: + target_url: http://example.com + listen_path: /example + strip_listen_path: true +``` + +If you want to disable tracking only for selected endpoints, then the process is similar to that defined in configuring the middleware in the Tyk Classic API Definition, i.e. you must add a new `do_not_track_endpoints` list to the extended_paths section of your API definition. +This should contain a list of objects representing each endpoint `path` and `method` that should have tracking disabled: + +```yaml {linenos=true, linenostart=1, hl_lines=["31-33"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-endpoint-tracking +spec: + name: httpbin - Endpoint Track + use_keyless: true + protocol: http + active: true + do_not_track: false + proxy: + target_url: http://httpbin.org/ + listen_path: /httpbin + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + track_endpoints: + - method: GET + path: "/get" + do_not_track_endpoints: + - method: GET + path: "/headers" +``` + +In the example above we can see that the `do_not_track_endpoints` list is configured so that requests to `GET /headers` will have tracking disabled. + diff --git a/api-management/traffic-transformation/go-templates.mdx b/api-management/traffic-transformation/go-templates.mdx new file mode 100644 index 00000000..22641de4 --- /dev/null +++ b/api-management/traffic-transformation/go-templates.mdx @@ -0,0 +1,229 @@ +--- +title: "Go Templates" +'og:description': "How to configure Go Templates traffic transformation middleware in Tyk" +tags: ['Traffic Transformation', 'Go Templates'] +sidebarTitle: "Go Templates" +--- + +Tyk's [request](/api-management/traffic-transformation/request-body) and [response](/api-management/traffic-transformation/response-body) body transform middleware use the [Go template language](https://golang.org/pkg/text/template/) to parse and modify the provided input. + +Go templates are also used by Tyk's [webhook event handler](/api-management/gateway-events#event-handling-with-webhooks) to produce the payload for the HTTP request sent to the target system. + +In this section of the documentation, we provide some guidance and a few examples on the use of Go templating with Tyk. + +## Data format conversion using helper functions + +Tyk provides two helper functions to assist with data format translation between JSON and XML: +- `jsonMarshal` performs JSON style character escaping on an XML field and, for complex objects, serialises them to a JSON string ([example](#xml-to-json-conversion-using-jsonmarshal)) +- `xmlMarshal` performs the equivalent conversion from JSON to XML ([example](#json-to-xml-conversion-using-xmlmarshal)) + +When creating these functions within your Go templates, please note: +- the use of `.` in the template refers to the entire input, whereas something like `.myField` refers to just the `myField` field of the input +- the pipe `|` joins together the things either side of it, which is typically input data on the left and a receiving command to process the data on the right, such as `jsonMarshal` + +Hence `{{ . | jsonMarshal }}` will pass the entire input to the `jsonMarshal` helper function. + +## Using functions within Go templates + +You can define and use functions in the Go templates that are used for body transforms in Tyk. Functions allow you to abstract common template logic for cleaner code and to aid reusability. Breaking the template into functions improves readability of more complex tenplates. + +Here is an example where we define a function called `myFunction` that accepts one parameter: +```go +{{- define "myFunction" }} + Hello {{.}}! +{{- end}} +``` + +We can call that function and pass "world" as the parameter: +```go +{ + "message": {{ call . "myFunction" "world"}} +} +``` + +The output would be: +```json +{ + "message": "Hello world!" +} +``` + +We have bundled the [Sprig Library (v3)](http://masterminds.github.io/sprig/) which provides over 70 pre-written functions for transformations to assist the creation of powerful Go templates to transform your API requests. + +## Additional resources + +Here's a useful [blog post](https://blog.gopheracademy.com/advent-2017/using-go-templates/) and [YouTube tutorial](https://www.youtube.com/watch?v=k5wJv4XO7a0) that can help you to learn about using Go templates. + +## Go templating examples +Here we provide worked examples for both [JSON](#example-json-transformation-template) and [XML](#example-xml-transformation-template) formatted inputs. We also explain examples using the [jsonMarshal](#xml-to-json-conversion-using-jsonmarshal) and [xmlMarshal](#json-to-xml-conversion-using-xmlmarshal) helper functions. + +### Example JSON transformation template +Imagine you have a published API that accepts the request listed below, but your upstream service requires a few alterations, namely: +- swapping the values of parameters `value1` and `value2` +- renaming the `value_list` to `transformed_list` +- adding a `user-id` extracted from the session metadata +- adding a `client-ip` logging the client IP +- adding a `req-type` that logs the value provided in query parameter `type` + +**Input** +- Session metadata `uid` = `user123` +- IP address of calling client = `192.0.0.1` +- Query parameter `type` = `strict` +```json expandable +{ + "value1": "value-1", + "value2": "value-2", + "value_list": [ + "one", + "two", + "three" + ] +} +``` + +**Template** +```go expandable +{ + "value1": "{{.value2}}", + "value2": "{{.value1}}", + "transformed_list": [ + {{range $index, $element := index . "value_list"}} + {{if $index}}, {{end}} + "{{$element}}" + {{end}} + ], + "user-id": "{{._tyk_meta.uid}}", + "user-ip": "{{._tyk_context.remote_addr}}", + "req-type": "{{ ._tyk_context.request_data.param.type }}" +} +``` +In this template: +- `.value1` accesses the "value1" field of the input JSON +- we swap value1 and value2 +- we use the range function to loop through the "value_list" array +- `._tyk_meta.uid` injects the "uid" session metadata value +- `._tyk_context.remote_addr` injects the client IP address from the context +- `._tyk_context.request_data.param.type` injects query parameter "type" + +**Output** +``` .json expandable +{ + "value1": "value-2", + "value2": "value-1", + "transformed_list": [ + "one", + "two", + "three" + ], + "user-id": "user123" + "user-ip": "192.0.0.1" + "req-type": "strict" +} +``` + +### Example XML transformation template +XML cannot be as easily decoded into strict structures as JSON, so the syntax is a little different when working with an XML document. Here we are performing the reverse translation, starting with XML and converting to JSON. + +**Input** +- Session metadata `uid` = `user123` +- IP address of calling client = `192.0.0.1` +- Query parameter `type` = `strict` +```xml expandable + + + + value-1 + value-2 + + one + two + three + + + +``` + +**Template** +``` .xml expandable + + + + {{ .data.body.value2 }} + {{ .data.body.value1 }} + + {{range $index, $element := .data.body.valueList.item }} + {{$element}} + {{end}} + + {{ ._tyk_meta.uid }} + {{ ._tyk_context.remote_addr }} + {{ ._tyk_context.request_data.param.type }} + + +``` +In this template: +- `.data.body.value1` accesses the "value1" field of the input XML +- we swap value1 and value2 +- we use the range function to loop through the "value_list" array +- `._tyk_meta.uid` injects the "uid" session metadata value +- `._tyk_context.remote_addr` injects the client IP address from the context +- `._tyk_context.request_data.param.type` injects query parameter "type" + +**Output** +``` .xml expandable + + + + value-2 + value-1 + + one + two + three + + user123 + 192.0.0.1 + strict + + +``` + +### XML to JSON conversion using jsonMarshal +The `jsonMarshal` function converts XML formatted input into JSON, for example: + +**Input** +```xml +world +``` + +**Template** +```go +{{ . | jsonMarshal }} +``` + +**Output** +```json +{"hello":"world"} +``` + +Note that in this example, Go will step through the entire data structure provided to the template. When used in the [Request](/api-management/traffic-transformation/request-body#data-accessible-to-the-middleware) or [Response](/api-management/traffic-transformation/request-body#data-accessible-to-the-middleware) Body Transform middleware, this would include Context Variables and Session Metadata if provided to the middleware. + +### JSON to XML conversion using xmlMarshal +The `xmlMarshal` function converts JSON formatted input into XML, for example: + +**Input** +```json +{"hello":"world"} +``` +**Template** +``` .go +{{ . | xmlMarshal }} +``` + +**Output** +```xml +world +``` + +Note that in this example, Go will step through the entire data structure provided to the template. When used in the [Request](/api-management/traffic-transformation/request-body#data-accessible-to-the-middleware) or [Response](/api-management/traffic-transformation/request-body#data-accessible-to-the-middleware) Body Transform middleware, this would include Context Variables and Session Metadata if provided to the middleware. + diff --git a/api-management/traffic-transformation/ignore-authentication.mdx b/api-management/traffic-transformation/ignore-authentication.mdx new file mode 100644 index 00000000..b8a2eacc --- /dev/null +++ b/api-management/traffic-transformation/ignore-authentication.mdx @@ -0,0 +1,299 @@ +--- +title: "Ignore Authentication" +'og:description': "How to configure Ignore Authentication traffic transformation middleware in Tyk" +tags: ['Traffic Transformation', 'Ignore Authentication'] +sidebarTitle: "Ignore Authentication" +--- + +

Overview

+The Ignore Authentication middleware instructs Tyk Gateway to skip the authentication step for calls to an endpoint, even if authentication is enabled for the API. + +### Use Cases + +#### Health and liveness endpoints + +This plugin can be very useful if you have an endpoint (such as a ping or health check) that you don’t need to secure. + +### Working + +When the Ignore Authentication middleware is configured for a specific endpoint, it instructs the gateway to bypass the client authentication process for requests made to that endpoint. If other (non-authentication) middleware are configured for the endpoint, they will still execute on the request. + +It is important to exercise caution when using the Ignore Authentication middleware, as it effectively disables Tyk's security features for the ignored paths. Only endpoints that are designed to be public or have independent security mechanisms should be configured to bypass authentication in this way. When combining Ignore Authentication with response transformations be careful not to inadvertently expose sensitive data or rely on authentication or session data that is not present. + +#### Case sensitivity + +By default the ignore authentication middleware is case-sensitive. If, for example, you have defined the endpoint `GET /ping` in your API definition then only calls to `GET /ping` will ignore the authentication step: calls to `GET /Ping` or `GET /PING` will require authentication. You can configure the middleware to be case insensitive at the endpoint level. + +You can also set case sensitivity for the entire Tyk Gateway in its [configuration file](/tyk-oss-gateway/configuration#ignore_endpoint_case) `tyk.conf`. If case insensitivity is configured at the gateway level, this will override the endpoint-level setting. + +#### Endpoint parsing + +When using the ignore authentication middleware, we recommend that you familiarize yourself with Tyk's [URL matching](/getting-started/key-concepts/url-matching) options. + +
+ + +Tyk recommends that you use [exact](/getting-started/key-concepts/url-matching#exact-match) matching for maximum security, though prefix and wildcard strategies might also apply for your particular deployment or use case. + + + +
+ +If you're using Tyk OAS APIs, then you can find details and examples of how to configure the ignore authentication middleware [here](#ignore-authentication-using-tyk-oas). + +If you're using Tyk Classic APIs, then you can find details and examples of how to configure the ignore authentication middleware [here](#ignore-authentication-using-classic). + +{/* proposed "summary box" to be shown graphically on each middleware page + # Ignore Authentication middleware summary + - The Ignore Authentication middleware is an optional stage in Tyk's API Request processing chain, sitting between the [TBC]() and [TBC]() middleware. + - The Ignore Authentication middleware can be configured at the per-endpoint level within the API Definition and is supported by the API Designer within the Tyk Dashboard. */} + + +

Using Tyk OAS

+The [Ignore Authentication](/api-management/traffic-transformation/ignore-authentication) middleware instructs Tyk Gateway to skip the authentication step for calls to an endpoint, even if authentication is enabled for the API. + +When working with Tyk OAS APIs the middleware is configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation). You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the legacy Tyk Classic APIs, then check out the [Tyk Classic](#ignore-authentication-using-classic) page. + +### API Definition + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. Endpoint `paths` entries (and the associated `operationId`) can contain wildcards in the form of any string bracketed by curly braces, for example `/status/{code}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +The ignore authentication middleware (`ignoreAuthentication`) can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + +The `ignoreAuthentication` object has the following configuration: +- `enabled`: enable the middleware for the endpoint +- `ignoreCase`: if set to `true` then the path matching will be case insensitive + +For example: +```json {hl_lines=["65-69"],linenos=true, linenostart=1} expandable +{ + "info": { + "title": "example-ignore-authentication", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "servers": [ + { + "url": "http://localhost:8181/example-ignore-authentication/" + } + ], + "security": [ + { + "authToken": [] + } + ], + "paths": { + "/anything": { + "get": { + "operationId": "anythingget", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "components": { + "securitySchemes": { + "authToken": { + "type": "apiKey", + "in": "header", + "name": "Authorization" + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-ignore-authentication", + "state": { + "active": true, + "internal": false + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "authentication": { + "enabled": true, + "securitySchemes": { + "authToken": { + "enabled": true + } + } + }, + "listenPath": { + "strip": true, + "value": "/example-ignore-authentication/" + } + }, + "middleware": { + "operations": { + "anythingget": { + "ignoreAuthentication": { + "enabled": true + } + } + } + } + } +} +``` + +In this example the ignore authentication middleware has been configured for requests to the `GET /anything` endpoint. Any such calls will skip the authentication step in the Tyk Gateway's processing chain. +- the middleware has been configured to be case sensitive, so calls to `GET /Anything` will not skip authentication + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the Ignore Authentication middleware. + +### API Designer + +Adding and configuring the Ignore Authentication middleware to your API endpoints is easy when using the API Designer in the Tyk Dashboard, simply follow these steps: + +1. **Add an endpoint** + + From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + + Tyk OAS API Designer showing no endpoints created + + Adding an endpoint to an API using the Tyk OAS API Designer + + Tyk OAS API Designer showing no middleware enabled on endpoint + +2. **Select the Ignore Authentication middleware** + + Select **ADD MIDDLEWARE** and choose the **Ignore Authentication** middleware from the *Add Middleware* screen. + + Adding the Ignore Authentication middleware + +3. **Optionally configure case-insensitivity** + + If you want to disable case-sensitivity for the path that you wish to skip authentication, then you must select **EDIT** on the Ignore Authentication icon. + + Ignore Authentication middleware added to endpoint - click through to edit the config + + This takes you to the middleware configuration screen where you can alter the case sensitivity setting. + Configuring case sensitivity for the path for which to ignore authentication + + Select **UPDATE MIDDLEWARE** to apply the change to the middleware configuration. + +4. **Save the API** + + Select **SAVE API** to apply the changes to your API. + +

Using Classic

+The [Ignore Authentication](/api-management/traffic-transformation/ignore-authentication) middleware instructs Tyk Gateway to skip the authentication step for calls to an endpoint, even if authentication is enabled for the API. + +When working with Tyk Classic APIs the middleware is configured in the Tyk Classic API Definition. You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the newer Tyk OAS APIs, then check out the [Tyk OAS](#ignore-authentication-using-tyk-oas) page. + +If you're using Tyk Operator then check out the [configuring the middleware in Tyk Operator](#tyk-operator) section below. + +### API Definition + +To enable the middleware you must add a new `ignored` object to the `extended_paths` section of your API definition. + +The `ignored` object has the following configuration: +- `path`: the endpoint path +- `method`: this should be blank +- `ignore_case`: if set to `true` then the path matching will be case insensitive +- `method_actions`: a shared object used to configure the [mock response](/api-management/traffic-transformation/mock-response#when-is-it-useful) middleware + +The `method_actions` object should be configured as follows, with an entry created for each allowed method on the path: +- `action`: this should be set to `no_action` +- `code`: this should be set to `200` +- `headers` : this should be blank + +For example: +```json {linenos=true, linenostart=1} expandable +{ + "extended_paths": { + "ignored": [ + { + "disabled": false, + "path": "/status/200", + "method": "", + "ignore_case": false, + "method_actions": { + "GET": { + "action": "no_action", + "code": 200, + "headers": {} + } + } + } + ] + } +} +``` + +In this example the ignore authentication middleware has been configured for requests to the `GET /status/200` endpoint. Any such calls will skip the authentication step in the Tyk Gateway's processing chain. +- the middleware has been configured to be case sensitive, so calls to `GET /Status/200` will not skip authentication + +### API Designer + +You can use the API Designer in the Tyk Dashboard to configure the Ignore Authentication middleware for your Tyk Classic API by following these steps. + +1. **Add an endpoint for the path and select the plugin** + + From the **Endpoint Designer** add an endpoint that matches the path for which you want to ignore authentication. Select the **Ignore** plugin. + + Adding the ignore authentication middleware to a Tyk Classic API endpoint + +2. **Configure the middleware** + + Once you have selected the Ignore middleware for the endpoint, the only additional feature that you need to configure is whether to make it case-insensitive by selecting **Ignore Case**. + + Ignore options + +3. **Save the API** + + Use the *save* or *create* buttons to save the changes and activate the middleware. + +### Tyk Operator + +The process for configuring the middleware in Tyk Operator is similar to that explained in configuring the middleware in the Tyk Classic API Definition. It is possible to configure an enforced timeout using the `ignored` object within the `extended_paths` section of the API Definition. + +In the example below the ignore authentication middleware has been configured for requests to the `GET /get` endpoint. Any such calls will skip the authentication step in the Tyk Gateway's processing chain. +- the middleware has been configured to be case insensitive, so calls to `GET /Get` will also skip authentication + +```yaml {linenos=true, linenostart=1, hl_lines=["27-35"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-ignored +spec: + name: httpbin-ignored + use_keyless: false + use_standard_auth: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org/ + listen_path: /httpbin + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + ignored: + - ignore_case: true + method_actions: + GET: + action: "no_action" + code: 200 + data: "" + headers: {} + path: "/get" +``` \ No newline at end of file diff --git a/api-management/traffic-transformation/internal-endpoint.mdx b/api-management/traffic-transformation/internal-endpoint.mdx new file mode 100644 index 00000000..ee8f6fa7 --- /dev/null +++ b/api-management/traffic-transformation/internal-endpoint.mdx @@ -0,0 +1,268 @@ +--- +title: "Internal Endpoint" +'og:description': "How to configure Internal Endpoint traffic transformation middleware in Tyk" +tags: ['Traffic Transformation', 'Internal Endpoint'] +sidebarTitle: "Internal Endpoint" +--- + +

Overview

+The Internal Endpoint middleware instructs Tyk Gateway to ignore external requests to the endpoint (which is a combination of HTTP method and path). Internal requests from other APIs will be processed. + +### Use Cases + +#### Internal routing decisions + +Internal endpoints are frequently used to make complex routing decisions that cannot be handled by the standard routing features. A single externally published endpoint can receive requests and then, based on inspection of the requests, the [URL rewrite](/transform-traffic/url-rewriting#url-rewrite-middleware) middleware can route them to different internal endpoints and on to the appropriate upstream services. + +### Working + +When the Internal Endpoint middleware is configured for a specific endpoint, it instructs the Gateway to ignore requests to the endpoint that originate from outside Tyk. + +An internal endpoint can be targeted from another API deployed on Tyk using the `tyk://` prefix instead of `http://`. + +For example, if `GET /status/200` is configured to be an Internal Endpoint on the listen path `http://my-tyk-install.org/my-api/` then external calls to this endpoint will be rejected with `HTTP 403 Forbidden`. Other APIs on Tyk will be able to direct traffic to this endpoint by setting their `target_url` to `tyk://my-api/status/200`. + +#### Addressing an internal endpoint + +An internal endpoint can be addressed using three different identifiers in the format `tyk://{identifier}/{endpoint}`. + +The options for the `identifier` are: +- `self` (only if the endpoint is in the same API) +- `api_id` (the unique API Identifier assigned to the API within Tyk) +- listen path (the listen path defined for the API) + +For example, let's say you have two APIs: + +| api_id | listen path | Endpoint 1 | Endpoint 2 (with internal endpoint middleware) | +| :-------- | :------------- | :-------------- | :------------------------------------------------ | +| f1c63fa5177de2719 | `/api1` | `endpoint1_ext` | `endpoint1_int` | +| 2e90b33a879945918 | `/api2` | `endpoint2_ext` | `endpoint2_int` | + +An external request directed at `/api1/endpoint1_int` will be rejected with `HTTP 403 Forbidden`, since this is an internal endpoint. + +This endpoint could, however, be called from within either endpoint in `/api2` as either: +- `tyk://api1/endpoint1_int` +- `tyk://f1c63fa5177de2719/endpoint1_int` + +Or from within `/api1/endpoint1_ext` as: +- `tyk://api1/endpoint1_int` +- `tyk://f1c63fa5177de2719/endpoint1_int` +- `tyk://self/endpoint1_int` + +
+ +If you're using Tyk OAS APIs, then you can find details and examples of how to configure the Internal Endpoint middleware [here](#internal-endpoint-using-tyk-oas). + +If you're using Tyk Classic APIs, then you can find details and examples of how to configure the Internal Endpoint middleware [here](#internal-endpoint-using-classic). + +{/* proposed "summary box" to be shown graphically on each middleware page + # Internal Endpoint middleware summary + - The Internal Endpoint middleware is an optional stage in Tyk's API Request processing chain, sitting between the [TBC]() and [TBC]() middleware. + - The Internal Endpoint middleware can be configured at the per-endpoint level within the API Definition and is supported by the API Designer within the Tyk Dashboard. */} + + + +

Using Tyk OAS

+The [Internal Endpoint](#internal-endpoint-overview) middleware instructs Tyk Gateway not to process external requests to the endpoint (which is a combination of HTTP method and path). Internal requests from other APIs will be processed. + +When working with Tyk OAS APIs, the middleware is configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation). You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the legacy Tyk Classic APIs, then check out the [Tyk Classic](#internal-endpoint-using-classic) page. + +### API Definition + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. Endpoint `paths` entries (and the associated `operationId`) can contain wildcards in the form of any string bracketed by curly braces, for example `/status/{code}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +The internal endpoint middleware (`internal`) can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + +The `internal` object has the following configuration: +- `enabled`: enable the middleware for the endpoint + +For example: +```json {hl_lines=["49-50"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-internal-endpoint", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/anything": { + "get": { + "operationId": "anythingget", + "responses": { + "200": { + "description": "" + } + } + } + }, + "/redirect": { + "get": { + "operationId": "redirectget", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-internal-endpoint", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-internal-endpoint/", + "strip": true + } + }, + "middleware": { + "operations": { + "anythingget": { + "internal": { + "enabled": true + } + }, + "redirectget": { + "urlRewrite": { + "enabled": true, + "pattern": ".*", + "rewriteTo": "tyk://self/anything" + } + } + } + } + } +} +``` + +In this example, two endpoints have been defined: +- the internal endpoint middleware has been configured for requests to the `GET /anything` endpoint +- the [URL rewrite](/transform-traffic/url-rewriting#url-rewrite-middleware) middleware has been configured for requests to the `GET /redirect` endpoint + +Any calls made directly to `GET /example-internal-endpoint/anything` will be rejected, with Tyk returning `HTTP 403 Forbidden`, since the `/anything` endpoint is internal. + +Any calls made to `GET /example-internal-endpoint/redirect` will be redirected to `GET /example-internal-endpoint/anything`. These will be proxied to the upstream because they originate from within Tyk Gateway (i.e. they are internal requests) - so the response from `GET http://httpbin.org/anything` will be returned. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the internal endpoint middleware. + +### API Designer + +Adding the Internal Endpoint middleware to your API endpoints is easy when using the API Designer in the Tyk Dashboard, simply follow these steps: + +1. **Add an endpoint** + + From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + + Tyk OAS API Designer showing no endpoints created + + Adding an endpoint to an API using the Tyk OAS API Designer + + Tyk OAS API Designer showing no middleware enabled on endpoint + +2. **Select the Internal Endpoint middleware** + + Select **ADD MIDDLEWARE** and choose the **Internal** middleware from the *Add Middleware* screen. + + Adding the Internal Endpoint middleware + +3. **Save the API** + + Select **SAVE API** to apply the changes to your API. + +

Using Classic

+The [Internal Endpoint](#internal-endpoint-overview) middleware instructs Tyk Gateway not to process external requests to the endpoint (which is a combination of HTTP method and path). Internal requests from other APIs will be processed. + +When working with Tyk Classic APIs, the middleware is configured in the Tyk Classic API Definition. You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the newer Tyk OAS APIs, then check out the [Tyk OAS](#internal-endpoint-using-tyk-oas) page. + +If you're using Tyk Operator then check out the [configuring the middleware in Tyk Operator](#tyk-operator) section below. + +### API Definition + +To enable the middleware you must add a new `internal` object to the `extended_paths` section of your API definition. + +The `internal` object has the following configuration: +- `path`: the endpoint path +- `method`: the endpoint HTTP method + +For example: +```.json {linenos=true, linenostart=1} expandable +{ + "extended_paths": { + "internal": [ + { + "disabled": false, + "path": "/status/200", + "method": "GET" + } + ] + } +} +``` + +In this example the internal endpoint middleware has been configured for HTTP `GET` requests to the `/status/200` endpoint. Any requests made to this endpoint that originate externally to Tyk will be rejected with `HTTP 403 Forbidden`. Conversely, the endpoint can be reached internally by another API at `tyk:///status/200`. + +### API Designer + +You can use the API Designer in the Tyk Dashboard to configure the internal endpoint middleware for your Tyk Classic API by following these steps. + +1. **Add an endpoint for the path and select the plugin** + + From the **Endpoint Designer** add an endpoint that matches the path that you wish to set as internal. Select the **Internal** plugin. + + Adding the internal endpoint middleware to a Tyk Classic API endpoint + +2. **Save the API** + + Use the *save* or *create* buttons to save the changes and activate the middleware. + +### Tyk Operator + +The process for configuring the middleware in Tyk Operator is similar to that explained in configuring the middleware in the Tyk Classic API Definition. The middleware can be configured by adding a new `internal` object to the `extended_paths` section of your API definition. + +In the example below the internal endpoint middleware has been configured for HTTP `GET` requests to the `/status/200` endpoint. Any requests made to this endpoint that originate externally to Tyk will be rejected with `HTTP 403 Forbidden`. Conversely, the endpoint can be reached internally by another API at `tyk:///status/200`. + +```yaml {linenos=true, linenostart=1, hl_lines=["26-28"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-endpoint-internal +spec: + name: httpbin - Endpoint Internal + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org/ + listen_path: /httpbin-internal + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + internal: + - path: /status/200 + method: GET +``` + + + diff --git a/api-management/traffic-transformation/jq-transforms.mdx b/api-management/traffic-transformation/jq-transforms.mdx new file mode 100644 index 00000000..73fb20dd --- /dev/null +++ b/api-management/traffic-transformation/jq-transforms.mdx @@ -0,0 +1,65 @@ +--- +title: "JQ Transforms" +'og:description': "How to configure JQ Transforms traffic transformation middleware in Tyk" +tags: ['Traffic Transformation', 'JQ Transforms'] +sidebarTitle: "JQ Transforms" +--- + + + +This feature is experimental and can be used only if you compile Tyk yourself own using `jq` tag: `go build --tags 'jq'` + + + + +If you work with JSON you are probably aware of the popular `jq` command line JSON processor. For more details, see https://stedolan.github.io/jq/. + +Now you can use the full power of its queries and transformations to transform requests, responses, headers and even context variables. + +We have added two new plugins: + +* `transform_jq` - for request transforms. +* `transform_jq_response` - for response transforms + +Both have the same structure, similar to the rest of our plugins: +`{ "path": "", "method": "", "filter": "" }` + +## Request Transforms +Inside a request transform you can use following variables: + +* `.body` - your current request body +* `._tyk_context` - Tyk context variables. You can use it to access request headers as well. + +Your JQ request transform should return an object in the following format: +`{ "body": , "rewrite_headers": , "tyk_context": }`. + +`body` is required, while `rewrite_headers` and `tyk_context` are optional. + +## Response Transforms +Inside a response transform you can use following variables: + +* `.body` - your current response body +* `._tyk_context` - Tyk context variables. You can use it to access request headers as well. +* `._tyk_response_headers` - Access to response headers + +Your JQ response transform should return an object in the following format: +`{ "body": , "rewrite_headers": }`. + +`body` is required, while `rewrite_headers` is optional. + +## Example +```{.json} expandable +"extended_paths": { + "transform_jq": [{ + "path": "/post", + "method": "POST", + "filter": "{\"body\": (.body + {\"TRANSFORMED-REQUEST-BY-JQ\": true, path: ._tyk_context.path, user_agent: ._tyk_context.headers_User_Agent}), \"rewrite_headers\": {\"X-added-rewrite-headers\": \"test\"}, \"tyk_context\": {\"m2m_origin\": \"CSE3219/C9886\", \"deviceid\": .body.DEVICEID}}" + }], + "transform_jq_response": [{ + "path": "/post", + "method": "POST", + "filter": "{\"body\": (.body + {\"TRANSFORMED-RESPONSE-BY-JQ\": true, \"HEADERS-OF-RESPONSE\": ._tyk_response_headers}), \"rewrite_headers\": {\"JQ-Response-header\": .body.origin}}" + }] +} +``` + diff --git a/api-management/traffic-transformation/mock-response.mdx b/api-management/traffic-transformation/mock-response.mdx new file mode 100644 index 00000000..2eec6246 --- /dev/null +++ b/api-management/traffic-transformation/mock-response.mdx @@ -0,0 +1,797 @@ +--- +title: "Mock Response" +'og:description': "How to configure Mock Response traffic transformation middleware in Tyk" +tags: ['Traffic Transformation', 'Mock Response'] +sidebarTitle: "Mock Response" +--- + +

Overview

+A mock response is a simulated API response that can be returned by the API gateway without actually sending the request to the backend API service. Mock responses are an integral feature for API development, enabling developers to emulate API behavior without the need for upstream execution. + +Tyk's mock response middleware offers this functionality by allowing the configuration of custom responses for designated endpoints. This capability is crucial for facilitating front-end development, conducting thorough testing, and managing unexpected scenarios or failures. + +### When is it useful + +#### Rapid API Prototyping + +Developers can create predefined static (mock) responses during early API prototyping phases to simulate responses without any backend. This is useful for several reasons: + +- **Validate API Design Early**: It enables [trying an API before writing any code](https://tyk.io/blog/3-ways-to-try-out-your-api-design-before-you-build). API designers and product managers can validate concepts without waiting for full API implementations. +- **Enable Dependent Development**: Allows development of apps and tooling that depend on the upstream service to progress. For example, the front-end team can build their interface based on the mocked responses. +- **Support Test-Driven Development (TDD) and Behavior-Driven Development (BDD)**: Supports writing test cases for the API before implementation, enabling design and testing of the API without writing any backend code. + +#### Legacy System Migration + +When migrating from a legacy system to new APIs, mock responses can be used to emulate the old system's outputs during the transitional phases. This provides continuity for client apps relying on the old system while new APIs are built that will eventually replace the legacy hooks. + +#### Disaster Recovery Testing + +The ability for a gateway to return well-formed mocks when backend APIs are unavailable helps test disaster recovery plans. By intentionally taking APIs offline and verifying the mocks' surface instead, developers gain confidence in the gateway's fallback and business continuity capabilities + +#### Enhanced CI/CD pipeline + +Test cases that rely on API interactions can mock those dependencies and provide virtual test data. This removes wait times for real API calls to complete during automated builds. Consumer testing can verify that provider APIs meet expected contracts using mocks in the CI pipeline. This ensures the contract remains intact across code changes before deployment. Front-end/client code can scale release cycles faster than backend APIs by using mocks to simulate planned API behaviors before they are ready. + +### Working + +When the Mock Response middleware is configured for a specific endpoint, it terminates requests to the endpoint and generates an HTTP response that will be returned to the client as if it had come from the upstream service. No request will be passed to the upstream. The mock response to an API request should be designed to emulate how the service would respond to requests. To enable this, the content of the response can be configured to match the API contract for the service: you can set the HTTP status code, body and headers for the response. + +### Advanced mock responses with Tyk OAS + +When working with Tyk OAS APIs, Tyk Gateway can parse the [examples and schema](/api-management/traffic-transformation/mock-response#mock-responses-using-openapi-metadata) in the OpenAPI description and use this to automatically generate responses using those examples. Where multiple examples are defined, for example for different response codes, Tyk enables you to [configure special headers](#multiple-mock-responses-for-a-single-endpoint) in the request to select the desired mock response. + +### Middleware execution order during request processing + +#### With **Tyk OAS APIs** + +- the mock response middleware is executed at the **end** of the request processing chain, immediately prior to the request being proxied to the upstream +- all other request processing middleware (e.g. authentication, request transforms) will be executed prior to the mock response. + +#### With **Tyk Classic APIs** + +- the mock response middleware is executed at the **start** of the request processing chain +- an endpoint with a mock response will not run any other middleware and will immediately return the mocked response for any request. As such, it is always an unauthenticated (keyless) call. + +
+ +If you’re using Tyk OAS APIs, then you can find details and examples of how to configure the mock response middleware [here](#mock-response-using-tyk-oas). + +If you’re using Tyk Classic APIs, then you can find details and examples of how to configure the response body transformation middleware [here](#mock-response-using-classic). + +{/* proposed "summary box" to be shown graphically on each middleware page + ### Mock Response middleware summary + - The Mock Response middleware is an optional stage in Tyk's API Request processing chain, sitting between the [TBC]() and [TBC]() middleware. + - The Mock Response middleware can be configured at the per-endpoint level within the API Definition and is supported by the API Designer within the Tyk Dashboard. */} + + + +## Mock Responses using OpenAPI Metadata + +The [OpenAPI Specification](https://learn.openapis.org/specification/docs.html#adding-examples) provides metadata that can be used by automatic documentation generators to create comprehensive reference guides for APIs. Most objects in the specification include a `description` field that offers additional human-readable information for documentation. Alongside descriptions, some OpenAPI objects can include sample values in the OpenAPI Document, enhancing the generated documentation by providing representative content that the upstream service might return in responses. + +Tyk leverages examples from your API documentation (in OpenAPI Spec format) to generate mock responses for the API exposed via the gateway. Based on this data, Tyk adds a new middleware named "Mock Response" and returns various mock responses according to your spec. Refer to the [Mock configuration guide](#automatic-configuration-inferred-from-your-openapi-document) to learn how to do this. + +The specification provides three methods for Tyk to deduce the mock response: `example`, `examples` and `schema`. +1. `example`: A sample value that could be returned in a specific field in a response (see [below](#using-example-to-generate-a-mock-response)) +2. `examples`: A map pairing an example name with an Example Object (see [below](#using-examples-to-generate-a-mock-response)) +3. `schema`: JSON schema for the expected response body (see [below](#using-schema-to-generate-a-mock-response) + +Note: +- `example` and `examples` are mutually exclusive within the OpenAPI Document for a field in the `responses` object: the developer cannot provide both for the same object. +- The `content-type` (e.g. `application/json`, `text/plain`), per OpenAPI Specification, must be declared for each `example` or `examples` in the API description. + +Let's see how to use each method: + + +### Using `example` to generate a mock response + +In the following extract from an OpenAPI description, a single `example` has been declared for a request to `GET /get` - the API developer indicates that such a call could return `HTTP 200` and the body value `Response body example` in plain text format. + +```json {hl_lines=["9-11"],linenos=true, linenostart=1} expandable +{ + "paths": { + "/get": { + "get": { + "operationId": "getget", + "responses": { + "200": { + "content": { + "text/plain": { + "example": "Response body example" + } + }, + "description": "200 OK response for /get with a plain text" + } + } + } + } + } +} +``` + +### Using `examples` to generate a mock response + +In this extract, the API developer also indicates that a call to `GET /get` could return `HTTP 200` but here provides two example body values `Response body from first-example` and `Response body from second-example`, again in plain text format. + +``` json {hl_lines=["9-18"],linenos=true, linenostart=1} expandable +{ + "paths": { + "/get": { + "get": { + "operationId": "getget", + "responses": { + "200": { + "content": { + "text/plain": { + "examples": { + "first-example": { + "value": "Response body from first-example" + }, + "second-example": { + "value": "Response body from second-example" + } + } + } + }, + "description": "This is a mock response example with 200OK" + } + } + } + } + } +} +``` + +The `exampleNames` for these two values have been configured as `first-example` and `second-example` and can be used to [invoke the desired response](#multiple-mock-responses-for-a-single-endpoint) from a mocked endpoint. + +### Using `schema` to generate a mock response + +If there is no `example` or `examples` defined for an endpoint, Tyk will try to find a `schema` for the response. If there is a schema, it will be used to generate a mock response. Tyk can extract values from referenced or nested schema objects when creating the mock response. + +### Response headers schema +Response headers do not have standalone `example` or `examples` attributes, however, they can have a `schema` - the Mock Response middleware will include these in the mock response if provided in the OpenAPI description. + +The schema properties may have an `example` field, in which case they will be used to build a mock response. If there is no `example` value in the schema then default values are used to build a response as follows: +- `string` > `"string"` +- `integer` > `0` +- `boolean` > `true` + +For example, below is a partial OpenAPI description, that defines a schema for the `GET /get` endpoint + +```json {hl_lines=["10-13", "18-33"],linenos=true, linenostart=1} expandable +{ + "paths": { + "/get": { + "get": { + "operationId": "getget", + "responses": { + "200": { + "headers": { + "X-Status": { + "schema": { + "type": "string", + "example": "status-example" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "lastName": { + "example": "Lastname-placeholder", + "type": "string" + }, + "firstname": { + "type": "string" + }, + "id": { + "type": "integer" + } + } + } + } + }, + "description": "This is a mock response example with 200OK" + } + } + } + } + } +} +``` + +Tyk Gateway could use the above to generate the following mock response: + +```http expandable +HTTP/1.1 200 OK +X-Status: status-example +Content-Type: application/json + +{ + "lastName": "Lastname-placeholder", + "firstname": "string", + "id": 0 +} +``` +Notice that in the mock response above, `firstname` has the value `string` since there was no example for it in the OpenAP document so Tyk used the word `string` as the value for this field. + + +

Using Tyk OAS

+This tutorial is for Tyk OAS API definition users. If you're using the legacy Tyk Classic APIs, please refer to the [Tyk Classic Mock Response tutorial](#mock-response-using-classic). + +The [Mock Response](/api-management/traffic-transformation/mock-response) middleware allows you to configure Tyk to return a response for an API endpoint without requiring an upstream service. + +When working with Tyk OAS APIs, this middleware is executed at the **end** of the request processing chain immediately prior to the upstream proxy stage. Thus, any other request processing middleware - including authentication - will be run before the request reaches the mock response. + +The middleware is defined in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation). To create this definition, you can use the Tyk Dashboard API or the API Designer in the Tyk Dashboard UI. + +To configure or create a Mock Response middleware you have two modes, manual and automatic. Following please find detailed guidance for each mode. + +### Manual configuration + +You can configure a mock response directly in the API definition, in the middleware object under the Tyk extension section, `x-tyk-api-gateway`. Once added, you need to update or import it to Tyk Dashboard using Tyk Dashboard API or via Tyk Dashboard UI. This is useful when you have already tested your API in dev environments and now you need to deploy it to staging or production in a declarative manner. + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. + +The mock response middleware (`mockResponse`) can be added to the `x-tyk-api-gateway.middleware.operations` section (Tyk OAS Extension) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + +For basic operation, the `mockResponse` object has the following configuration: +- `enabled`: enable the middleware for the endpoint +- `code`: the HTTP status code to be provided with the response (this defaults to `200` if not set) +- `body`: the payload to be returned as the body of the response +- `headers`: the headers to inject with the response + +Please remember that this API definition needs to be updated in Tyk Dashboard. In the Dashboard UI it might be trivial but if you are doing it declaratively, you need to use the Tyk Dashboard API endpoint to update an existing API (PUT) or import as a new API (POST). Once updated, Tyk Gateway/s will return mock responses to all valid requests to the endpoint, depending on the [content of the request](#multiple-mock-responses-for-a-single-endpoint). + +In the following example, we configure a mock response middleware for requests to the `GET /example-mock-response1/anything` endpoint: + +```json {hl_lines=["39-49"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-mock-response1", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/anything": { + "get": { + "operationId": "anythingget", + "responses": { + "200": { + "description": "200OK for /anything using anythingget" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-mock-response1", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-mock-response1/", + "strip": true + } + }, + "middleware": { + "operations": { + "anythingget": { + "mockResponse": { + "enabled": true, + "code": 200, + "body": "This is the mock response body", + "headers": [ + { + "name": "X-Mock-Example", + "value": "mock-header-value" + } + ] + } + } + } + } + } +} +``` + +Once this API definition is updated in Tyk Dashboard, a call to `GET /example-mock-response1/anything` would return: + +```bash +HTTP/1.1 200 OK +X-Mock-Example: mock-header-value +Content-Type: text/plain; charset=utf-8 + +This is the mock response body +``` + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the mock response middleware. + +### Automatic configuration inferred from your OpenAPI Document + +Tyk will parse the [examples and schema](/api-management/traffic-transformation/mock-response#mock-responses-using-openapi-metadata) in the OpenAPI document and use them to generate mock responses automatically. + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. Endpoint `paths` entries (and the associated `operationId`) can contain wildcards in the form of any string bracketed by curly braces, for example `/status/{code}`. These wildcards are so they are human-readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +The mock response middleware (`mockResponse`) can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + +For basic operation, the `mockResponse` object has the following configuration: +- `enabled`: enable the middleware for the endpoint +- `fromOASExamples`: an object used to instruct Tyk Gateway to return a response from the OpenAPI description + +The `fromOASExamples` object has the following configuration: +- `enabled`: enable the automatic configuration of mock response +- `code`: [optional] identifies which HTTP status code to be provided with the response (defaults to `200` if not set) +- `contentType`: [optional] identifies which response body type to use (defaults to `application/json` if not set) +- `exampleName`: [optional] the sample response to be returned from an `examples` list + +The three optional fields (`code`, `contentType`, `exampleName`) are used to identify which sample response should be returned by the mock if multiple sample responses are declared in the OpenAPI description. + +In the following example, the OpenAPI description declares three possible responses: two for HTTP 200 and one for HTTP 300. We have configured the Mock Response middleware to return the value defined for HTTP 200 (code) with `exampleName` set to "second-example". The JSON below shows the updated Tyk OAS API definition, after Tyk has generated and added the mock response middleware: + +```json {hl_lines=["15-24", "29-33", "59-67"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-mock-response2", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/anything": { + "get": { + "operationId": "anythingget", + "responses": { + "200": { + "content": { + "text/plain": { + "examples": { + "first-example": { + "value": "My favorite is pasta" + }, + "second-example": { + "value": "My second favorite is pizza" + } + } + } + }, + "description": "" + }, + "300": { + "content": { + "text/plain": { + "example": "There's too much choice" + } + }, + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-mock-response2", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-mock-response2/", + "strip": true + } + }, + "middleware": { + "operations": { + "anythingget": { + "mockResponse": { + "enabled": true, + "fromOASExamples": { + "enabled": true, + "code": 200, + "contentType": "text/plain", + "exampleName": "second-example" + } + } + } + } + } + } +} +``` +Once this API definition is updated in Tyk Dashboard, a call to `GET /example-mock-response2/anything` would return: + +```bash +HTTP/1.1 200 OK +Content-Type: text/plain + +"My second favorite is pizza" +``` + +If you add `"code":300` in the `fromOASExamples` object, a call to `GET /example-mock-response2/anything` would instead respond as follows: + +```bash +HTTP/1.1 300 Multiple Choices +Content-Type: text/plain + +"There's too much choice" +``` + + + +If multiple `examples` are defined in the OpenAPI description but no default `exampleName` is set in the middleware configuration `fromOASExamples` Tyk will select randomly from the multiple `examples`. Yes, that means the response may change with every request. You can [control which response](#multiple-mock-responses-for-a-single-endpoint) will be returned using special headers in the request. + + + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the mock response middleware. + +### Multiple mock responses for a single endpoint + +When the mock response middleware in your Tyk OAS API is configured to return responses from the OpenAPI description within the API definition, you can invoke a specific response, overriding the defaults configured in the middleware, by providing specific headers in your request. + +To invoke a non-default response from a mocked endpoint, you must add *one or more* special headers to the request: +- `Accept`: This standard HTTP header will override the response content type (e.g. `application/json`, `text/plain`) +- `X-Tyk-Accept-Example-Code`: This will select the HTTP response code for which to return the example response (e.g. `400`) +- `X-Tyk-Accept-Example-Name`: This identifies which example to select from an `examples` list + +If an example response can’t be found for the configured `code`, `contentType` or `exampleName`, an HTTP 404 error will be returned to inform the client that there is no declared example for that configuration. + +In the example below, the OpenAPI document declares two possible responses: one for HTTP 200 and one for HTTP 300. We have configured the Mock Response middleware to return the value defined for HTTP 200 for which the body (content) is in JSON format and a custom header `X-Status` which will take the default value of `true`. +```json {hl_lines=["15-19", "22-39", "45-50", "53-55", "82-89"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-mock-response3", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/anything": { + "get": { + "operationId": "anythingget", + "responses": { + "200": { + "headers": { + "X-Status": { + "schema": { + "type": "boolean" + } + } + }, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "lastName": { + "example": "Bar", + "type": "string" + }, + "name": { + "example": "Foo", + "type": "string" + }, + "id": { + "type": "integer" + } + } + } + } + }, + "description": "" + }, + "300": { + "headers": { + "X-Status": { + "schema": { + "type": "boolean", + "example": false + } + } + }, + "content": { + "text/plain": { + "example": "Baz" + } + }, + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-mock-response3", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-mock-response3/", + "strip": true + } + }, + "middleware": { + "operations": { + "anythingget": { + "mockResponse": { + "enabled": true, + "fromOASExamples": { + "enabled": true, + "code": 200, + "contentType": "application/json" + } + } + } + } + } + } +} +``` + +You can trigger the mock response for HTTP 300 by adding the following headers to your request: +- `X-Tyk-Accept-Example-Code`: 300 +- `Accept`: text/plain + +This would return a plain text body and the `X-Status` header set to `false`. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the mock response middleware. + +### Configuring mock response using Tyk Dashboard UI + +Adding a mock response to your API endpoints is easy when using the API Designer in the Tyk Dashboard UI, simply follow the steps appropriate to the configuration method you wish to use: +- [manual configuration](#manual-configuration) of the middleware config +- [automatic configuration](#automatic-configuration) from the OpenAPI description + +#### Manual configuration + +1. **Add an endpoint** + + From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + + Tyk OAS API Designer showing no endpoints created + + Adding an endpoint to an API using the Tyk OAS API Designer + + Tyk OAS API Designer showing no middleware enabled on endpoint + +2. **Select the Mock Response middleware** + + Select **ADD MIDDLEWARE** and choose **Mock Response** middleware from the *Add Middleware* screen. + + Adding the Mock Response middleware + +3. **Configure the middleware** + + Select **Manually configure mock response** + + Mock Response middleware added to endpoint - select the configuration method you require + + This takes you to the middleware configuration screen where you can: + - choose the HTTP status code that you want Tyk Gateway to return + - select the content type + - add a description for your mock response + - define headers to be provided with the response + - define the body that will be returned in the response (note that this must be defined as a JSON schema) + + Configuring the mock response + + Select **UPDATE MIDDLEWARE** to apply the change to the middleware configuration. + +4. **Save the API** + + Select **SAVE API** to apply the changes to your API. + +#### Automatic configuration + +1. **Import an OpenAPI Document containing sample responses or schema** + + Import your OpenAPI Document (from file, URL or by pasting the JSON into the text editor) configure the **upstream URL** and **listen path**, and select **Auto-generate middleware to deliver mock responses**. + + Selecting this option will cause Tyk Dashboard to check for sample responses or schema in the OpenAPI description and will automatically add the Mock Response middleware for any endpoints that have suitable data. + + Configuring the OpenAPI document import to create Mock Responses + +2. **Edit the Mock Response middleware** + + Select **EDIT** and then the **Mock Response** middleware from the **Endpoints** tab. This will take you to the Edit Middleware screen. Note that *Use mock response from Open API Specification* has been selected. + + Editing the Mock Response middleware + +3. **Configure the middleware** + + Tyk Dashboard will automatically select a valid HTTP response code from the drop-down. When you select a valid `content-type` for which a mock response is configured in the OpenAPI specification, the API Designer will display the associated response. + + Mock Response middleware automatically configured from OpenAPI description + + Here you can edit the mock response: + - modify, add or delete Response Body examples (note that this must follow the selected `content-type`) + - choose a default Response Body example that will be provided (unless [overridden in the request](#multiple-mock-responses-for-a-single-endpoint)) + - add a description for your mock response + - define headers to be provided with the response (note that these must be defined as a JSON schema) + - add a schema + + You can create and edit mock responses for multiple HTTP status codes by choosing a different status code from the drop-down. + + Select **UPDATE MIDDLEWARE** to apply the change to the middleware configuration. + +4. **Save the API** + + Select **SAVE API** to apply the changes to your API. + + + + + Modifying the automatically configured Mock Response middleware will update the OpenAPI description part of your Tyk OAS API definition, as the detail of the mock response is not set in the `x-tyk-api-gateway` extension but is automatically generated in response to the particular request received to the endpoint. + + + +

Using Classic

+The [Mock Response](/api-management/traffic-transformation/mock-response) middleware allows you to configure Tyk to return a response for an API endpoint without requiring an upstream service. This can be useful when creating a new API or making a development API available to an external team. + +When working with Tyk Classic APIs, this middleware is executed at the start of the request processing chain. Thus an endpoint with the mock response middleware will not be authenticated, will not process other middleware configured for the API (neither API nor endpoint level) and will have no analytics created. It will simply return the configured response for any request made to that endpoint. + +The middleware is configured in the Tyk Classic API Definition. You can do this via the Tyk Dashboard API, the API Designer or in [Tyk Operator](#tyk-operator). + +If you're using the newer Tyk OAS APIs, then check out the [Tyk OAS](#mock-response-using-tyk-oas) page. + +### API Definition + +If you're using Tyk Operator then check out the [configuring the middleware in Tyk Operator](#tyk-operator) section below. + +To enable mock response, you must first add the endpoint to a list - one of [allow list](/api-management/traffic-transformation/allow-list), [block list](/api-management/traffic-transformation/block-list) or [ignore authentication](/api-management/traffic-transformation/ignore-authentication). This will add a new object to the `extended_paths` section of your API definition - `white_list`, `black_list` or `ignored`. The mock response can then be configured within the `method_actions` element within the new object. + +The `white_list`, `black_list` and `ignored` objects all have the same structure and configuration as follows: + +- `path`: the endpoint path +- `method`: this should be blank +- `ignore_case`: if set to `true` then the path matching will be case insensitive +- `method_actions`: the configuration of the mock response + +The `method_actions` object should be configured as follows, with an entry created for each method on the path for which you wish to configure the mock response: + +- `action`: this should be set to `reply` +- `code`: the HTTP status code to be provided with the response +- `headers`: the headers to inject with the response +- `data`: the payload to be returned as the body of the response + +For example: + +```json {linenos=true, linenostart=1} expandable +{ + "extended_paths": { + "white_list": [ + { + "disabled": false, + "path": "/anything", + "method": "", + "ignore_case": false, + "method_actions": { + "GET": { + "action": "reply", + "code": 200, + "data": "This is the mock response body", + "headers": { + "X-Example-Header": "foobar" + } + } + } + } + ] + } +} +``` + +In this example the mock response middleware has been configured for requests to the `GET /anything` endpoint. The [allow list](/api-management/traffic-transformation/allow-list) middleware has been enabled for this endpoint and is case sensitive, so calls to `GET /Anything` will not return the mock response. + +A call to `GET /anything` would return: + +``` expandable +HTTP/1.1 200 OK +X-Example-Header: foobar +Date: Wed, 31 Jan 2024 16:21:05 GMT +Content-Length: 30 +Content-Type: text/plain; charset=utf-8 + +This is the mock response body +``` + +### API Designer + +You can use the API Designer in the Tyk Dashboard to configure the Mock Response middleware for your Tyk Classic API by following these steps. + +1. **Add an endpoint for the path and configure a list plugin** + + For the mock response to be enabled, the endpoint must also be in a list. We recommend adding the path to an allow list by [selecting](/api-management/traffic-transformation/allow-list#api-definition) the **Allow List** plugin. + +2. **Add the mock response plugin** + + Now select the **Mock response** plugin. + + Selecting the mock response middleware for a Tyk Classic API + +3. **Configure the middleware** + + Once you have selected the Mock response middleware for the endpoint, you can configure the HTTP status code, headers and body to be included in the response. Remember to click **ADD**, to add each header to the response. + + Configuring the mock response middleware for a Tyk Classic API + +4. **Save the API** + + Use the *save* or *create* buttons to save the changes and activate the middleware. + + + + + For the mock response to be enabled, the endpoint must also be in a list. We recommend adding the path to an [allow list](/api-management/traffic-transformation/allow-list#allow-list-using-tyk-oas). If this isn't done, then the mock will not be saved when you save your API in the designer. + + + +### Tyk Operator + +The process of configuring a mock response is similar to that defined in the configuring the middleware in Tyk Classic API definition section. + +To configure a mock response, you must first add a mock response configuration object to the `extended_paths` section, e.g. one of allow list (`white_list`), block list (`black_list`) or ignore authentication (`ignore`). The mock response configuration object has the following properties: + +- path: the path of the endpoint, e.g `/foo`. +- ignore_case: when set to true the path matching is case insensitive. +- method_actions: a configuration object that allows the mock response to be configured for a given method, including the response body, response headers and status code. This object should also contain an `action` field with a value set to `reply`. + +In the example below we can see that a mock response is configured to ignore authentication (`ignore`) for the `GET /foo` endpoint. When a request is made to the endpoint then authentication will be ignored and a mock response is returned with status code `200` and a response body payload of `{"foo": "bar"}`. The middleware has been configured to be case sensitive, so calls to `GET /Foo` will not ignore authentication. + +```yaml {linenos=true, linenostart=1, hl_lines=["26-34"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin +spec: + name: httpbin + protocol: http + active: true + use_keyless: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + ignored: + - ignore_case: false + method_actions: + GET: + action: "reply" + code: 200 + data: "{\"foo\": \"bar\"}" + headers: {} + path: /foo +``` + + diff --git a/api-management/traffic-transformation/request-body.mdx b/api-management/traffic-transformation/request-body.mdx new file mode 100644 index 00000000..d5a4f17d --- /dev/null +++ b/api-management/traffic-transformation/request-body.mdx @@ -0,0 +1,394 @@ +--- +title: "Request Body" +'og:description': "How to configure Request Body traffic transformation middleware in Tyk" +tags: ['Traffic Transformation', 'Request Body'] +sidebarTitle: "Request Body " +--- + +

Overview

+Tyk enables you to modify the payload of API requests before they are proxied to the upstream. This makes it easy to transform between payload data formats or to expose legacy APIs using newer schema models without having to change any client implementations. This middleware is only applicable to HTTP methods that can support a request body (i.e. PUT, POST or PATCH). + +With the body transform middleware you can modify XML or JSON formatted payloads to ensure that the response contains the information required by your upstream service. You can enrich the request by adding contextual data that is held by Tyk but not included in the original request from the client. + +This middleware changes only the payload and not the headers. You can, however, combine this with the [Request Header Transform](/api-management/traffic-transformation/request-headers) middleware to apply more complex transformation to requests. + +There is a closely related [Response Body Transform](/api-management/traffic-transformation/response-body) middleware that provides the same functionality on the response from the upstream, prior to it being returned to the client. + +### Use Cases + +#### Maintaining compatibility with legacy clients + +Sometimes you might have a legacy API and need to migrate the transactions to a new upstream service but do not want to upgrade all the existing clients to the newer upstream API. Using request body transformation, you can convert the incoming legacy XML or JSON request structure into a newer, cleaner JSON format that your upstream services expect. + +#### Shaping requests received from different devices + +You can detect device types via headers or context variables and transform the request payload to optimize it for that particular device. For example, you might send extra metadata to the upstream for mobile apps. + +#### SOAP to REST translation + +A common use of the request body transform middleware is to surface a legacy SOAP service with a REST API. Full details of how to perform this conversion using Tyk are provided [here](/advanced-configuration/transform-traffic/soap-rest). + +### Working + +Tyk's body transform middleware uses the [Go template language](https://golang.org/pkg/text/template/) to parse and modify the provided input. We have bundled the [Sprig Library (v3)](http://masterminds.github.io/sprig/) which provides over 70 pre-written functions for transformations to assist the creation of powerful Go templates to transform your API requests. + +The Go template can be defined within the API Definition or can be read from a file that is accessible to Tyk, for example alongside your [error templates](/api-management/gateway-events#error-templates). + +We have provided more detail, links to reference material and some examples of the use of Go templating [here](/api-management/traffic-transformation/go-templates). + + + +Tyk evaluates templates stored in files on startup, so if you make changes to a template you must remember to restart the gateway. + + + +#### Supported request body formats + +The body transformation middleware can modify request payloads in the following formats: +- JSON +- XML + +When working with JSON format data, the middleware will unmarshal the data into a data structure, and then make that data available to the template in dot-notation. + +#### Data accessible to the middleware + +The middleware has direct access to the request body and also to dynamic data as follows: + - [context variables](/api-management/traffic-transformation/request-context-variables), extracted from the request at the start of the middleware chain, can be injected into the template using the `._tyk_context.KEYNAME` namespace + - [session metadata](/api-management/policies#what-is-a-session-metadata), from the Tyk Session Object linked to the request, can be injected into the template using the `._tyk_meta.KEYNAME` namespace + - inbound form or query data can be accessed through the `._tyk_context.request_data` namespace where it will be available in as a `key:[]value` map + - values from [key-value (KV) storage](/tyk-configuration-reference/kv-store#transformation-middleware) can be injected into the template using the notation appropriate to the location of the KV store + +The request body transform middleware can iterate through list indices in dynamic data so, for example, calling `{{ index ._tyk_context.request_data.variablename 0 }}` in a template will expose the first entry in the `request_data.variablename` key/value array. + + + +As explained in the [documentation](https://pkg.go.dev/text/template), templates are executed by applying them to a data structure. The template receives the decoded JSON or XML of the request body. If session variables or meta data are enabled, additional fields will be provided: `_tyk_context` and `_tyk_meta` respectively. + + + +#### Automatic XML <-> JSON Transformation + +A very common transformation that is applied in the API Gateway is to convert between XML and JSON formatted body content. + +The Request Body Transform supports two helper functions that you can use in your Go templates to facilitate this: + - `jsonMarshal` performs JSON style character escaping on an XML field and, for complex objects, serialises them to a JSON string ([example](/api-management/traffic-transformation/go-templates#xml-to-json-conversion-using-jsonmarshal)) + - `xmlMarshal` performs the equivalent conversion from JSON to XML ([example](/api-management/traffic-transformation/go-templates#json-to-xml-conversion-using-xmlmarshal)) + +
+ +If you're using Tyk OAS APIs, then you can find details and examples of how to configure the request body transformation middleware [here](#request-body-using-tyk-oas). + +If you're using Tyk Classic APIs, then you can find details and examples of how to configure the request body transformation middleware [here](#request-body-using-classic). + +{/* proposed "summary box" to be shown graphically on each middleware page + # Request Body Transform middleware summary + - The Request Body Transform middleware is an optional stage in Tyk's API Request processing chain, sitting between the [TBC]() and [TBC]() middleware. + - The Request Body Transform middleware can be configured at the per-endpoint level within the API Definition and is supported by the API Designer within the Tyk Dashboard. + - Request Body Transform can access both [session metadata](/api-management/policies#what-is-a-session-metadata) and [request context variables](/api-management/traffic-transformation/request-context-variables). */} + +

Using Tyk OAS

+The [request body transform](/api-management/traffic-transformation/request-body) middleware provides a way to modify the payload of API requests before they are proxied to the upstream. + +The middleware is configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation). You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the legacy Tyk Classic APIs, then check out the [Tyk Classic](#request-body-using-classic) page. + +### API Definition + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. Endpoint `paths` entries (and the associated `operationId`) can contain wildcards in the form of any string bracketed by curly braces, for example `/status/{code}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +The request body transformation middleware (`transformRequestBody`) can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + +The `transformRequestBody` object has the following configuration: +- `enabled`: enable the middleware for the endpoint +- `format`: the format of input data the parser should expect (either `xml` or `json`) +- `body`: [see note] this is a `base64` encoded representation of your template +- `path`: [see note] this is the path to the text file containing the template + + + + + You should configure only one of `body` or `path` to indicate whether you are embedding the template within the middleware or storing it in a text file. The middleware will automatically select the correct source based on which of these fields you complete. If both are provided, then `body` will take precedence and `path` will be ignored. + + + +For example: +```json {hl_lines=["39-43"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-request-body-transform", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/anything": { + "put": { + "operationId": "anythingput", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-request-body-transform", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-request-body-transform/", + "strip": true + } + }, + "middleware": { + "operations": { + "anythingput": { + "transformRequestBody": { + "enabled": true, + "format": "json", + "body": "ewogICJ2YWx1ZTEiOiAie3sudmFsdWUyfX0iLAogICJ2YWx1ZTIiOiAie3sudmFsdWUxfX0iLAogICJyZXEtaGVhZGVyIjogInt7Ll90eWtfY29udGV4dC5oZWFkZXJzX1hfSGVhZGVyfX0iLAogICJyZXEtcGFyYW0iOiAie3suX3R5a19jb250ZXh0LnJlcXVlc3RfZGF0YS5wYXJhbX19Igp9" + } + } + } + } + } +} +``` + +In this example the request body transform middleware has been configured for requests to the `PUT /anything` endpoint. The `body` contains a base64 encoded Go template (which you can check by pasting the value into a service such as [base64decode.org](https://www.base64decode.org)). + +Decoded, this template is: +```json expandable +{ + "value1": "{{.value2}}", + "value2": "{{.value1}}", + "req-header": "{{._tyk_context.headers_X_Header}}", + "req-param": "{{._tyk_context.request_data.param}}" +} +``` + +So if you make a request to `PUT /anything?param=foo` as follows: +```bash expandable +PUT /anything?param=foo +HTTP/1.1 +Host: my-gateway.host +X-Header: bar + +{ + "value1": "world", + "value2": "hello" +} +``` + +You will receive a response from the upstream with this payload: +```json expandable +{ + "req-header": "bar", + "req-param": "[foo]", + "value1": "hello", + "value2": "world" +} +``` + +The `/anything` endpoint returns the details of the request that was received by httpbin.org. You can see that Tyk has swapped `value1` and `value2` and embedded the `X-Header` header and `param` query values into the body of the request. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the mock response middleware. + + + +If using a template in a file (i.e. you configure `path` in the `transformRequestBody` object), remember that Tyk will load and evaluate the template when the Gateway starts up. If you modify the template, you will need to restart Tyk in order for the changes to take effect. + + + +### API Designer + +Adding Request Body Transformation to your API endpoints is easy when using the API Designer in the Tyk Dashboard, simply follow the following steps: + +1. **Add an endpoint** + + From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + + Tyk OAS API Designer showing no endpoints created + + Adding an endpoint to an API using the Tyk OAS API Designer + + Tyk OAS API Designer showing no middleware enabled on endpoint + +2. **Select the Request Body Transform middleware** + + Select **ADD MIDDLEWARE** and choose the **Request Body Transform** middleware from the *Add Middleware* screen. + + Adding the Request Body Transform middleware + +3. **Configure the middleware** + + Now you can select the request body format (JSON or XML) and add either a path to the file containing the template, or directly enter the transformation template in the text box. + + Configuring the Request Body Transform middleware + + The **Test with data** control will allow you to test your body transformation function by providing an example request body and generating the output from the transform. It is not possible to configure headers, other request parameters, context or session metadata to this template test so if you are using these data sources in your transform it will not provide a complete output, for example: + + Testing the Request Body Transform + +4. **Save the API** + + Select **SAVE API** to apply the changes to your API. + +

Using Classic

+The [request body transform](/api-management/traffic-transformation/request-body) middleware provides a way to modify the payload of API requests before they are proxied to the upstream. + +This middleware is configured in the Tyk Classic API Definition at the endpoint level. You can do this via the Tyk Dashboard API or in the API Designer. + +If you want to use dynamic data from context variables, you must [enable](/api-management/traffic-transformation/request-context-variables#enabling-context-variables-for-use-with-tyk-classic-apis) context variables for the API to be able to access them from the request header transform middleware. + +If you're using the newer Tyk OAS APIs, then check out the [Tyk OAS](#request-body-using-tyk-oas) page. + +If you're using Tyk Operator then check out the [Configuring the middleware in Tyk Operator](#tyk-operator) section below. + +### API Definition + +To enable the middleware you must add a new `transform` object to the `extended_paths` section of your API definition. + +The `transform` object has the following configuration: +- `path`: the path to match on +- `method`: this method to match on +- `template_data`: details of the Go template to be applied for the transformation of the request body + +The Go template is described in the `template_data` object by the following fields: +- `input_type`: the format of input data the parser should expect (either `xml` or `json`) +- `enable_session`: set this to `true` to make session metadata available to the transform template +- `template_mode`: instructs the middleware to look for the template either in a `file` or in a base64 encoded `blob`; the actual file location (or base64 encoded template) is provided in `template_source` +- `template_source`: if `template_mode` is set to `file`, this will be the path to the text file containing the template; if `template_mode` is set to `blob`, this will be a `base64` encoded representation of your template + +For example: +```json {linenos=true, linenostart=1} expandable +{ + "extended_paths": { + "transform": [ + { + "path": "/anything", + "method": "POST", + "template_data": { + "template_mode": "file", + "template_source": "./templates/transform_test.tmpl", + "input_type": "json", + "enable_session": true + } + } + ] + } +} +``` + +In this example, the Request Body Transform middleware is directed to use the template located in the `file` at location `./templates/transform_test.tmpl`. The input (pre-transformation) request payload will be `json` format and session metadata will be available for use in the transformation. + + + +Tyk will load and evaluate the template file when the Gateway starts up. If you modify the template, you will need to restart Tyk in order for the changes to take effect. + + + +### API Designer + +You can use the API Designer in the Tyk Dashboard to configure the request body transform middleware for your Tyk Classic API by following these steps. + +1. **Add an endpoint for the path and select the plugin** + +From the **Endpoint Designer** add an endpoint that matches the path for which you want to perform the transformation. Select the **Body Transforms** plugin. + +Endpoint designer + +2. **Configure the middleware** + +Ensure that you have selected the `REQUEST` tab, then select your input type, and then add the template you would like to use to the **Template** input box. + +Setting the body request transform + +3. **Test the Transform** + +If sample input data is available, you can use the Input box to add it, and then test it using the **Test** button. You will see the effect of the template on the sample input displayed in the Output box. + +Testing the body transform function + +4. **Save the API** + +Use the *save* or *create* buttons to save the changes and activate the Request Body Transform middleware. + +### Tyk Operator + +The process for configuring a request body transform is similar to that defined in section configuring the middleware in the Tyk Classic API Definition. Tyk Operator allows you to configure a request body transform by adding a `transform` object to the `extended_paths` section of your API definition. + +In the example below the Request Body middleware (`transform`) has been configured for `HTTP POST` requests to the `/anything` endpoint. The Request Body Transform middleware is directed to use the template located in the blob included in the `template_source` field. The input (pre-transformation) request payload will be json format and session metadata will be available for use in the transformation. + +```yaml {linenos=true, linenostart=1, hl_lines=["32-40"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-transform +spec: + name: httpbin-transform + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin-transform + strip_listen_path: true + response_processors: + - name: response_body_transform + - name: header_injector + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + transform: + - method: POST + path: /anything + template_data: + enable_session: false + input_type: json + template_mode: blob + # base64 encoded template + template_source: eyJiYXIiOiAie3suZm9vfX0ifQ== + transform_headers: + - delete_headers: + - "remove_this" + add_headers: + foo: bar + path: /anything + method: POST + transform_response: + - method: GET + path: /xml + template_data: + enable_session: false + input_type: xml + template_mode: blob + # base64 encoded template + template_source: e3sgLiB8IGpzb25NYXJzaGFsIH19 + transform_response_headers: + - method: GET + path: /xml + add_headers: + Content-Type: "application/json" + act_on: false + delete_headers: [] +``` + diff --git a/api-management/traffic-transformation/request-context-variables.mdx b/api-management/traffic-transformation/request-context-variables.mdx new file mode 100644 index 00000000..b27ee458 --- /dev/null +++ b/api-management/traffic-transformation/request-context-variables.mdx @@ -0,0 +1,110 @@ +--- +title: "Request Context Variables" +'og:description': "How to configure Request Context Variables traffic transformation middleware in Tyk" +tags: ['Traffic Transformation', 'Request Context Variables'] +sidebarTitle: "Request Context Variables" +--- + +Context variables are extracted from the request at the start of the middleware chain. These values can be very useful for later transformation of request data, for example, in converting a form POST request into a JSON PUT request or to capture an IP address as a header. + + + +When using Tyk Classic APIs, you must [enable](#enabling-context-variables-for-use-with-tyk-classic-apis) context variables for the API to be able to access them. When using Tyk OAS APIs, the context variables are always available to the context-aware middleware. + + + + +## Available context variables +* `request_data`: If the inbound request contained any query data or form data, it will be available in this object. For the header injector Tyk will format this data as `key:value1,value2,valueN;key:value1,value2` etc. +* `path_parts`: The components of the path, split on `/`. These values should be in the format of a comma delimited list. +* `token`: The inbound raw token (if bearer tokens are being used) of this user. +* `path`: The path that is being requested. +* `remote_addr`: The IP address of the connecting client. +* `request_id` Allows the injection of request correlation ID (for example X-Request-ID) +* `jwt_claims_CLAIMNAME` - If JWT tokens are being used, then each claim in the JWT is available in this format to the context processor. `CLAIMNAME` is case sensitive so use the exact claim. +* `cookies_COOKIENAME` - If there are cookies, then each cookie is available in context processor in this format. `COOKIENAME` is case sensitive so use the exact cookie name and replace any `-` in the cookie name with `_`. +* `headers_HEADERNAME` - Headers are obviously exposed in context processor. You can access any header in the request using the following format: Convert the **first letter** in each word of an incoming header is to Capital Case. This is due to the way GoLang handles header parsing. You also need to replace any `-` in the `HEADERNAME` name with `_`.
+For example, to get the value stored in `test-header`, the syntax would be `$tyk_context.headers_Test_Header`. + + +## Middleware that can use context variables: +Context variables are exposed in three middleware plugins but are accessed differently depending on the caller as follows: + +1. URL Rewriter - Syntax is `$tyk_context.CONTEXTVARIABLES`. See [URL Rewriting](/transform-traffic/url-rewriting#url-rewrite-middleware) for more details. +2. Modify Headers - Syntax is `$tyk_context.CONTEXTVARIABLES`. See [Request Headers](/api-management/traffic-transformation/request-headers) for more details. +3. Body Transforms - Syntax is `{{ ._tyk_context.CONTEXTVARIABLES }}`. See [Body Transforms](/api-management/traffic-transformation/request-body) for more details. + + + + + The Body Transform can fully iterate through list indices within context data so, for example, calling `{{ index ._tyk_context.path_parts 0 }}` in the Go Template in a Body Transform will expose the first entry in the `path_parts` list. + + URL Rewriter and Header Transform middleware cannot iterate through list indices. + + + + +## Example use of context variables + +### Examples of the syntax to use with all the available context varibles: +``` expandable +"x-remote-addr": "$tyk_context.remote_addr", +"x-token": "$tyk_context.token", +"x-jwt-sub": "$tyk_context.jwt_claims_sub", +"x-part-path": "$tyk_context.path_parts", +"x-jwt-pol": "$tyk_context.jwt_claims_pol", +"x-cookie": "$tyk_context.cookies_Cookie_Context_Var", +"x-cookie-sensitive": "$tyk_context.cookies_Cookie_Case_sensitive", +"x-my-header": "$tyk_context.headers_My_Header", +"x-path": "$tyk_context.path", +"x-request-data": "$tyk_context.request_data", +"x-req-id": "$tyk_context.request_id" +``` +Example of the syntax in the UI + +### The context variable values in the response: +``` expandable +"My-Header": "this-is-my-header", +"User-Agent": "PostmanRuntime/7.4.0", +"X-Cookie": "this-is-my-cookie", +"X-Cookie-Sensitive": "case-sensitive", +"X-Jwt-Pol": "5bca6a739afe6a00017eb267", +"X-Jwt-Sub": "john.doe@test.com", +"X-My-Header": "this-is-my-header", +"X-Part-Path": "context-var-example,anything", +"X-Path": "/context-var-example/anything", +"X-Remote-Addr": "127.0.0.1", +"X-Req-Id": "e3e99350-b87a-4d7d-a75f-58c1f89b2bf3", +"X-Request-Data": "key1:val1;key2:val2", +"X-Token": "5bb2c2abfb6add0001d65f699dd51f52658ce2d3944d3d6cb69f07a2" +``` + +## Enabling Context Variables for use with Tyk Classic APIs +1. In the your Tyk Dashboard, select `APIs` from the `System Management` menu +2. Open the API you want to add Context Variable to +3. Click the `Advanced Options` tab and then select the `Enable context variables` option + +Context Variables + +If not using a Tyk Dashboard, add the field `enable_context_vars` to your API definition file at root level and set it to `true`. + +If you are using Tyk Operator, set the field `spec.enable_context_vars` to `true`. + +The example API Definition below enabled context variable: + +```yaml {linenos=true, linenostart=1, hl_lines=["10-10"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin +spec: + name: httpbin + use_keyless: true + protocol: http + active: true + enable_context_vars: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true +``` diff --git a/api-management/traffic-transformation/request-headers.mdx b/api-management/traffic-transformation/request-headers.mdx new file mode 100644 index 00000000..36364d50 --- /dev/null +++ b/api-management/traffic-transformation/request-headers.mdx @@ -0,0 +1,545 @@ +--- +title: "Request Headers" +'og:description': "How to configure Request Headers traffic transformation middleware in Tyk" +tags: ['Traffic Transformation', 'Request Headers'] +sidebarTitle: "Request Headers " +--- + +

Overview

+Tyk allows you to modify the headers of incoming requests to your API endpoints before they are passed to your upstream service. + +There are two options for this: +- API-level modification that is applied to all requests to the API +- endpoint-level modification that is applied only to requests to a specific endpoint + +With the header transform middleware you can append or delete any number of headers to ensure that the request contains the information required by your upstream service. You can enrich the request by adding contextual data that is held by Tyk but not included in the original request from the client. + +This middleware changes only the headers and not the method or payload. You can, however, combine this with the [Request Method Transform](/api-management/traffic-transformation/request-method) and [Request Body Tranform](/api-management/traffic-transformation/request-body) to apply more complex transformation to requests. + +There are related [Response Header Transform](/api-management/traffic-transformation/response-headers) middleware (at API-level and endpoint-level) that provide the same functionality on the response from your upstream, prior to it being returned to the client. + +### Use Cases + +#### Adding Custom Headers + +A common use of this feature is to add custom headers to requests, such as adding a secure header to all upstream requests (to verify that traffic is coming from the gateway), or adding a timestamp for tracking purposes. + +#### Modifying Headers for Compatibility + +You could use the request header transform middleware to modify headers for compatibility with a downstream system, such as changing the Content-Type header from "application/json" to "application/xml" for an API that only accepts XML requests while using the [Request Body Tranform](/api-management/traffic-transformation/request-body) to transform the payload. + +#### Prefixing or Suffixing Headers + +Upstream systems or corporate policies might mandate that a prefix or suffix is added to header names, such as adding a "Bearer" prefix to all Authorization headers for easier identification internally, without modifying the externally published API consumed by the client applications. + +#### Adding multi-user access to a service + +You can add multi-user access to an upstream API that has a single authentication key and you want to add multi-user access to it without modifying it or adding clunky authentication methods to it to support new users. + +### Working + +The request header transform can be applied per-API or per-endpoint; each has a separate entry in the API definition so that you can configure both API-level and endpoint-level transforms for a single API. + +The middleware is configured with a list of headers to delete from the request and a list of headers to add to the request. Each header to be added to the request is configured as a key:value pair. + +The "delete header" functionality is intended to ensure that any header in the delete list is not present once the middleware completes - so if a header is not originally present in the request but is on the list to be deleted, the middleware will ignore its omission. + +The "add header" functionality will capitalize any header name provided, for example if you configure the middleware to append `x-request-id` it will be added to the request as `X-Request-Id`. + +In the request middleware chain, the API-level transform is applied before the endpoint-level transform so if both middleware are enabled, the endpoint-level transform will operate on the headers that have been added by the API-level transform (and will not receive those that have been deleted by it). + +#### Injecting dynamic data into headers + +You can enrich the request headers by injecting data from context variables or session objects into the headers. +- [context variables](/api-management/traffic-transformation/request-context-variables) are extracted from the request at the start of the middleware chain and can be injected into added headers using the `$tyk_context.` namespace +- [session metadata](/api-management/policies#what-is-a-session-metadata), from the Tyk Session Object linked to the request, can be injected into added headers using the `$tyk_meta.` namespace +- values from [key-value (KV) storage](/tyk-configuration-reference/kv-store#transformation-middleware) can be injected into added headers using the notation appropriate to the location of the KV store + +
+ +If you're using Tyk OAS APIs, then you can find details and examples of how to configure the request header transform middleware [here](#request-headers-using-tyk-oas). + +If you're using Tyk Classic APIs, then you can find details and examples of how to configure the request header transform middleware [here](#request-headers-using-classic). + +{/* proposed "summary box" to be shown graphically on each middleware page + # Request Header Transform middleware summary + - The Request Header Transform is an optional stage in Tyk's API Request processing chain, sitting between the [TBC]() and [TBC]() middleware. + - The Request Header Transform can be configured at the per-endpoint or per-API level within the API Definition and is supported by the API Designer within the Tyk Dashboard. */} + +

Using Tyk OAS

+Tyk's [request header transform](/api-management/traffic-transformation/request-headers) middleware enables you to append or delete headers on requests to your API endpoints before they are passed to your upstream service. + +There are two options for this: +- API-level modification that is applied to all requests to the API +- endpoint-level modification that is applied only to requests to a specific endpoint + + + + + If both API-level and endpoint-level middleware are configured, the API-level transformation will be applied first. + + + +When working with Tyk OAS APIs the transformation is configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation). You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the legacy Tyk Classic APIs, then check out the [Tyk Classic](#request-headers-using-classic) page. + +### API Definition + +The API-level and endpoint-level request header transforms are configured in different sections of the API definition, though have a common configuration. + +### API-level transform + +To append headers to, or delete headers from, all requests to your API (i.e. for all endpoints) you must add a new `transformRequestHeaders` object to the `middleware.global` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition. + +You only need to enable the middleware (set `enabled:true`) and then configure the details of headers to `add` and those to `remove`. + +For example: +```json {hl_lines=["38-56"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-request-header", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/status/200": { + "get": { + "operationId": "status/200get", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-request-header", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-request-header/", + "strip": true + } + }, + "middleware": { + "global": { + "transformRequestHeaders": { + "enabled": true, + "remove": [ + "Auth_Id" + ], + "add": [ + { + "name": "X-Static", + "value": "foobar" + }, + { + "name": "X-Request-ID", + "value": "$tyk_context.request_id" + }, + { + "name": "X-User-ID", + "value": "$tyk_meta.uid" + } + ] + } + } + } + } +} +``` + +This configuration will add three new headers to each request: +- `X-Static` with the value `foobar` +- `X-Request-ID` with a dynamic value taken from the `request_id` [context variables](/api-management/traffic-transformation/request-context-variables) +- `X-User-ID` with a dynamic value taken from the `uid` field in the [session metadata](/api-management/policies#what-is-a-session-metadata) + +It will also delete one header (if present) from each request: +- `Auth_Id` + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the API-level request header transform. + +### Endpoint-level transform + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. Endpoint `paths` entries (and the associated `operationId`) can contain wildcards in the form of any string bracketed by curly braces, for example `/status/{code}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +The request header transform middleware (`transformRequestHeaders`) can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + +The `transformRequestHeaders` object has the following configuration: +- `enabled`: enable the middleware for the endpoint +- `add`: a list of headers, in key:value pairs, to be appended to the request +- `remove`: a list of headers to be deleted from the request (if present) + +For example: +```json {hl_lines=["39-50"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-request-header", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/status/200": { + "get": { + "operationId": "status/200get", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-request-header", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-request-header/", + "strip": true + } + }, + "middleware": { + "operations": { + "status/200get": { + "transformRequestHeaders": { + "enabled": true, + "remove": [ + "X-Static" + ], + "add": [ + { + "name": "X-Secret", + "value": "the-secret-key-is-secret" + } + ] + } + } + } + } + } +} +``` + +In this example the Request Header Transform middleware has been configured for requests to the `GET /status/200` endpoint. Any request received to that endpoint will have the `X-Static` header removed and the `X-Secret` header added, with the value set to `the-secret-key-is-secret`. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the endpoint-level request header transform. + +### Combining API-level and Endpoint-level transforms + +If the API-level transform in the previous [example](/api-management/traffic-transformation/request-headers#api-level-transform) is applied to the same API, then because the API-level transformation is performed first, the `X-Static` header will be added (by the API-level transform) and then removed (by the endpoint-level transform) such that the overall effect of the two transforms for a call to `GET /status/200` would be to add three headers: + - `X-Request-ID` + - `X-User-ID` + - `X-Secret` + +and to remove one: + - `Auth_Id` + +### API Designer + +Adding and configuring the transforms to your API endpoints is easy when using the API Designer in the Tyk Dashboard, simply follow these steps: + +### Adding an API-level transform + +From the **API Designer** on the **Settings** tab, after ensuring that you are in *edit* mode, toggle the switch to **Enable Transform request headers** in the **Middleware** section: +Tyk OAS API Designer showing API-level Request Header Transform + +Then select **NEW HEADER** as appropriate to add or remove a header from API requests. You can add or remove multiple headers by selecting **ADD HEADER** to add another to the list: +Configuring the API-level Request Header Transform in Tyk OAS API Designer + +### Adding an endpoint level transform + +1. **Add an endpoint** + + From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + + Tyk OAS API Designer showing no endpoints created + + Adding an endpoint to an API using the Tyk OAS API Designer + + Tyk OAS API Designer showing no middleware enabled on endpoint + +2. **Select the Request Header Transform middleware** + + Select **ADD MIDDLEWARE** and choose the **Request Header Transform** middleware from the *Add Middleware* screen. + + Adding the Request Header Transform middleware + +3. **Configure header transformation** + + Select **NEW HEADER** to configure a header to be added to or removed from the request. + + Configuring the Request Header transformation + + You can add multiple headers to either list by selecting **NEW HEADER** again. + + Adding another header to the transformation + +4. **Save the API** + + Select **ADD MIDDLEWARE** to save the middleware configuration. Remember to select **SAVE API** to apply the changes. + +

Using Classic

+Tyk's [request header transform](/api-management/traffic-transformation/request-headers) middleware enables you to append or delete headers on requests to your API endpoints before they are passed to your upstream service. + +There are two options for this: +- API-level modification that is applied to all requests to the API +- endpoint-level modification that is applied only to requests to a specific endpoint + + + + + If both API-level and endpoint-level middleware are configured, the API-level transformation will be applied first. + + + +When working with Tyk Classic APIs the transformation is configured in the Tyk Classic API Definition. You can do this via the Tyk Dashboard API or in the API Designer. + +If you want to use dynamic data from context variables, you must [enable](/api-management/traffic-transformation/request-context-variables#enabling-context-variables-for-use-with-tyk-classic-apis) context variables for the API to be able to access them from the request header transform middleware. + +If you're using the newer Tyk OAS APIs, then check out the [Tyk OAS](#request-headers-using-tyk-oas) page. + +If you're using Tyk Operator then check out the [configuring the Request Header Transform in Tyk Operator](#tyk-operator) section below. + +### API Definition + +The API-level and endpoint-level request header transforms have a common configuration but are configured in different sections of the API definition. + +

API-level transform

+To **append** headers to all requests to your API (i.e. for all endpoints) you must add a new `global_headers` object to the `versions` section of your API definition. This contains a list of key:value pairs, being the names and values of the headers to be added to requests. + +To **delete** headers from all requests to your API, you must add a new `global_headers_remove` object to the `versions` section of the API definition. This contains a list of the names of existing headers to be removed from requests. + +For example: +```json {hl_lines=["39-45"],linenos=true, linenostart=1} expandable +{ + "version_data": { + "versions": { + "Default": { + "global_headers": { + "X-Static": "foobar", + "X-Request-ID":"$tyk_context.request_id", + "X-User-ID": "$tyk_meta.uid" + }, + "global_headers_remove": [ + "Auth_Id" + ] + } + } + }, +} +``` + +This configuration will add three new headers to each request: +- `X-Static` with the value `foobar` +- `X-Request-ID` with a dynamic value taken from the `request_id` [context variables](/api-management/traffic-transformation/request-context-variables) +- `X-User-ID` with a dynamic value taken from the `uid` field in the [session metadata](/api-management/policies#what-is-a-session-metadata) + +It will also delete one header (if present) from each request: +- `Auth_Id` + +

Endpoint-level transform

+To configure a transformation of the request header for a specific endpoint you must add a new `transform_headers` object to the `extended_paths` section of your API definition. + +It has the following configuration: +- `path`: the endpoint path +- `method`: the endpoint HTTP method +- `delete_headers`: A list of the headers that should be deleted from the request +- `add_headers`: A list of headers, in key:value pairs, that should be added to the request + +The `path` can contain wildcards in the form of any string bracketed by curly braces, for example `{user_id}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +For example: +```json expandable +{ + "transform_headers": [ + { + "path": "status/200", + "method": "GET", + "delete_headers": ["X-Static"], + "add_headers": {"X-Secret": "the-secret-key-is-secret"} + } + ] +} +``` + +In this example the Request Header Transform middleware has been configured for HTTP `GET` requests to the `/status/200` endpoint. Any request received to that endpoint will have the `X-Static` header removed and the `X-Secret` header added, with the value set to `the-secret-key-is-secret`. + +#### Combining API-level and Endpoint-level transforms + +If the API-level transform in the previous [example](/api-management/traffic-transformation/request-headers#api-level-transform) is applied to the same API, then because the API-level transformation is performed first, the `X-Static` header will be added (by the API-level transform) and then removed (by the endpoint-level transform) such that the overall effect of the two transforms for a call to `GET /status/200` would be to add three headers: +- `X-Request-ID` +- `X-User-ID` +- `X-Secret` + +and to remove one: +- `Auth_Id` + +### API Designer + +You can use the API Designer in the Tyk Dashboard to configure the request header transform middleware for your Tyk Classic API by following these steps. + +#### API-level transform + +Configuring the API-level request header transform middleware is very simple when using the Tyk Dashboard. + +In the Endpoint Designer you should select the **Global Version Settings** and ensure that you have selected the **Request Headers** tab: + +Global version settings + +Note that you must click **ADD** to add a header to the list (for appending or deletion). + +#### Endpoint-level transform + +1. **Add an endpoint for the path and select the Header Transform plugin** + + From the **Endpoint Designer** add an endpoint that matches the path for which you want to perform the transformation. Select the **Modify Headers** plugin. + + Endpoint designer + +2. **Select the "Request" tab** + + This ensures that this will only be applied to inbound requests. + + Request tab + +3. **Declare the headers to be modified** + + Select the headers to delete and insert using the provided fields. You need to click **ADD** to ensure they are added to the list. + + Header transforms + +4. **Save the API** + + Use the *save* or *create* buttons to save the changes and activate the middleware. + +### Tyk Operator + +The process for configuring a request header transform is similar to that defined in section Configuring the Request Header Transform in the Tyk Classic API Definition. Tyk Operator allows you to configure a request size limit for [all endpoints of an API](#tyk-operator-api) or for a [specific API endpoint](#tyk-operator-endpoint). + +

API-level transform

+Request headers can be removed and inserted using the following fields within an `ApiDefinition`: + +- `global_headers`: Mapping of key values corresponding to headers to add to API requests. +- `global_headers_remove`: List containing the name of headers to remove from API requests. + +The example below shows an `ApiDefinition` custom resource that adds *foo-req* and *bar-req* headers to the request before it is sent upstream. The *foo-req* header has a value of *foo-val* and the *bar-req* header has a value of *bar-val*. Furthermore, the *hello* header is removed from the request before it is sent upstream. + +```yaml {linenos=true, linenostart=1, hl_lines=["25-29"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-global-headers +spec: + name: httpbin-global-headers + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin-global-headers + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + global_headers: + foo-req: my-foo + bar-req: my-bar + global_headers_remove: + - hello +``` + +

Endpoint-level transform

+The process of configuring a transformation of a request header for a specific endpoint is similar to that defined in section [Endpoint-level transform](#tyk-classic-endpoint). To configure a transformation of the request header for a specific endpoint you must add a new `transform_headers` object to the `extended_paths` section of your API definition. + +In the example below the Request Header Transform middleware (`transform_headers`) has been configured for HTTP `POST` requests to the `/anything` endpoint. Any request received to that endpoint will have the `remove_this` header removed and the `foo` header added, with the value set to `bar`. + +```yaml {linenos=true, linenostart=1, hl_lines=["41-47"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-transform +spec: + name: httpbin-transform + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin-transform + strip_listen_path: true + response_processors: + - name: response_body_transform + - name: header_injector + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + transform: + - method: POST + path: /anything + template_data: + enable_session: false + input_type: json + template_mode: blob + # base64 encoded template + template_source: eyJiYXIiOiAie3suZm9vfX0ifQ== + transform_headers: + - delete_headers: + - "remove_this" + add_headers: + foo: bar + path: /anything + method: POST + transform_response: + - method: GET + path: /xml + template_data: + enable_session: false + input_type: xml + template_mode: blob + # base64 encoded template + template_source: e3sgLiB8IGpzb25NYXJzaGFsIH19 + transform_response_headers: + - method: GET + path: /xml + add_headers: + Content-Type: "application/json" + act_on: false + delete_headers: [] +``` + diff --git a/api-management/traffic-transformation/request-method.mdx b/api-management/traffic-transformation/request-method.mdx new file mode 100644 index 00000000..8454b009 --- /dev/null +++ b/api-management/traffic-transformation/request-method.mdx @@ -0,0 +1,242 @@ +--- +title: "Request Method" +'og:description': "How to configure Request Method traffic transformation middleware in Tyk" +tags: ['Traffic Transformation', 'Request Method'] +sidebarTitle: "Request Method " +--- + +

Overview

+Tyk's Request Method Transform middleware allows you to modify the HTTP method of incoming requests to an API endpoint prior to the request being proxied to the upstream service. You might use this to map `POST` requests from clients to upstream services that support only `PUT` and `DELETE` operations, providing a modern interface to your users. It is a simple middleware that changes only the method and not the payload or headers. You can, however, combine this with the [Request Header Transform](/api-management/traffic-transformation/request-headers) and [Request Body Tranform](/api-management/traffic-transformation/request-body) to apply more complex transformation to requests. + +### Use Cases + +#### Simplifying API consumption + +In cases where an upstream API requires different methods (e.g. `PUT` or `DELETE`) for different functionality but you want to wrap this in a single client-facing API, you can provide a simple interface offering a single method (e.g. `POST`) and then use the method transform middleware to map requests to correct upstream method. + +#### Enforcing API governance and standardization + +You can use the transform middleware to ensure that all requests to a service are made using the same HTTP method, regardless of the original method used by the client. This can help maintain consistency across different client applications accessing the same upstream API. + +#### Error Handling and Redirection + +You can use the method transformation middleware to handle errors and redirect requests to different endpoints, such as changing a DELETE request to a GET request when a specific resource is no longer available, allowing for graceful error handling and redirection. + +#### Testing and debugging + +Request method transformation can be useful when testing or debugging API endpoints; temporarily changing the request method can help to identify issues or test specific functionalities. + +### Working + +This is a very simple middleware that is assigned to an endpoint and configured with the HTTP method to which the request should be modified. The Request Method Transform middleware modifies the request method for the entire request flow, not just for the specific upstream request, so all subsequent middleware in the processing chain will use the new (transformed) method. + +
+ +If you're using Tyk OAS APIs, then you can find details and examples of how to configure the request method transform middleware [here](#request-method-using-tyk-oas). + +If you're using Tyk Classic APIs, then you can find details and examples of how to configure the request method transform middleware [here](#request-method-using-classic). + +{/* proposed "summary box" to be shown graphically on each middleware page + # Request Method Transform middleware summary + - The Request Method Transform is an optional stage in Tyk's API Request processing chain, sitting between the [TBC]() and [TBC]() middleware. + - The Request Method Transform is configured at the per-endpoint level within the API Definition and is supported by the API Designer within the Tyk Dashboard. */} + +

Using Tyk OAS

+Tyk's [request method transform](/api-management/traffic-transformation/request-method) middleware is configured at the endpoint level, where it modifies the HTTP method used in the request to a configured value. + +When working with Tyk OAS APIs the transformation is configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation). You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the legacy Tyk Classic APIs, then check out the [Tyk Classic](#request-method-using-classic) page. + +### API Definition + +The request method transform middleware (`transformRequestMethod`) can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). Endpoint `paths` entries (and the associated `operationId`) can contain wildcards in the form of any string bracketed by curly braces, for example `/status/{code}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +You only need to enable the middleware (set `enabled:true`) and then configure `toMethod` as the new HTTP method to which the request should be transformed. The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the method should be transformed. + +All standard HTTP methods are supported: `GET`, `PUT`, `POST`, `PATCH`, `DELETE`, `HEAD`, `OPTIONS`. + +For example: +```json {hl_lines=["39-41"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-request-method", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/status/200": { + "get": { + "operationId": "status/200get", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-request-method", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-request-method/", + "strip": true + } + }, + "middleware": { + "operations": { + "status/200get": { + "transformRequestMethod": { + "enabled": true, + "toMethod": "POST" + } + } + } + } + } +} +``` + +In this example the Request Method Transform middleware has been configured for requests to the `GET /status/200` endpoint. Any request received to that endpoint will be modified to `POST /status/200`. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the request method transform. + +### API Designer + +Adding the transform to your API endpoints is easy when using the API Designer in the Tyk Dashboard, simply follow these steps: + +1. **Add an endpoint** + + From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + + Tyk OAS API Designer showing no endpoints created + + Adding an endpoint to an API using the Tyk OAS API Designer + + Tyk OAS API Designer showing no middleware enabled on endpoint + +2. **Select the Method Transform middleware** + + Select **ADD MIDDLEWARE** and choose the **Method Transform** middleware from the *Add Middleware* screen. + + Adding the Request Method Transform middleware + +3. **Configure the middleware** + + Select the new HTTP method to which requests to this endpoint should be transformed + + Selecting the new HTTP method for requests to the endpoint + + Select **ADD MIDDLEWARE** to apply the change to the middleware configuration. + +4. **Save the API** + + Select **SAVE API** to apply the changes to your API. + +

Using Classic

+Tyk's [request method transform](/api-management/traffic-transformation/request-method) middleware is configured at the endpoint level, where it modifies the HTTP method used in the request to a configured value. + +When working with Tyk Classic APIs the transformation is configured in the Tyk Classic API Definition. You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the newer Tyk OAS APIs, then check out the [Tyk OAS](#request-method-using-tyk-oas) page. + +If you're using Tyk Operator then check out the [configuring a Request Method Transform in Tyk Operator](#tyk-operator) section below. + +### API Definition + +To configure a transformation of the request method you must add a new `method_transforms` object to the `extended_paths` section of your API definition. + +It has the following configuration: +- `path`: the endpoint path +- `method`: the endpoint HTTP method +- `to_method`: The new HTTP method to which the request should be transformed + +All standard HTTP methods are supported: `GET`, `PUT`, `POST`, `PATCH`, `DELETE`, `HEAD`, `OPTIONS`. + +For example: +```json expandable +{ + "method_transforms": [ + { + "path": "/status/200", + "method": "GET", + "to_method": "POST" + } + ] +} +``` + +In this example the Request Method Transform middleware has been configured for HTTP `GET` requests to the `/status/200` endpoint. Any request received to that endpoint will be modified to `POST /status/200`. + +### API Designer + +You can use the API Designer in the Tyk Dashboard to configure the request method transform middleware for your Tyk Classic API by following these steps. + +1. **Add an endpoint for the path and select the Method Transform plugin** + + From the **Endpoint Designer** add an endpoint that matches the path for which you want to perform the transformation. Select the **Method Transform** plugin. + + Method Transform + +2. **Configure the transform** + + Then select the HTTP method to which you wish to transform the request. + + Method Path + +3. **Save the API** + + Use the *save* or *create* buttons to save the changes and activate the middleware. + +### Tyk Operator + +The process for configuring a request method transform for an endpoint in Tyk Operator is similar to that defined in section configuring a Request Method Transform in the Tyk Classic API Definition. + +To configure a transformation of the request method you must add a new `method_transforms` object to the `extended_paths` section of your API definition: + +```yaml {linenos=true, linenostart=1, hl_lines=["26-29"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin +spec: + name: httpbin + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.default.svc:8000 + listen_path: /transform + strip_listen_path: true + version_data: + default_version: v1 + not_versioned: true + versions: + v1: + name: v1 + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + method_transforms: + - path: /anything + method: GET + to_method: POST +``` + +The example API Definition above configures an API to listen on path `/transform` and forwards requests upstream to http://httpbin.org. + +In this example the Request Method Transform middleware has been configured for `HTTP GET` requests to the `/anything` endpoint. Any request received to that endpoint will be modified to `POST /anything`. + diff --git a/api-management/traffic-transformation/request-size-limits.mdx b/api-management/traffic-transformation/request-size-limits.mdx new file mode 100644 index 00000000..5bf4c2ac --- /dev/null +++ b/api-management/traffic-transformation/request-size-limits.mdx @@ -0,0 +1,343 @@ +--- +title: "Request Size Limits" +'og:description': "How to configure Request Size Limits traffic transformation middleware in Tyk" +tags: ['Traffic Transformation', 'Request Size Limits'] +sidebarTitle: "Request Size Limits" +--- + +

Overview

+With Tyk, you can apply limits to the size of requests made to your HTTP APIs. You might use this feature to protect your Tyk Gateway or upstream services from excessive memory usage or brute force attacks. + +Tyk Gateway offers a flexible tiered system of limiting request sizes ranging from globally applied limits across all APIs deployed on the gateway down to specific size limits for individual API endpoints. + +### Use Case + +#### Protecting the entire Tyk Gateway from DDoS attacks +You can configure a system-wide request size limit that protects all APIs managed by the Tyk Gateway from being overwhelmed by excessively large requests, which could be part of a DDoS attack, ensuring the stability and availability of the gateway. + +#### Limiting request sizes for a lightweight microservice +You might expose an API for a microservice that is designed to handle lightweight, fast transactions and is not equipped to process large payloads. You can set an API-level size limit that ensures the microservice behind this API is not forced to handle requests larger than it is designed for, maintaining its performance and efficiency. + +#### Controlling the size of GraphQL queries +A GraphQL API endpoint might be susceptible to complex queries that can lead to performance issues. By setting a request size limit for the GraphQL endpoint, you ensure that overly complex queries are blocked, protecting the backend services from potential abuse and ensuring a smooth operation. + +#### Restricting upload size on a file upload endpoint +An API endpoint is designed to accept file uploads, but to prevent abuse, you want to limit the size of uploads to 1MB. To enforce this, you can enable the Request Size Limit middleware for this endpoint, configuring a size limit of 1MB. This prevents users from uploading excessively large files, protecting your storage and bandwidth resources. + +### Working + +Tyk compares each incoming API request with the configured maximum size for each level of granularity in order of precedence and will reject any request that exceeds the size you have set at any level of granularity, returning an HTTP 4xx error as detailed below. + +All size limits are stated in bytes and are applied only to the request _body_ (or payload), excluding the headers. + +| Precedence | Granularity | Error returned on failure | +| :------------ | :------------------ | :-------------------------------- | +| 1st | System (gateway) | `413 Request Entity Too Large` | +| 2nd | API | `400 Request is too large` | +| 3rd | Endpoint | `400 Request is too large` | + + + +The system level request size limit is the only size limit applied to [TCP](/key-concepts/tcp-proxy) and [Websocket](/advanced-configuration/websockets) connections. + + + +
+ +#### Applying a system level size limit +You can configure a request size limit (in bytes) that will be applied to all APIs on your Tyk Gateway by adding `max_request_body_size` to the `http_server_options` [element](/tyk-oss-gateway/configuration#http_server_optionsmax_request_body_size) of your `tyk.conf` Gateway configuration. For example: +```yaml +"max_request_body_size": 5000 +``` +A value of zero (default) means that no maximum is set and the system-wide size limit check will not be performed. + +This limit will be evaluated before API-level or endpoint-level configurations. If this test fails, the Tyk Gateway will return an error `HTTP 413 Request Entity Too Large`. + + + +Tyk Cloud Classic enforces a strict request size limit of 1MB on all inbound requests via our cloud architecture. This limit does not apply to Tyk Cloud users. + + + +
+ +If you're using Tyk OAS APIs, then you can find details and examples of how to configure an API or endpoint-level request size limit [here](#request-size-limits-using-tyk-oas). + +If you're using Tyk Classic APIs, then you can find details and examples of how to configure an API or endpoint-level request size limit [here](#request-size-limits-using-classic). + +{/* proposed "summary box" to be shown graphically on each middleware page + # Request Size Limit middleware summary + - The Request Size Limit middleware is an optional stage in Tyk's API Request processing chain, sitting between the [TBC]() and [TBC]() middleware. + - The Request Size Limit middleware can be configured at the system level within the Gateway config, or per-API or per-endpoint level within the API Definition and is supported by the API Designer within the Tyk Dashboard. */} + + +

Using Tyk OAS

+The [request size limit](/api-management/traffic-transformation/request-size-limits) middleware enables you to apply limits to the size of requests made to your HTTP APIs. You might use this feature to protect your Tyk Gateway or upstream services from excessive memory usage or brute force attacks. + +The middleware is configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation). You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the legacy Tyk Classic APIs, then check out the [Tyk Classic](#request-size-limits-using-classic) page. + +### API Definition + +There are three different levels of granularity that can be used when configuring a request size limit. +- [system-wide](#applying-a-system-level-size-limit): affecting all APIs deployed on the gateway +- [API-level](#applying-a-size-limit-for-a-specific-api): affecting all endpoints for an API +- [endpoint-level](#applying-a-size-limit-for-a-specific-endpoint): affecting a single API endpoint + +#### Applying a size limit for a specific API + +The API-level size limit has not yet been implemented for Tyk OAS APIs. + +You can work around this by implementing a combination of endpoint-level size limits and [allow](/api-management/traffic-transformation/allow-list#api-definition) or [block](/api-management/traffic-transformation/block-list#api-designer) lists. + +#### Applying a size limit for a specific endpoint + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. Endpoint `paths` entries (and the associated `operationId`) can contain wildcards in the form of any string bracketed by curly braces, for example `/status/{code}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +The virtual endpoint middleware (`requestSizeLimit`) can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + +The `requestSizeLimit` object has the following configuration: +- `enabled`: enable the middleware for the endpoint +- `value`: the maximum size permitted for a request to the endpoint (in bytes) + +For example: +```json {hl_lines=["39-44"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-request-size-limit", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/anything": { + "post": { + "operationId": "anythingpost", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-request-size-limit", + "state": { + "active": true, + "internal": false + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-request-size-limit/", + "strip": true + } + }, + "middleware": { + "operations": { + "anythingpost": { + "requestSizeLimit": { + "enabled": true, + "value": 100 + } + } + } + } + } +} +``` + +In this example the endpoint-level Request Size Limit middleware has been configured for HTTP `POST` requests to the `/anything` endpoint. For any call made to this endpoint, Tyk will check the size of the payload (Request body) and, if it is larger than 100 bytes, will reject the request, returning `HTTP 400 Request is too large`. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the virtual endpoint middleware. + +### API Designer + +Adding the Request Size Limit middleware to your API endpoints is easy when using the API Designer in the Tyk Dashboard, simply follow these steps: + +1. **Add an endpoint for the path** + + From the **API Designer** add an endpoint that matches the path for you want to limit the size of requests. + + Tyk OAS API Designer showing no endpoints created + + Adding an endpoint to an API using the Tyk OAS API Designer + + Tyk OAS API Designer showing no middleware enabled on endpoint + +2. **Select the Request Size Limit middleware** + + Select **ADD MIDDLEWARE** and choose the **Request Size Limit** middleware from the *Add Middleware* screen. + + Adding the Request Size Limit middleware + +3. **Configure the middleware** + + Now you can set the **size limit** that the middleware should enforce - remember that this is given in bytes. + + Setting the size limit that should be enforced + +4. **Save the API** + + Select **ADD MIDDLEWARE** to save the middleware configuration. Remember to select **SAVE API** to apply the changes to your API. + +

Using Classic

+The [request size limit](/api-management/traffic-transformation/request-size-limits) middleware enables you to apply limits to the size of requests made to your HTTP APIs. You might use this feature to protect your Tyk Gateway or upstream services from excessive memory usage or brute force attacks. + +This middleware is configured in the Tyk Classic API Definition. You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the newer Tyk OAS APIs, then check out the [Tyk OAS](#request-size-limits-using-tyk-oas) page. + +If you're using Tyk Operator then check out the [configuring the middleware in Tyk Operator](#tyk-operator) section below. + +### API Definition + +There are three different levels of granularity that can be used when configuring a request size limit. +- [system-wide](#applying-a-system-level-size-limit): affecting all APIs deployed on the gateway +- [API-level](/api-management/traffic-transformation/request-headers#tyk-classic-api): affecting all endpoints for an API +- [endpoint-level](#tyk-classic-endpoint): affecting a single API endpoint + +

Applying a size limit for a specific API

+You can configure a request size limit (in bytes) to an API by configuring the `global_size_limit` within the `version` element of the API Definition, for example: +``` +"global_size_limit": 2500 +``` + +A value of zero (default) means that no maximum is set and the API-level size limit check will not be performed. + +This limit is applied for all endpoints within an API. It is evaluated after the Gateway-wide size limit and before any endpoint-specific size limit. If this test fails, the Tyk Gateway will report `HTTP 400 Request is too large`. + +

Applying a size limit for a specific endpoint

+The most granular control over request sizes is provided by the endpoint-level configuration. This limit will be applied after any Gateway-level or API-level size limits and is given in bytes. If this test fails, the Tyk Gateway will report `HTTP 400 Request is too large`. + +To enable the middleware you must add a new `size_limits` object to the `extended_paths` section of your API definition. + +The `size_limits` object has the following configuration: +- `path`: the endpoint path +- `method`: the endpoint HTTP method +- `size_limit`: the maximum size permitted for a request to the endpoint (in bytes) + +For example: +```.json {linenos=true, linenostart=1} expandable +{ + "extended_paths": { + "size_limits": [ + { + "disabled": false, + "path": "/anything", + "method": "POST", + "size_limit": 100 + } + ] + } +} +``` + +In this example the endpoint-level Request Size Limit middleware has been configured for HTTP `POST` requests to the `/anything` endpoint. For any call made to this endpoint, Tyk will check the size of the payload (Request body) and, if it is larger than 100 bytes, will reject the request, returning `HTTP 400 Request is too large`. + +### API Designer + +You can use the API Designer in the Tyk Dashboard to configure a request size limit for your Tyk Classic API by following these steps. + +1. **Add an endpoint for the path and select the plugin** + + From the **Endpoint Designer** add an endpoint that matches the path for which you want to limit the size of requests. Select the **Request size limit** plugin. + + Select middleware + +2. **Configure the middleware** + + Set the request size limit, in bytes. + + Configure limit + +3. **Save the API** + + Use the *save* or *create* buttons to save the changes and activate the middleware. + + + + + The Tyk Classic API Designer does not provide an option to configure `global_size_limit`, but you can do this from the Raw Definition editor. + + + +### Tyk Operator + +The process for configuring a request size limit is similar to that defined in section configuring the middleware in the Tyk Classic API Definition. Tyk Operator allows you to configure a request size limit for [all endpoints of an API](#tyk-operator-api) or for a [specific API endpoint](#tyk-operator-endpoint). + +

Applying a size limit for a specific API

+{/* Need an example */} +The process for configuring the request size_limits middleware for a specific API is similar to that explained in [applying a size limit for a specific API](#tyk-classic-api). + +You can configure a request size limit (in bytes) for all endpoints within an API by configuring the `global_size_limit` within the `version` element of the API Definition, for example: + +```yaml {linenos=true, linenostart=1, hl_lines=["19"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-global-limit +spec: + name: httpbin-global-limit + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin-global-limit + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + global_size_limit: 5 + name: Default +``` + +The example API Definition above configures an API to listen on path `/httpbin-global-limit` and forwards requests upstream to http://httpbin.org. + +In this example the request size limit is set to 5 bytes. If the limit is exceeded then the Tyk Gateway will report `HTTP 400 Request is too large`. + +

Applying a size limit for a specific endpoint

+The process for configuring the request size_limits middleware for a specific endpoint is similar to that explained in [applying a size limit for a specific endpoint](#tyk-classic-endpoint). + +To configure the request size_limits middleware you must add a new `size_limits` object to the `extended_paths` section of your API definition, for example: + +```yaml {linenos=true, linenostart=1, hl_lines=["22-25"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-limit +spec: + name: httpbin-limit + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin-limit + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + extended_paths: + size_limits: + - method: POST + path: /post + size_limit: 5 +``` + +The example API Definition above configures an API to listen on path `/httpbin-limit` and forwards requests upstream to http://httpbin.org. + +In this example the endpoint-level Request Size Limit middleware has been configured for `HTTP POST` requests to the `/post` endpoint. For any call made to this endpoint, Tyk will check the size of the payload (Request body) and, if it is larger than 5 bytes, will reject the request, returning `HTTP 400 Request is too large`. \ No newline at end of file diff --git a/api-management/traffic-transformation/request-validation.mdx b/api-management/traffic-transformation/request-validation.mdx new file mode 100644 index 00000000..e3e41e8e --- /dev/null +++ b/api-management/traffic-transformation/request-validation.mdx @@ -0,0 +1,397 @@ +--- +title: "Request Validation" +'og:description': "How to configure Request Validation traffic transformation middleware in Tyk" +tags: ['Traffic Transformation', 'Request Validation'] +sidebarTitle: "Request Validation" +--- + +

Overview

+Requests to your upstream services should meet the contract that you have defined for those APIs. Checking the content and format of incoming requests before they are passed to the upstream APIs can avoid unexpected errors and provide additional security to those services. Tyk's request validation middleware provides a way to validate the presence, correctness and conformity of HTTP requests to make sure they meet the expected format required by the upstream API endpoints. + +Request validation enables cleaner backend APIs, better standardization across consumers, easier API evolution and reduced failure risk leading to higher end-to-end reliability. + +### Use Cases + +#### Improving security of upstream services + +Validating incoming requests against a defined schema protects services from unintended consequences arising from bad input, such as SQL injection or buffer overflow errors, or other unintended failures caused by missing parameters or invalid data types. Offloading this security check to the API Gateway provides an early line of defense as potentially bad requests are not proxied to your upstream services. + +#### Offloading contract enforcement + +You can ensure that client requests adhere to a defined contract specifying mandatory headers or parameters before sending requests upstream. Performing these validation checks in the API Gateway allows API developers to focus on core domain logic. + +#### Supporting data transformation + +Validation goes hand-in-hand with request [header](/api-management/traffic-transformation/request-headers) and [body](/api-management/traffic-transformation/request-body) transformation by ensuring that a request complies with the expected schema prior to transformation. For example, you could validate that a date parameter is present, then transform it into a different date format as required by your upstream API dynamically on each request. + +### Working + +The incoming request is compared with a defined schema, which is a structured description of the expected format for requests to the endpoint. This request schema defines the required and optional elements such as headers, path/query parameters, payloads and their data types. It acts as a contract for clients. + +If the incoming request does not match the schema, it will be rejected with an `HTTP 422 Unprocessable Entity` error. This error code can be customized if required. + +When using [Tyk OAS APIs](/api-management/traffic-transformation/request-validation#request-validation-using-tyk-oas), request validation is performed by the `Validate Request` middleware which can be enabled per-endpoint. The schema against which requests are compared is defined in the OpenAPI description of the endpoint. All elements of the request can have a `schema` defined in the OpenAPI description so requests to Tyk OAS APIs can be validated for headers, path/query parameters and body (payload). + +When using the legacy [Tyk Classic APIs](/api-management/traffic-transformation/request-validation#request-validation-using-classic), request validation is performed by the `Validate JSON` middleware which can be enabled per-endpoint. The schema against which requests are compared is defined in the middleware configuration and is limited to the request body (payload). Request headers and path/query parameters cannot be validated when using Tyk Classic APIs. + +
+ +If you're using Tyk OAS APIs, then you can find details and examples of how to configure the request validation middleware [here](/api-management/traffic-transformation/request-validation#request-validation-using-tyk-oas). + +If you're using Tyk Classic APIs, then you can find details and examples of how to configure the request validation middleware [here](/api-management/traffic-transformation/request-validation#request-validation-using-classic). + +{/* proposed "summary box" to be shown graphically on each middleware page + # Validate Request middleware summary + - The Validate Request middleware is an optional stage in Tyk's API Request processing chain, sitting between the [TBC]() and [TBC]() middleware. + - The Validate Request middleware can be configured at the per-endpoint level within the API Definition and is supported by the API Designer within the Tyk Dashboard. */} + + +

Using Tyk OAS

+The [request validation](#request-validation-overview) middleware provides a way to validate the presence, correctness and conformity of HTTP requests to make sure they meet the expected format required by the upstream API endpoints. If the incoming request fails validation, the Tyk Gateway will reject the request with an `HTTP 422 Unprocessable Entity` response. Tyk can be [configured](#configuring-the-request-validation-middleware) to return a different HTTP status code if required. + +The middleware is configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation). You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the legacy Tyk Classic APIs, then check out the [Tyk Classic](/api-management/traffic-transformation/request-validation#request-validation-using-classic) page. + +### Request schema in OpenAPI Specification + +The OpenAPI Specification supports the definition of a [schema](https://learn.openapis.org/specification/content.html#the-schema-object) to describe and limit the content of any field in an API request or response. + +Tyk's request validation middleware automatically parses the schema for the request in the OpenAPI description part of the Tyk OAS API Definition and uses this to compare against the incoming request. + +An OpenAPI schema can reference other schemas defined elsewhere, letting you write complex validations very efficiently since you don’t need to re-define the validation for a particular object every time you wish to refer to it. Tyk only supports local references to schemas (within the same OpenAPI document). + +As explained in the OpenAPI [documentation](https://learn.openapis.org/specification/parameters.html), the structure of an API request is described by two components: +- parameters (headers, query parameters, path parameters) +- request body (payload) + +#### Request parameters + +The `parameters` field in the OpenAPI description is an array of [parameter objects](https://swagger.io/docs/specification/describing-parameters/) that each describe one variable element in the request. Each `parameter` has two mandatory fields: +- `in`: the location of the parameter (`path`, `query`, `header`) +- `name`: a unique identifier within that location (i.e. no duplicate header names for a given operation/endpoint) + +There are also optional `description` and `required` fields. + +For each parameter, a schema can be declared that defines the `type` of data that can be stored (e.g. `boolean`, `string`) and any `example` or `default` values. + +##### Operation (endpoint-level) parameters + +An operation is a combination of HTTP method and path or, as Tyk calls it, an endpoint - for example `GET /users`. Operation, or endpoint-level parameters can be defined in the OpenAPI description and will apply only to that operation within the API. These can be added or modified within Tyk Dashboard's [API designer](#api-designer). + +##### Common (path-level) parameters + +[Common parameters](https://swagger.io/docs/specification/v3_0/describing-parameters/#common-parameters), that apply to all operations within a path, can be defined at the path level within the OpenAPI description. Tyk refers to these as path-level parameters and displays them as read-only fields in the Dashboard's API designer. If you need to add or modify common parameters you must use the *Raw Definition* editor, or edit your OpenAPI document outside Tyk and [update](/api-management/gateway-config-managing-oas#updating-an-api) the API. + +#### Request body + +The `requestBody` field in the OpenAPI description is a [Request Body Object](https://swagger.io/docs/specification/describing-request-body/). This has two optional fields (`description` and `required`) plus the `content` section which allows you to define a schema for the expected payload. Different schemas can be declared for different media types that are identified by content-type (e.g. `application/json`, `application/xml` and `text/plain`). + +### Configuring the request validation middleware + +When working with Tyk OAS APIs, the request validation middleware automatically determines the validation rules based on the API schema. The only configurable option for the middleware is to set the desired HTTP status code that will be returned if a request fails validation. The default response will be `HTTP 422 Unprocessable Entity` unless otherwise configured. + +### Enabling the request validation middleware + +If the middleware is enabled for an endpoint, then Tyk will automatically validate requests made to that endpoint against the schema defined in the API definition. + +When you create a Tyk OAS API by importing your OpenAPI description, you can instruct Tyk to enable request validation [automatically](#automatically-enabling-the-request-validation-middleware) for all endpoints with defined schemas. + +If you are creating your API without import, or if you only want to enable request validation for some endpoints, you can [manually enable](#manually-enabling-the-request-validation-middleware) the middleware in the Tyk OAS API definition. + +#### Automatically enabling the request validation middleware + +The request validation middleware can be enabled for all endpoints that have defined schemas when [importing](/api-management/gateway-config-managing-oas#importing-an-openapi-description-to-create-an-api) an OpenAPI Document to create a Tyk OAS API. +- if you are using the `POST /apis/oas/import` endpoint in the [Tyk Dashboard API](/tyk-dashboard-api) or [Tyk Gateway API](/tyk-gateway-api) then you can do this by setting the `validateRequest=true` query parameter +- if you are using the API Designer, select the **Auto-generate middleware to validate requests** option on the **Import API** screen + +Select the option during OpenAPI import to validate requests + +As noted, the automatic application of request validation during import will apply the middleware to all endpoints declared in your OpenAPI description. If you want to adjust this configuration, for example to remove validation from specific endpoints or to change the HTTP status code returned on error, you can update the Tyk OAS API definition as described [here](#manually-enabling-the-request-validation-middleware). + +#### Manually enabling the request validation middleware + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. Endpoint `paths` entries (and the associated `operationId`) can contain wildcards in the form of any string bracketed by curly braces, for example `/status/{code}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +The request validation middleware (`validateRequest`) can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId`. The `operationId` for an endpoint can be found within the `paths` section of your [OpenAPI specification](https://swagger.io/docs/specification/paths-and-operations/?sbsearch=operationIds). + +The `validateRequest` object has the following configuration: +- `enabled`: enable the middleware for the endpoint +- `errorResponseCode`: [optional] the HTTP status code to be returned if validation fails (this defaults to `HTTP 422 Unprocessable Entity` if not set) + +For example: +```json {hl_lines=["69-72"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-validate-request", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/anything": { + "get": { + "operationId": "anythingget", + "parameters": [ + { + "in": "header", + "name": "X-Security", + "required": true, + "schema": { + "type": "boolean" + } + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "properties": { + "firstname": { + "description": "The person's first name", + "type": "string" + }, + "lastname": { + "description": "The person's last name", + "type": "string" + } + }, + "type": "object" + } + } + } + }, + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-validate-request", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-validate-request/", + "strip": true + } + }, + "middleware": { + "operations": { + "anythingget": { + "validateRequest": { + "enabled": true, + "errorResponseCode": 400 + } + } + } + } + } +} +``` + +In this example the request validation middleware has been configured for requests to the `GET /anything` endpoint. The middleware will check for the existence of a header named `X-Security` and the request body will be validated against the declared schema. If there is no match, the request will be rejected and Tyk will return `HTTP 400` (as configured in `errorResponseCode`). + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the request validation middleware. + +### API Designer + +Adding and configuring Request Validation for your API endpoints is easy when using the API Designer in the Tyk Dashboard, simply follow these steps: + +1. **Add an endpoint** + + From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + + Tyk OAS API Designer showing no endpoints created + + Adding an endpoint to an API using the Tyk OAS API Designer + + Tyk OAS API Designer showing no middleware enabled on endpoint + +2. **Select the Validate Request middleware** + + Select **ADD MIDDLEWARE** and choose **Validate Request** from the *Add Middleware* screen. + + Adding the Validate Request middleware + + The API Designer will show you the request body and request parameters schema detected in the OpenAPI description of the endpoint. + + Validate Request middleware schema is automatically populated + +3. **Configure the middleware** + + If required, you can select an alternative HTTP status code that will be returned if request validation fails. + + Configuring the Request Validation error response + +4. **Save the API** + + Select **ADD MIDDLEWARE** to save the middleware configuration. Remember to select **SAVE API** to apply the changes. + + +

Using Classic

+The [request validation](#request-validation-overview) middleware provides a way to validate the presence, correctness and conformity of HTTP requests to make sure they meet the expected format required by the upstream API endpoints. + +When working with legacy Tyk Classic APIs, request validation is performed by the `Validate JSON` middleware which can be enabled per-endpoint. The schema against which requests are compared is defined in the middleware configuration and is limited to the request body (payload). Request headers and path/query parameters cannot be validated when using Tyk Classic APIs. + +This middleware is configured in the Tyk Classic API Definition. You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the newer Tyk OAS APIs, then check out the [Tyk OAS](/api-management/traffic-transformation/request-validation#request-validation-using-tyk-oas) page. + +If you're using Tyk Operator then check out the [configuring the middleware in Tyk Operator](#tyk-operator) section below. + +### API Definition + +To enable the middleware you must add a new `validate_json` object to the `extended_paths` section of your API definition. + +The `validate_json` object has the following configuration: + +- `path`: the endpoint path +- `method`: the endpoint HTTP method +- `schema`: the [JSON schema](https://json-schema.org/understanding-json-schema/basics) against which the request body will be compared +- `error_response_code`: the HTTP status code that will be returned if validation fails (defaults to `422 Unprocessable Entity`) + +For example: + +```json {linenos=true, linenostart=1} expandable +{ + "extended_paths": { + "validate_json": [ + { + "disabled": false, + "path": "/register", + "method": "POST", + "schema": { + "type": "object", + "properties": { + "firstname": { + "type": "string", + "description": "The person's first name" + }, + "lastname": { + "type": "string", + "description": "The person's last name" + } + } + }, + "error_response_code": 422 + } + ] + } +} +``` + +In this example the Validate JSON middleware has been configured for requests to the `POST /register` endpoint. For any call made to this endpoint, Tyk will compare the request body with the schema and, if it does not match, the request will be rejected with the error code `HTTP 422 Unprocessable Entity`. + +#### Understanding JSON Schema Version Handling + +The Gateway automatically detects the version of the JSON schema from the `$schema` field in your schema definition. This field specifies the version of the [JSON schema standard](https://json-schema.org/specification-links) to be followed. + +From Tyk 5.8 onwards, supported versions are [draft-04](https://json-schema.org/draft-04/schema), [draft-06](https://json-schema.org/draft-06/schema) and [draft-07](https://json-schema.org/draft-07/schema). + +In previous versions of Tyk, only [draft-04](https://json-schema.org/draft-04/schema) is supported. Please be careful if downgrading from Tyk 5.8 to an earlier version that your JSON is valid as you might experience unexpected behaviour if using features from newer drafts of the JSON schema. + +- If the `$schema` field is present, the Gateway strictly follows the rules of the specified version. +- If the `$schema` field is missing or the version is not specified, the Gateway uses a hybrid mode that combines features from multiple schema versions. This mode ensures that the validation will still work, but may not enforce the exact rules of a specific version. + +To ensure consistent and predictable validation, it is recommended to always include the `$schema` field in your schema definition. For example: + +```json expandable +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "firstname": { + "type": "string" + }, + "lastname": { + "type": "string" + } + } +} +``` + +By including `$schema`, the validator can operate in strict mode, ensuring that the rules for your chosen schema version are followed exactly. + +### API Designer + +You can use the API Designer in the Tyk Dashboard to configure the request validation middleware for your Tyk Classic API by following these steps. + +1. **Add an endpoint for the path and select the plugin** + + From the **Endpoint Designer** add an endpoint that matches the path for which you want to validate the request payload. Select the **Validate JSON** plugin. + + validate json plugin + +2. **Configure the middleware** + + Once you have selected the request validation middleware for the endpoint, you can select an error code from the drop-down list (if you don't want to use the default `422 Unprocessable Entity`) and enter your JSON schema in the editor. + + Adding schema to the Validate JSON middleware + +3. **Save the API** + + Use the *save* or *create* buttons to save the changes and activate the middleware. + +### Tyk Operator + +The process for configuring the middleware in Tyk Operator is similar to that explained in configuring the middleware in the Tyk Classic API Definition. To configure the request validation middleware you must add a new `validate_json` object to the `extended_paths` section of your API definition, for example: + +The example API Definition below configures an API to listen on path `/httpbin` and forwards requests upstream to http://httpbin.org. + +In this example, the Validate JSON middleware has been configured for requests to the `GET /get` endpoint. For any call made to this endpoint, Tyk will compare the request body with the schema and, if it does not match, the request will be rejected with the error code `HTTP 422 Unprocessable Entity`. + +```yaml {linenos=true, linenostart=1, hl_lines=["26-41"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-json-schema-validation +spec: + name: httpbin-json-schema-validation + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + validate_json: + - error_response_code: 422 + disabled: false + path: /get + method: GET + schema: + properties: + userName: + type: string + minLength: 2 + age: + type: integer + minimum: 1 + required: + - userName + type: object +``` + diff --git a/api-management/traffic-transformation/response-body.mdx b/api-management/traffic-transformation/response-body.mdx new file mode 100644 index 00000000..cf48a5a4 --- /dev/null +++ b/api-management/traffic-transformation/response-body.mdx @@ -0,0 +1,479 @@ +--- +title: "Response Body" +'og:description': "How to configure Response Body traffic transformation middleware in Tyk" +tags: ['Traffic Transformation', 'Response Body'] +sidebarTitle: "Response Body" +--- + +

Overview

+Tyk enables you to modify the payload of API responses received from your upstream services before they are passed on to the client that originated the request. This makes it easy to transform between payload data formats or to expose legacy APIs using newer schema models without having to change any client implementations. This middleware is only applicable to endpoints that return a body with the response. + +With the body transform middleware you can modify XML or JSON formatted payloads to ensure that the response contains the information required by your upstream service. You can enrich the response by adding contextual data that is held by Tyk but not included in the original response from the upstream. + +This middleware changes only the payload and not the headers. You can, however, combine this with the [Response Header Transform](/api-management/traffic-transformation/response-headers) to apply more complex transformation to responses. + +There is a closely related [Request Body Transform](/api-management/traffic-transformation/request-body) middleware that provides the same functionality on the request sent by the client prior to it being proxied to the upstream. + +### Use Cases + +#### Maintaining compatibility with legacy clients + +Sometimes you might have a legacy API and need to migrate the transactions to a new upstream service but do not want to upgrade all the existing clients to the newer upstream API. Using response body transformation, you can convert the new format that your upstream services provide into legacy XML or JSON expected by the clients. + +#### Shaping responses for different devices + +You can detect the client device types via headers or context variables and transform the response payload to optimize it for that particular device. For example, you might optimize the response content for mobile apps. + +#### SOAP to REST translation + +A common use of the response body transform middleware is when surfacing a legacy SOAP service with a REST API. Full details of how to perform this conversion using Tyk are provided [here](/advanced-configuration/transform-traffic/soap-rest). + +### Working + +Tyk's body transform middleware uses the [Go template language](https://golang.org/pkg/text/template/) to parse and modify the provided input. We have bundled the [Sprig Library (v3)](http://masterminds.github.io/sprig/) which provides over 70 pre-written functions for transformations to assist the creation of powerful Go templates to transform your API responses. + +The Go template can be defined within the API Definition or can be read from a file that is accessible to Tyk, for example alongside your [error templates](/api-management/gateway-events#error-templates). + +We have provided more detail, links to reference material and some examples of the use of Go templating [here](/api-management/traffic-transformation/go-templates). + + + +Tyk evaluates templates stored in files on startup, so if you make changes to a template you must remember to restart the gateway. + + + +#### Supported response body formats + +The body transformation middleware can modify response payloads in the following formats: +- JSON +- XML + +When working with JSON format data, the middleware will unmarshal the data into a data structure, and then make that data available to the template in dot-notation. + +#### Data accessible to the middleware + +The middleware has direct access to the response body and also to dynamic data as follows: +- [Context variables](/api-management/traffic-transformation/request-context-variables), extracted from the request at the start of the middleware chain, can be injected into the template using the `._tyk_context.KEYNAME` namespace +- [Session metadata](/api-management/policies#what-is-a-session-metadata), from the Tyk Session Object linked to the request, can be injected into the template using the `._tyk_meta.KEYNAME` namespace +- Inbound form or query data can be accessed through the `._tyk_context.request_data` namespace where it will be available in as a `key:[]value` map +- values from [key-value (KV) storage](/tyk-configuration-reference/kv-store#transformation-middleware) can be injected into the template using the notation appropriate to the location of the KV store + +The response body transform middleware can iterate through list indices in dynamic data so, for example, calling `{{ index ._tyk_context.request_data.variablename 0 }}` in a template will expose the first entry in the `request_data.variablename` key/value array. + + + +As explained in the [documentation](https://pkg.go.dev/text/template), templates are executed by applying them to a data structure. The template receives the decoded JSON or XML of the response body. If session variables or meta data are enabled, additional fields will be provided: `_tyk_context` and `_tyk_meta` respectively. + + + +#### Automatic XML <-> JSON Transformation + +A very common transformation that is applied in the API Gateway is to convert between XML and JSON formatted body content. + +The Response Body Transform supports two helper functions that you can use in your Go templates to facilitate this: +- `jsonMarshal` performs JSON style character escaping on an XML field and, for complex objects, serialises them to a JSON string ([example](/api-management/traffic-transformation/go-templates#xml-to-json-conversion-using-jsonmarshal)) +- `xmlMarshal` performs the equivalent conversion from JSON to XML ([example](/api-management/traffic-transformation/go-templates#json-to-xml-conversion-using-xmlmarshal)) + +
+ +If you're using Tyk OAS APIs, then you can find details and examples of how to configure the response body transformation middleware [here](#response-body-using-tyk-oas). + +If you're using Tyk Classic APIs, then you can find details and examples of how to configure the response body transformation middleware [here](#response-body-using-classic). + +{/* proposed "summary box" to be shown graphically on each middleware page + # Response Body Transform middleware summary + - The Response Body Transform middleware is an optional stage in Tyk's API Response processing chain, sitting between the [TBC]() and [TBC]() middleware. + - The Response Body Transform middleware can be configured at the per-endpoint level within the API Definition and is supported by the API Designer within the Tyk Dashboard. + - Response Body Transform can access both [session metadata](/api-management/policies#what-is-a-session-metadata) and [request context variables](/api-management/traffic-transformation/request-context-variables). */} + + +

Using Tyk OAS

+The [response body transform](/api-management/traffic-transformation/response-body) middleware provides a way to modify the payload of API responses before they are returned to the client. + +The middleware is configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation). You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the legacy Tyk Classic APIs, then check out the [Tyk Classic](#response-body-using-classic) page. + +### API Definition + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. Endpoint `paths` entries (and the associated `operationId`) can contain wildcards in the form of any string bracketed by curly braces, for example `/status/{code}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +The response body transformation middleware (`transformResponseBody`) can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + +The `transformResponseBody` object has the following configuration: +- `enabled`: enable the middleware for the endpoint +- `format`: the format of input data the parser should expect (either `xml` or `json`) +- `body`: [see note] this is a `base64` encoded representation of your template +- `path`: [see note] this is the path to the text file containing the template + + + + + You should configure only one of `body` or `path` to indicate whether you are embedding the template within the middleware or storing it in a text file. The middleware will automatically select the correct source based on which of these fields you complete. If both are provided, then `body` will take precedence and `path` will be ignored. + + + +For example: +```json {hl_lines=["39-43"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-response-body-transform", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/anything": { + "put": { + "operationId": "anythingput", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-response-body-transform", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-response-body-transform/", + "strip": true + } + }, + "middleware": { + "operations": { + "anythingput": { + "transformResponseBody": { + "enabled": true, + "format": "json", + "body": "ewogICJ2YWx1ZTEiOiAie3sudmFsdWUyfX0iLAogICJ2YWx1ZTIiOiAie3sudmFsdWUxfX0iLAogICJyZXEtaGVhZGVyIjogInt7Ll90eWtfY29udGV4dC5oZWFkZXJzX1hfSGVhZGVyfX0iLAogICJyZXEtcGFyYW0iOiAie3suX3R5a19jb250ZXh0LnJlcXVlc3RfZGF0YS5wYXJhbX19Igp9" + } + } + } + } + } +} +``` + +In this example the response body transform middleware has been configured for requests to the `PUT /anything` endpoint. The `body` contains a base64 encoded Go template (which you can check by pasting the value into a service such as [base64decode.org](https://www.base64decode.org)). + +Decoded, this template is: +```go expandable +{ + "value1": "{{.value2}}", + "value2": "{{.value1}}", + "req-header": "{{._tyk_context.headers_X_Header}}", + "req-param": "{{._tyk_context.request_data.param}}" +} +``` + +So if you make a request to `PUT /anything?param=foo`, configuring a header `X-Header`:`bar` and providing this payload: +```json +{ + "value1": "world", + "value2": "hello" +} +``` + +httpbin.org will respond with the original payload in the response and, if you do not have the response body transform middleware enabled, the response from Tyk will include: +```json +{ + "value1": "world", + "value2": "hello" +} +``` + +If, however, you enable the response body transform middleware, Tyk will modify the response to include this content: +```json expandable +{ + "req-header": "bar", + "req-param": "[foo]", + "value1": "hello", + "value2": "world" +} +``` + +You can see that Tyk has swapped `value1` and `value2` and embedded the `X-Header` header and `param` query values from the request into the body of the response. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the mock response middleware. + + + +If using a template in a file (i.e. you configure `path` in the `transformResponseBody` object), remember that Tyk will load and evaluate the template when the Gateway starts up. If you modify the template, you will need to restart Tyk in order for the changes to take effect. + + + +### API Designer + +Adding Response Body Transformation to your API endpoints is easy when using the API Designer in the Tyk Dashboard, simply follow the following steps: + +1. **Add an endpoint** + + From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + + Tyk OAS API Designer showing no endpoints created + + Adding an endpoint to an API using the Tyk OAS API Designer + + Tyk OAS API Designer showing no middleware enabled on endpoint + +2. **Select the Response Body Transform middleware** + + Select **ADD MIDDLEWARE** and choose the **Response Body Transform** middleware from the *Add Middleware* screen. + + Adding the Response Body Transform middleware + +3. **Configure the middleware** + + Now you can select the response body format (JSON or XML) and add either a path to the file containing the template, or directly enter the transformation template in the text box. + + Configuring the Response Body Transform middleware + + The **Test with data** control will allow you to test your body transformation function by providing an example response body and generating the output from the transform. It is not possible to configure headers, other request parameters, context or session metadata to this template test so if you are using these data sources in your transform it will not provide a complete output, for example: + + Testing the Response Body Transform + +4. **Save the API** + + Select **SAVE API** to apply the changes to your API. + +

Using Classic

+The [response body transform](/api-management/traffic-transformation/response-body) middleware provides a way to modify the payload of API responses before they are returned to the client. + +This middleware is configured in the Tyk Classic API Definition at the endpoint level. You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the newer Tyk OAS APIs, then check out the [Tyk OAS](#response-body-using-tyk-oas) page. + +If you're using Tyk Operator then check out the [configuring the middleware in Tyk Operator](#tyk-operator) section below. + +### API Definition + +To enable the middleware you must add a new `transform_response` object to the `extended_paths` section of your API definition. + +The `transform_response` object has the following configuration: +- `path`: the path to match on +- `method`: this method to match on +- `template_data`: details of the Go template to be applied for the transformation of the response body + +The Go template is described in the `template_data` object by the following fields: +- `input_type`: the format of input data the parser should expect (either `xml` or `json`) +- `enable_session`: set this to `true` to make session metadata available to the transform template +- `template_mode`: instructs the middleware to look for the template either in a `file` or in a base64 encoded `blob`; the actual file location (or base64 encoded template) is provided in `template_source` +- `template_source`: if `template_mode` is set to `file`, this will be the path to the text file containing the template; if `template_mode` is set to `blob`, this will be a `base64` encoded representation of your template + +For example: +```json {linenos=true, linenostart=1} expandable +{ + "extended_paths": { + "transform_response": [ + { + "path": "/anything", + "method": "POST", + "template_data": { + "template_mode": "file", + "template_source": "./templates/transform_test.tmpl", + "input_type": "json", + "enable_session": true + } + } + ] + } +} +``` + +In this example, the Response Body Transform middleware is directed to use the template located in the `file` at location `./templates/transform_test.tmpl`. The input (pre-transformation) response payload will be `json` format and session metadata will be available for use in the transformation. + + + +Tyk will load and evaluate the template file when the Gateway starts up. If you modify the template, you will need to restart Tyk in order for the changes to take effect. + + + + + +Prior to Tyk 5.3, there was an additional step to enable response body transformation. You would need to add the following to the Tyk Classic API definition: + +```json +{ + "response_processors":[ + {"name": "response_body_transform"} + ] +} +``` + +If using the Endpoint Designer in the Tyk Dashboard, this would be added automatically. + +We removed the need to configure the `response_processors` element in Tyk 5.3.0. + + + +### API Designer + +You can use the API Designer in the Tyk Dashboard to configure the response body transform middleware for your Tyk Classic API by following these steps. + +1. **Add an endpoint for the path and select the plugin** + + From the **Endpoint Designer** add an endpoint that matches the path for which you want to perform the transformation. Select the **Body Transforms** plugin. + + Endpoint designer + +2. **Configure the middleware** + + Ensure that you have selected the `RESPONSE` tab, then select your input type, and then add the template you would like to use to the **Template** input box. + + Setting the body response transform + +3. **Test the Transform** + + If you have sample input data, you can use the Input box to add it, and then test it using the **Test** button. You will see the effect of the template on the sample input in the Output box. + + Testing the body transform function + +4. **Save the API** + + Use the *save* or *create* buttons to save the changes and activate the Response Body Transform middleware. + +### Tyk Operator + +The process of configuring a transformation of a response body for a specific endpoint is similar to that defined in section configuring the middleware in the Tyk Classic API Definition for the Tyk Classic API definition. To enable the middleware you must add a new `transform_response` object to the `extended_paths` section of your API definition. + +In the examples below, the Response Body Transform middleware (`transform_response`) is directed to use the template located in the `template_source`, decoding the xml in the base64 encoded string. The input (pre-transformation) response payload will be `xml` format and there is no session metadata provided for use in the transformation. + +#### Example + +```yaml {linenos=true, linenostart=1, hl_lines=["45-53"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-transform +spec: + name: httpbin-transform + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin-transform + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + transform: + - method: POST + path: /anything + template_data: + enable_session: false + input_type: json + template_mode: blob + # base64 encoded template + template_source: eyJiYXIiOiAie3suZm9vfX0ifQ== + transform_headers: + - delete_headers: + - "remove_this" + add_headers: + foo: bar + path: /anything + method: POST + transform_response: + - method: GET + path: /xml + template_data: + enable_session: false + input_type: xml + template_mode: blob + # base64 encoded template + template_source: e3sgLiB8IGpzb25NYXJzaGFsIH19 + transform_response_headers: + - method: GET + path: /xml + add_headers: + Content-Type: "application/json" + act_on: false + delete_headers: [] +``` + +

Tyk Gateway < 5.3.0 Example

+If using Tyk Gateway < v5.3.0 then a `response_processor` object must be added to the API definition containing a `response_body_transform` item, as highlighted below: + +```yaml {linenos=true, linenostart=1, hl_lines=["17-18", "48-56"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-transform +spec: + name: httpbin-transform + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin-transform + strip_listen_path: true + response_processors: + - name: response_body_transform + - name: header_injector + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + transform: + - method: POST + path: /anything + template_data: + enable_session: false + input_type: json + template_mode: blob + # base64 encoded template + template_source: eyJiYXIiOiAie3suZm9vfX0ifQ== + transform_headers: + - delete_headers: + - "remove_this" + add_headers: + foo: bar + path: /anything + method: POST + transform_response: + - method: GET + path: /xml + template_data: + enable_session: false + input_type: xml + template_mode: blob + # base64 encoded template + template_source: e3sgLiB8IGpzb25NYXJzaGFsIH19 + transform_response_headers: + - method: GET + path: /xml + add_headers: + Content-Type: "application/json" + act_on: false + delete_headers: [] +``` + diff --git a/api-management/traffic-transformation/response-headers.mdx b/api-management/traffic-transformation/response-headers.mdx new file mode 100644 index 00000000..8961d5ca --- /dev/null +++ b/api-management/traffic-transformation/response-headers.mdx @@ -0,0 +1,661 @@ +--- +title: "Response Headers" +'og:description': "How to configure Response Headers traffic transformation middleware in Tyk" +tags: ['Traffic Transformation', 'Response Headers'] +sidebarTitle: "Response Headers" +--- + +

Overview

+Tyk enables you to modify header information when a response is proxied back to the client. This can be very useful in cases where you have an upstream API that potentially exposes sensitive headers that you need to remove. + +There are two options for this: +- API-level modification that is applied to responses for all requests to the API +- endpoint-level modification that is applied only to responses for requests to a specific endpoint + +With the header transform middleware you can append or delete any number of headers to ensure that the response contains the information required by your client. You can enrich the response by adding contextual data that is held by Tyk but not included in the original response from the upstream. + +This middleware changes only the headers and not the payload. You can, however, combine this with the [Response Body Transform](/api-management/traffic-transformation/response-body) to apply more complex transformation to responses. + +There are related [Request Header Transform](/api-management/traffic-transformation/request-headers) middleware (at API-level and endpoint-level) that provide the same functionality on the request from a client, prior to it being proxied to the upstream. + +### Use Cases + +#### Customizing responses for specific clients + +A frequent use case for response header transformation is when a client requires specific headers for their application to function correctly. For example, a client may require a specific header to indicate the status of a request or to provide additional information about the response. + +#### Adding security headers + +The response header transform allows you to add security headers to the response to protect against common attacks such as cross-site scripting (XSS) and cross-site request forgery (CSRF). Some security headers may be required for compliance with industry standards and, if not provided by the upstream, can be added by Tyk before forwarding the response to the client. + +#### Adding metadata to response headers + +Adding metadata to response headers can be useful for tracking and analyzing API usage, as well as for providing additional information to clients. For example, you may want to add a header that indicates the version of the API being used or the time taken to process the request. + +#### Modifying response headers for dynamic performance optimization + +You can use response header transformation to dynamically optimize the performance of the API. For example, you may want to indicate to the client the maximum number of requests that they can make in a given time period. By doing so through the response headers, you can perform dynamic optimization of the load on the upstream service without triggering the rate limiter and so avoiding errors being sent to the client. + +### Working + +The response header transform can be applied per-API or per-endpoint; each has a separate entry in the API definition so that you can configure both API-level and endpoint-level transforms for a single API. + +The middleware is configured with a list of headers to delete from the response and a list of headers to add to the response. Each header to be added to the response is configured as a key:value pair. +- the "delete header" functionality is intended to ensure that any header in the delete list is not present once the middleware completes. If a header in the delete list is not present in the upstream response, the middleware will ignore the omission +- the "add header" functionality will capitalize any header name provided. For example, if you configure the middleware to append `x-request-id` it will be added to the response as `X-Request-Id` + +In the response middleware chain, the endpoint-level transform is applied before the API-level transform. Subsequently, if both middleware are enabled, the API-level transform will operate on the headers that have been added by the endpoint-level transform (and will not have access to those that have been deleted by it). + +#### Injecting dynamic data into headers + +You can enrich the response headers by injecting data from context variables or session objects into the headers. +- [context variables](/api-management/traffic-transformation/request-context-variables), extracted from the request at the start of the middleware chain, can be injected into added headers using the `$tyk_context.` namespace +- [session metadata](/api-management/policies#what-is-a-session-metadata), from the Tyk Session Object linked to the request, can be injected into added headers using the `$tyk_meta.` namespace +- values from [key-value (KV) storage](/tyk-configuration-reference/kv-store#transformation-middleware) can be injected into added headers using the notation appropriate to the location of the KV store + +
+ +If you're using Tyk OAS APIs, then you can find details and examples of how to configure the response header transform middleware [here](#response-headers-using-tyk-oas). + +If you're using Tyk Classic APIs, then you can find details and examples of how to configure the response header transform middleware [here](#response-headers-using-classic). + +{/* proposed "summary box" to be shown graphically on each middleware page + # Response Header Transform middleware summary + - The Response Header Transform is an optional stage in Tyk's API Response processing chain, sitting between the [TBC]() and [TBC]() middleware. + - The Response Header Transform can be configured at the per-endpoint or per-API level within the API Definition and is supported by the API Designer within the Tyk Dashboard. */} + +

Using Tyk OAS

+Tyk's [response header transform](/api-management/traffic-transformation/response-headers) middleware enables you to append or delete headers on responses received from the upstream service before sending them to the client. + +There are two options for this: +- API-level modification that is applied to all responses for the API +- endpoint-level modification that is applied only to responses from a specific endpoint + + + + + If both API-level and endpoint-level middleware are configured, the endpoint-level transformation will be applied first. + + + +When working with Tyk OAS APIs the transformation is configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation). You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the legacy Tyk Classic APIs, then check out the [Tyk Classic](#response-headers-using-classic) page. + +### API Definition + +The API-level and endpoint-level response header transforms have a common configuration but are configured in different sections of the API definition. + +#### API-level transform + +To append headers to, or delete headers from, responses from all endpoints defined for your API you must add a new `transformResponseHeaders` object to the `middleware.global` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition. + +You only need to enable the middleware (set `enabled:true`) and then configure the details of headers to `add` and those to `remove`. + +For example: +```json {hl_lines=["38-57"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-response-header", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/status/200": { + "get": { + "operationId": "status/200get", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-response-header", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-response-header/", + "strip": true + } + }, + "middleware": { + "global": { + "transformResponseHeaders": { + "enabled": true, + "remove": [ + "X-Secret" + ], + "add": [ + { + "name": "X-Static", + "value": "foobar" + }, + { + "name": "X-Request-ID", + "value": "$tyk_context.request_id" + }, + { + "name": "X-User-ID", + "value": "$tyk_meta.uid" + } + ] + } + } + } + } +} +``` + +This configuration will add three new headers to each response: +- `X-Static` with the value `foobar` +- `X-Request-ID` with a dynamic value taken from the `request_id` [context variable](/api-management/traffic-transformation/request-context-variables) +- `X-User-ID` with a dynamic value taken from the `uid` field in the [session metadata](/api-management/policies#what-is-a-session-metadata) + +It will also delete one header (if present) from each response: +- `X-Secret` + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the API-level response header transform. + +#### Endpoint-level transform + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. Endpoint `paths` entries (and the associated `operationId`) can contain wildcards in the form of any string bracketed by curly braces, for example `/status/{code}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +The response header transform middleware (`transformResponseMethod`) can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + +You only need to enable the middleware (set `enabled:true`) and then configure the details of headers to `add` and those to `remove`. + +For example: +```json {hl_lines=["39-50"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-response-method", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/status/200": { + "get": { + "operationId": "status/200get", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-response-method", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-response-method/", + "strip": true + } + }, + "middleware": { + "operations": { + "status/200get": { + "transformResponseHeaders": { + "enabled": true, + "remove": [ + "X-Static" + ], + "add": [ + { + "name": "X-Secret", + "value": "the-secret-key-is-secret" + } + ] + } + } + } + } + } +} +``` + +In this example the Response Header Transform middleware has been configured for HTTP `GET` requests to the `/status/200` endpoint. Any response received from the upstream service following a request to that endpoint will have the `X-Static` header removed and the `X-Secret` and `X-New` headers added (with values set to `the-secret-key-is-secret` and `another-header`). + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the endpoint-level response header transform. + +#### Combining API-level and Endpoint-level transforms + +If the example [API-level](#api-level-transform) and [endpoint-level](#endpoint-level-transform) transforms are applied to the same API, then the `X-Secret` header will be added (by the endpoint-level transform first) and then removed (by the API-level transform). Subsequently, the result of the two transforms for a call to `GET /status/200` would be to add four headers: +- `X-Request-ID` +- `X-User-ID` +- `X-Static` +- `X-New` + +### API Designer + +Adding and configuring the transforms to your API endpoints is easy when using the API Designer in the Tyk Dashboard, simply follow these steps: + +#### Adding an API-level transform + +From the **API Designer** on the **Settings** tab, after ensuring that you are in *edit* mode, toggle the switch to **Enable Transform response headers** in the **Middleware** section: +Tyk OAS API Designer showing API-level Response Header Transform + +Then select **NEW HEADER** as appropriate to add or remove a header from API responses. You can add or remove multiple headers by selecting **ADD HEADER** to add another to the list: +Configuring the API-level Response Header Transform in Tyk OAS API Designer + +#### Adding an endpoint level transform + +1. **Add an endpoint** + + From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + Tyk OAS API Designer showing no endpoints created + + Adding an endpoint to an API using the Tyk OAS API Designer + + Tyk OAS API Designer showing no middleware enabled on endpoint + +2. **Select the Response Header Transform middleware** + + Select **ADD MIDDLEWARE** and choose the **Response Header Transform** middleware from the *Add Middleware* screen. + + Adding the URL Rewrite middleware + +3. **Configure header transformation** + + Select **NEW HEADER** to configure a header to be added to or removed from the response, you can add multiple headers to either list by selecting **NEW HEADER** again. + + Configuring the rewrite rules for Advanced Triggers + Configuring the Response Header Transform + +4. **Save the API** + + Select **ADD MIDDLEWARE** to save the middleware configuration. Remember to select **SAVE API** to apply the changes. + +

Using Classic

+Tyk's [response header transform](/api-management/traffic-transformation/response-headers) middleware enables you to append or delete headers on responses received from the upstream service before sending them to the client. + +There are two options for this: +- API-level modification that is applied to all responses for the API +- endpoint-level modification that is applied only to responses from a specific endpoint + + + + + If both API-level and endpoint-level middleware are configured, the endpoint-level transformation will be applied first. + + + +When working with Tyk Classic APIs the transformation is configured in the Tyk Classic API Definition. You can do this via the Tyk Dashboard API or in the API Designer. + +If you want to use dynamic data from context variables, you must [enable](/api-management/traffic-transformation/request-context-variables#enabling-context-variables-for-use-with-tyk-classic-apis) context variables for the API to be able to access them from the response header transform middleware. + +If you're using the newer Tyk OAS APIs, then check out the [Tyk OAS](#response-headers-using-tyk-oas) page. + +If you're using Tyk Operator then check out the [configuring the Response Header Transform in Tyk Operator](#tyk-operator) section below. + +### API Definition + +The API-level and endpoint-level response header transforms have a common configuration but are configured in different sections of the API definition. + + +Prior to Tyk 5.3.0, there was an additional step to enable response header transforms (both API-level and endpoint-level). You would need to add the following to the Tyk Classic API definition: + +```json +{ + "response_processors":[ + {"name": "header_injector"} + ] +} +``` + +If using the Endpoint Designer in the Tyk Dashboard, this would be added automatically. + +We removed the need to configure the `response_processors` element in Tyk 5.3.0. + + + +

API-level transform

+To **append** headers to all responses from your API (i.e. for all endpoints) you must add a new `global_response_headers` object to the `versions` section of your API definition. This contains a list of key:value pairs, being the names and values of the headers to be added to responses. + +To **delete** headers from all responses from your API (i.e. for all endpoints), you must add a new `global_response_headers_remove` object to the `versions` section of the API definition. This contains a list of the names of existing headers to be removed from responses. + +For example: +```json {linenos=true, linenostart=1} expandable +{ + "version_data": { + "versions": { + "Default": { + "global_response_headers": { + "X-Static": "foobar", + "X-Request-ID":"$tyk_context.request_id", + "X-User-ID": "$tyk_meta.uid" + }, + "global_response_headers_remove": [ + "X-Secret" + ] + } + } + }, +} +``` + +This configuration will add three new headers to each response: +- `X-Static` with the value `foobar` +- `X-Request-ID` with a dynamic value taken from the `request_id` [context variable](/api-management/traffic-transformation/request-context-variables) +- `X-User-ID` with a dynamic value taken from the `uid` field in the [session metadata](/api-management/policies#what-is-a-session-metadata) + +It will also delete one header (if present) from each response: + - `X-Secret` + +

Endpoint-level transform

+To configure response header transformation for a specific endpoint you must add a new `transform_response_headers` object to the `extended_paths` section of your API definition. + +It has the following configuration: +- `path`: the endpoint path +- `method`: the endpoint HTTP method +- `delete_headers`: a list of the headers that should be deleted from the response +- `add_headers`: a list of headers, in key:value pairs, that should be added to the response + +For example: +```json {linenos=true, linenostart=1} expandable +{ + "transform_response_headers": [ + { + "path": "status/200", + "method": "GET", + "delete_headers": ["X-Static"], + "add_headers": [ + {"X-Secret": "the-secret-key-is-secret"}, + {"X-New": "another-header"} + ], + } + ] +} +``` + +In this example the Response Header Transform middleware has been configured for HTTP `GET` requests to the `/status/200` endpoint. Any response received from the upstream service following a request to that endpoint will have the `X-Static` header removed and the `X-Secret` and `X-New` headers added (with values set to `the-secret-key-is-secret` and `another-header`). + +#### Combining API-level and Endpoint-level transforms + +If the example [API-level](#api-level-transform) and [endpoint-level](#endpoint-level-transform) transforms are applied to the same API, then the `X-Secret` header will be added (by the endpoint-level transform first) and then removed (by the API-level transform). Subsequently, the result of the two transforms for a call to `GET /status/200` would be to add four headers: +- `X-Request-ID` +- `X-User-ID` +- `X-Static` +- `X-New` + +#### Fixing response headers that leak upstream server data + +A middleware called `header_transform` was added in Tyk 2.1 specfically to allow you to ensure that headers such as `Location` and `Link` reflect the outward facade of your API Gateway and also align with the expected response location to be terminated at the gateway, not the hidden upstream proxy. + +This is configured by adding a new `rev_proxy_header_cleanup` object to the `response_processors` section of your API definition. + +It has the following configuration: +- `headers`: a list of headers in the response that should be modified +- `target_host`: the value to which the listed headers should be updated + +For example: +```json expandable +{ + "response_processors": [ + { + "name": "header_transform", + "options": { + "rev_proxy_header_cleanup": { + "headers": ["Link", "Location"], + "target_host": "http://TykHost:TykPort" + } + } + } + ] +} +``` + +In this example, the `Link` and `Location` headers will be modified from the server-generated response, with the protocol, domain and port of the value set in `target_host`. + +This feature is rarely used and has not been implemented in the Tyk Dashboard UI, nor in the [Tyk OAS API](#response-headers-using-tyk-oas). + +### API Designer + +You can use the API Designer in the Tyk Dashboard to configure the response header transform middleware for your Tyk Classic API by following these steps. + +#### API-level transform + +Configuring the API-level response header transform middleware is very simple when using the Tyk Dashboard. + +In the Endpoint Designer you should select the **Global Version Settings** and ensure that you have selected the **Response Headers** tab: + +Configuring the API-level response header transform + +Note that you must click **ADD** to add a header to the list (for appending or deletion). + +#### Endpoint-level transform + +1. **Add an endpoint for the path and select the Header Transform plugin** + + From the **Endpoint Designer** add an endpoint that matches the path for which you want to perform the transformation. Select the **Modify Headers** plugin. + + Adding the Modify Headers plugin to an endpoint + +2. **Select the "Response" tab** + + This ensures that the transform will be applied to responses prior to them being sent to the client. + + Selecting the response header transform + +3. **Declare the headers to be modified** + + Select the headers to delete and insert using the provided fields. You need to click **ADD** to ensure they are added to the list. + + Configuring the response header transform + +4. **Save the API** + + Use the *save* or *create* buttons to save the changes and activate the middleware. + +### Tyk Operator + +The process for configuring a response header transform in Tyk Operator is similar to that defined in section configuring the Response Header Transform in the Tyk Classic API Definition. Tyk Operator allows you to configure a response header transformation for [all endpoints of an API](#tyk-operator-endpoint) or for a [specific API endpoint](#tyk-operator-api). + +

API-level transform

+The process of configuring transformation of response headers for a specific API in Tyk Operator is similar to that defined in section [API-level transform](#tyk-classic-api) for the Tyk Classic API definition. + +To **append** headers to all responses from your API (i.e. for all endpoints) you must add a new `global_response_headers` object to the `versions` section of your API definition. This contains a list of key:value pairs, being the names and values of the headers to be added to responses. + +To **delete** headers from all responses from your API (i.e. for all endpoints), you must add a new `global_response_headers_remove` object to the `versions` section of the API definition. This contains a list of the names of existing headers to be removed from responses. + +An example is listed below: + +```yaml {linenos=true, linenostart=1, hl_lines=["25-30"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-global-header +spec: + name: httpbin-global-header + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin-global-header + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + global_response_headers: + X-Static: foobar + X-Request-ID: "$tyk_context.request_id" + X-User-ID: "$tyk_meta.uid" + global_response_headers_remove: + - X-Secret +``` + +The example API Definition above configures an API to listen on path `/httpbin-global-header` and forwards requests upstream to http://httpbin.org. + +This configuration will add three new headers to each response: + +- `X-Static` with the value `foobar` +- `X-Request-ID` with a dynamic value taken from the `request_id` [context variable](/api-management/traffic-transformation/request-context-variables) +- `X-User-ID` with a dynamic value taken from the `uid` field in the [session metadata](/api-management/policies#what-is-a-session-metadata) + +It will also delete one header (if present) from each response: + +- `X-Secret` + + +

Endpoint-level transform

+The process of configuring a transformation of a response header for a specific endpoint in Tyk Operator is similar to that defined in section [endpoint-level transform](#tyk-classic-endpoint) for the Tyk Classic API definition. To configure a transformation of the response headers for a specific endpoint you must add a new `transform_response_headers` object to the `extended_paths` section of your API definition. + +In this example the Response Header Transform middleware (`transform_response_headers`) has been configured for HTTP `GET` requests to the `/xml` endpoint. Any response received from the upstream service following a request to that endpoint will have the `Content-Type` header added with a value set to `application/json`. + +#### Example + +```yaml {linenos=true, linenostart=1, hl_lines=["54-60"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-transform +spec: + name: httpbin-transform + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin-transform + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + transform: + - method: POST + path: /anything + template_data: + enable_session: false + input_type: json + template_mode: blob + # base64 encoded template + template_source: eyJiYXIiOiAie3suZm9vfX0ifQ== + transform_headers: + - delete_headers: + - "remove_this" + add_headers: + foo: bar + path: /anything + method: POST + transform_response: + - method: GET + path: /xml + template_data: + enable_session: false + input_type: xml + template_mode: blob + # base64 encoded template + template_source: e3sgLiB8IGpzb25NYXJzaGFsIH19 + transform_response_headers: + - method: GET + path: /xml + add_headers: + Content-Type: "application/json" + act_on: false + delete_headers: [] +``` + +#### Tyk Gateway < 5.3.0 Example + +If using Tyk Gateway < v5.3.0 then a `response_processor` object must be added to the API definition containing a `header_injector` item, as highlighted below: + +```yaml {linenos=true, linenostart=1, hl_lines=["17", "19", "57-63"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-transform +spec: + name: httpbin-transform + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin-transform + strip_listen_path: true + response_processors: + - name: response_body_transform + - name: header_injector + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + transform: + - method: POST + path: /anything + template_data: + enable_session: false + input_type: json + template_mode: blob + # base64 encoded template + template_source: eyJiYXIiOiAie3suZm9vfX0ifQ== + transform_headers: + - delete_headers: + - "remove_this" + add_headers: + foo: bar + path: /anything + method: POST + transform_response: + - method: GET + path: /xml + template_data: + enable_session: false + input_type: xml + template_mode: blob + # base64 encoded template + template_source: e3sgLiB8IGpzb25NYXJzaGFsIH19 + transform_response_headers: + - method: GET + path: /xml + add_headers: + Content-Type: "application/json" + act_on: false + delete_headers: [] +``` \ No newline at end of file diff --git a/api-management/traffic-transformation/virtual-endpoints.mdx b/api-management/traffic-transformation/virtual-endpoints.mdx new file mode 100644 index 00000000..99889464 --- /dev/null +++ b/api-management/traffic-transformation/virtual-endpoints.mdx @@ -0,0 +1,755 @@ +--- +title: "Virtual Endpoints" +'og:description': "How to configure Virtual Endpoints traffic transformation middleware in Tyk" +tags: ['Traffic Transformation', 'Virtual Endpoints'] +sidebarTitle: "Virtual Endpoints" +--- + +

Overview

+Tyk's Virtual Endpoint is a programmable middleware component that is invoked towards the end of the request processing chain. It can be enabled at the per-endpoint level and can perform complex interactions with your upstream service(s) that cannot be handled by one of the other middleware components. + +Virtual endpoint middleware provides a serverless compute function that allows for the execution of custom logic directly within the gateway itself, without the need to proxy the request to an upstream service. This functionality is particularly useful for a variety of use cases, including request transformation, aggregation of responses from multiple services, or implementing custom authentication mechanisms. + +The Virtual Endpoint is an extremely powerful feature that is unique to Tyk and provides exceptional flexibility to your APIs. + +### Use Cases + +#### Aggregating data from multiple services + +From a virtual endpoint, you can make calls out to other internal and upstream APIs. You can then aggregate and process the responses, returning a single response object to the originating client. This allows you to configure a single externally facing API to simplify interaction with multiple internal services, leaving the heavy lifting to Tyk rather than starting up an aggregation service within your stack. + +#### Enforcing custom policies + +Tyk provides a very flexible [middleware chain](/api-management/traffic-transformation#request-middleware-chain) where you can combine functions to implement the access controls you require to protect your upstream services. Of course, not all scenarios can be covered by Tyk's standard middleware functions, but you can use a virtual endpoint to apply whatever custom logic you require to optimize your API experience. + +#### Dynamic Routing + +With a virtual endpoint you can implement complex dynamic routing of requests made to a single external endpoint on to different upstream services. The flexibility of the virtual endpoint gives access to data within the request (including the key session) and also the ability to make calls to other APIs to make decisions on the routing of the request. It can operate as a super-powered [URL rewrite](/transform-traffic/url-rewriting#url-rewrite-middleware) middleware. + +### Working + +The virtual endpoint middleware provides a JavaScript engine that runs the custom code that you provide either inline within the API definition or in a source code file accessible to the Gateway. The JavaScript Virtual Machine (JSVM) provided in the middleware is a traditional ECMAScript5 compatible environment which does not offer the more expressive power of something like Node.js. + +The virtual endpoint terminates the request, so the JavaScript function must provide the response to be passed to the client. When a request hits a virtual endpoint, the JSVM executes the JavaScript code which can modify the request, make calls to other APIs or upstream services, process data, and ultimately determines the response returned to the client. + + + +You will need to enable Tyk's JavaScript Virtual Machine by setting `enable_jsvm` to `true` in your `tyk.conf` [file](/tyk-oss-gateway/configuration#enable_jsvm) for your virtual endpoints to work. + + + +### Scripting virtual endpoint functions + +The [middleware scripting guide](/api-management/plugins/javascript#using-javascript-with-tyk) provides guidance on writing JS functions for your virtual endpoints, including how to access key session data and custom attributes from the API definition. + +#### Function naming + +The virtual endpoint middleware will invoke a named function within the JS code that you provide (either inline or in a file). Both the filename and function name are configurable per endpoint, but note that function names must be unique across your API portfolio because all plugins run in the same virtual machine. This means that you can share a single function definition across multiple endpoints and APIs but you cannot have two different functions with the same name (this applies across all [JavaScript middleware components](/api-management/plugins/javascript#)). + +Inline mode is mainly used by the dashboard to make code injection easier on multiple node deployments. + +### Virtual endpoint library + +We have put together a [library](https://github.com/TykTechnologies/custom-plugins#virtual-endpoints) of JS functions that you could use in your virtual endpoints. We welcome submissions from the Tyk community so if you've created a function that you think would be useful to other users, please open an issue in the Github repository and we can discuss bringing it into the library. + + + +Virtual endpoints are not available in Tyk Cloud Classic. + + + +
+ +If you're using Tyk OAS APIs, then you can find details and examples of how to configure the virtual endpoint middleware [here](#virtual-endpoints-using-tyk-oas). + +If you're using Tyk Classic APIs, then you can find details and examples of how to configure the virtual endpoint middleware [here](#virtual-endpoints-using-classic). + +{/* proposed "summary box" to be shown graphically on each middleware page + # Virtual Endpoint middleware summary + - The Virtual Endpoint middleware is an optional stage in Tyk's API Request processing chain, sitting between the [TBC]() and [TBC]() middleware. + - The Virtual Endpoint middleware can be configured at the per-endpoint level within the API Definition and is supported by the API Designer within the Tyk Dashboard. */} + + +

Using Tyk OAS

+The [virtual endpoint](/api-management/traffic-transformation/virtual-endpoints) middleware provides a serverless compute function that allows for the execution of custom logic directly within the gateway itself, without the need to proxy the request to an upstream service. This functionality is particularly useful for a variety of use cases, including request transformation, aggregation of responses from multiple services, or implementing custom authentication mechanisms. + +The middleware is configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation). You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the legacy Tyk Classic APIs, then check out the [Tyk Classic](#virtual-endpoints-using-classic) page. + +### API Definition + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. Endpoint `paths` entries (and the associated `operationId`) can contain wildcards in the form of any string bracketed by curly braces, for example `/status/{code}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +The virtual endpoint middleware (`virtualEndpoint`) can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + +The `virtualEndpoint` object has the following configuration: + +- `enabled`: enable the middleware for the endpoint +- `functionName`: the name of the JavaScript function that will be executed when the virtual endpoint is triggered +- `body`: [optional] a `base64` encoded string containing the JavaScript code +- `path`: [optional] the relative path to the source file containing the JavaScript code +- `proxyOnError`: [optional, defaults to `false`] a boolean that determines the behavior of the gateway if an error occurs during the execution of the virtual endpoint's function; if set to `true` the request will be proxied to upstream if the function errors, if set to `false` the request will not be proxied and Tyk will return an error response +- `requireSession`: [optional defaults to `false`] a boolean that indicates whether the virtual endpoint should have access to the session object; if `true` then the key session data will be provided to the function as the `session` variable + + + + + One of either `path` or `body` must be provided, depending on whether you are providing the JavaScript code in a file or inline within the API definition. If both are provided then `body` will take precedence. + + + +For example: + +```json {hl_lines=["39-50", "54-58"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-virtual-endpoint", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/anything": { + "get": { + "operationId": "anythingget", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-virtual-endpoint", + "state": { + "active": true, + "internal": false + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-virtual-endpoint/", + "strip": true + } + }, + "middleware": { + "global": { + "pluginConfig": { + "data": { + "enabled": true, + "value": { + "map": { + "key": 3 + }, + "num": 4, + "string": "example" + } + } + } + }, + "operations": { + "anythingget": { + "virtualEndpoint": { + "enabled": true, + "functionName": "myVirtualHandler", + "body": "ZnVuY3Rpb24gbXlWaXJ0dWFsSGFuZGxlciAocmVxdWVzdCwgc2Vzc2lvbiwgY29uZmlnKSB7ICAgICAgCiAgdmFyIHJlc3BvbnNlT2JqZWN0ID0gewogICAgQm9keTogIlZpcnR1YWwgRW5kcG9pbnQgIitjb25maWcuY29uZmlnX2RhdGEuc3RyaW5nLAogICAgSGVhZGVyczogewogICAgICAiZm9vLWhlYWRlciI6ICJiYXIiLAogICAgICAibWFwLWhlYWRlciI6IEpTT04uc3RyaW5naWZ5KGNvbmZpZy5jb25maWdfZGF0YS5tYXApLAogICAgICAic3RyaW5nLWhlYWRlciI6IGNvbmZpZy5jb25maWdfZGF0YS5zdHJpbmcsCiAgICAgICJudW0taGVhZGVyIjogSlNPTi5zdHJpbmdpZnkoY29uZmlnLmNvbmZpZ19kYXRhLm51bSkKICAgIH0sCiAgICBDb2RlOiAyMDAKICB9CiAgcmV0dXJuIFR5a0pzUmVzcG9uc2UocmVzcG9uc2VPYmplY3QsIHNlc3Npb24ubWV0YV9kYXRhKQp9" + } + } + } + } + } +} +``` + +In this example the virtual endpoint middleware has been configured for requests to the `GET /anything` endpoint. We have also configured the following custom attributes in the `pluginConfig` section of the API definition: + +```json expandable +{ + "map": { + "key": 3 + }, + "num": 4, + "string": "example" +} +``` + +The `body` field value is a `base64` encoded string containing this JavaScript code, which will be invoked by the virtual endpoint middleware: + +```js expandable +function myVirtualHandler (request, session, config) { + var responseObject = { + Body: "Virtual Endpoint "+config.config_data.string, + Headers: { + "foo-header": "bar", + "map-header": JSON.stringify(config.config_data.map), + "string-header": config.config_data.string, + "num-header": JSON.stringify(config.config_data.num) + }, + Code: 200 + } + return TykJsResponse(responseObject, session.meta_data) +} +``` + +A call to the `GET /anything` endpoint returns: + +```bash expandable +HTTP/1.1 200 OK +Date: Fri, 01 Mar 2024 12:14:36 GMT +Foo-Header: bar +Map-Header: {"key":3} +Num-Header: 4 +Server: tyk +String-Header: example +Content-Length: 24 +Content-Type: text/plain; charset=utf-8 + +Virtual Endpoint example +``` + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the virtual endpoint middleware. + +### API Designer + +Adding a Virtual Endpoint to your API endpoints is easy when using the API Designer in the Tyk Dashboard, simply follow these steps: + +1. **Add an endpoint** + + From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + + Tyk OAS API Designer showing no endpoints created + + Adding an endpoint to an API using the Tyk OAS API Designer + + Tyk OAS API Designer showing no middleware enabled on endpoint + +2. **Select the Virtual Endpoint middleware** + + Select **ADD MIDDLEWARE** and choose **Virtual Endpoint** from the *Add Middleware* screen. + + Adding the Virtual Endpoint middleware + +3. **Configure the middleware** + + Now you can provide either the path to a file containing the JavaScript function to be run by the middleare, or you can directly enter the JavaScript in the code editor. + + For both sources, you must provide the **function name** that should be called when the middleware executes. + + You can also optionally configure the behavior required if the function should return an error and also indicate to Tyk whether the virtual middleware requires access to the key session metadata. + + Configuring the Virtual Endpoint middleware + +4. **Save the API** + + Select **ADD MIDDLEWARE** to save the middleware configuration. Remember to select **SAVE API** to apply the changes. + +

Using Classic

+The [virtual endpoint](/api-management/traffic-transformation/virtual-endpoints) middleware provides a serverless compute function that allows for the execution of custom logic directly within the gateway itself, without the need to proxy the request to an upstream service. This functionality is particularly useful for a variety of use cases, including request transformation, aggregation of responses from multiple services, or implementing custom authentication mechanisms. + +This middleware is configured in the Tyk Classic API Definition. You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the newer Tyk OAS APIs, then check out the [Tyk OAS](#virtual-endpoints-using-tyk-oas) page. + +If you're using Tyk Operator then check out the [configuring the middleware in Tyk Operator](#tyk-operator) section below. + +### API Definition + +If you want to use Virtual Endpoints, you must [enable Tyk's JavaScript Virtual Machine](/tyk-oss-gateway/configuration#enable_jsvm) by setting `enable_jsvm` to `true` in your `tyk.conf` file. + +To enable the middleware you must add a new `virtual` object to the `extended_paths` section of your API definition. + +The `virtual` object has the following configuration: + +- `path`: the endpoint path +- `method`: the endpoint HTTP method +- `response_function_name`: this is the name of the JavaScript function that will be executed when the virtual endpoint is triggered +- `function_source_type`: instructs the middleware to look for the JavaScript code either in a `file` or in a base64 encoded `blob`; the actual file location (or base64 encoded code) is provided in `function_source_uri` +- `function_source_uri`: if `function_source_type` is set to `file`, this will be the relative path to the source file containing the JavaScript code; if `function_source_type` if set to `blob`, this will be a `base64` encoded string containing the JavaScript code +- `use_session`: a boolean that indicates whether the virtual endpoint should have access to the session object; if `true` then the key session data will be provided to the function as the `session` variable +- `proxy_on_error`: a boolean that determines the behavior of the gateway if an error occurs during the execution of the virtual endpoint's function; if set to `true` the request will be proxied to upstream if the function errors, if set to `false` the request will not be proxied and Tyk will return an error response + +For example: + +```json {linenos=true, linenostart=1} expandable +{ + "extended_paths": { + "virtual": [ + { + "response_function_name": "myUniqueFunctionName", + "function_source_type": "blob", + "function_source_uri": "ZnVuY3Rpb24gbXlVbmlxdWVGdW5jdGlvbk5hbWUocmVxdWVzdCwgc2Vzc2lvbiwgY29uZmlnKSB7CiB2YXIgcmVzcG9uc2VPYmplY3QgPSB7IAogIEJvZHk6ICJUSElTIElTIEEgVklSVFVBTCBSRVNQT05TRSIsIAogIENvZGU6IDIwMCAKIH0KIHJldHVybiBUeWtKc1Jlc3BvbnNlKHJlc3BvbnNlT2JqZWN0LCBzZXNzaW9uLm1ldGFfZGF0YSkKfQ==", + "path": "/anything", + "method": "GET", + "use_session": false, + "proxy_on_error": false + } + ] + } +} +``` + +In this example the Virtual Endpoint middleware has been configured for requests to the `GET /anything` endpoint. For any call made to this endpoint, Tyk will invoke the function `myUniqueFunctionName` that is `base64` encoded in the `function_source_uri` field. This virtual endpoint does not require access to the session data and will not proxy the request on to the upstream if there is an error when processing the `myUniqueFunctionName` function. + +Decoding the value in `function_source_uri` we can see that the JavaScript code is: + +```js {linenos=true, linenostart=1} expandable +function myUniqueFunctionName(request, session, config) { + var responseObject = { + Body: "THIS IS A VIRTUAL RESPONSE", + Code: 200 + } + return TykJsResponse(responseObject, session.meta_data) +} +``` + +This function will terminate the request without proxying it to the upstream returning `HTTP 200` as follows: + +```bash expandable +HTTP/1.1 200 OK +Date: Wed, 28 Feb 2024 20:52:30 GMT +Server: tyk +Content-Type: text/plain; charset=utf-8 +Content-Length: 26 + +THIS IS A VIRTUAL RESPONSE +``` + +If, however, we introduce an error to the JavaScript, such that Tyk fails to process the function, we will receive an `HTTP 500 Internal Server Error` as follows: + +```bash expandable +HTTP/1.1 500 Internal Server Error +Date: Wed, 28 Feb 2024 20:55:27 GMT +Server: tyk +Content-Type: application/json +Content-Length: 99 + +{ +"error": "Error during virtual endpoint execution. Contact Administrator for more details." +} +``` + +If we set `proxy_on_error` to `true` and keep the error in the Javascript, the request will be forwarded to the upstream and Tyk will return the response received from that service. + +### API Designer + +You can use the API Designer in the Tyk Dashboard to configure a virtual endpoint for your Tyk Classic API by following these steps. + +1. **Add an endpoint for the path and select the plugin** + + From the **Endpoint Designer** add an endpoint that matches the path for which you want to trigger the virtual endpoint. Select the **Virtual Endpoint** plugin. + + Selecting the middleware + +2. **Configure the middleware** + + Once you have selected the virtual endpoint middleware for the endpoint, you need to supply: + + - JS function to call + - Source type (`file` or `inline`) + + If you select source type `file` you must provide the path to the file: + Configuring file based JS code + + If you select `inline` you can enter the JavaScript code in the Code Editor window. + Configuring inline JS code + +3. **Save the API** + + Use the *save* or *create* buttons to save the changes and activate the Virtual Endpoint middleware. + + + + + The Tyk Classic API Designer does not provide options to configure `use_session` or `proxy_on_error`, but you can do this from the Raw Definition editor. + + + +### Tyk Operator + +The process for configuring a virtual endpoint using Tyk Operator is similar to that explained in configuring the middleware in the Tyk Classic API Definition + +The example API Definition below configures an API to listen on path `/httpbin` and forwards requests upstream to `http://httpbin.org`. The Virtual Endpoint middleware has been configured for requests to the `GET /virtual` endpoint. For any call made to this endpoint, Tyk will invoke the function `myVirtualHandler` that is base64 encoded in the `function_source_uri` field. This virtual endpoint does not require access to the session data and will not proxy the request on to the upstream if there is an error when processing the `myVirtualHandler` function. + +```yaml {linenos=true, linenostart=1, hl_lines=["23-35"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: test-config-data-test +spec: + name: test-config-data-test + protocol: http + proxy: + listen_path: /httpbin/ + target_url: http://httpbin.org + strip_listen_path: true + active: true + use_keyless: true + enable_context_vars: true + version_data: + default_version: Default + not_versioned: false + versions: + Default: + name: Default + use_extended_paths: true + extended_paths: + virtual: + - function_source_type: blob + response_function_name: myVirtualHandler + function_source_uri: "ZnVuY3Rpb24gbXlWaXJ0dWFsSGFuZGxlciAocmVxdWVzdCwgc2Vzc2lvbiwgY29uZmlnKSB7ICAgICAgCiAgdmFyIHJlc3BvbnNlT2JqZWN0ID0gewogICAgQm9keTogIlRISVMgSVMgQSAgVklSVFVBTCBSRVNQT05TRSIsCiAgICBIZWFkZXJzOiB7CiAgICAgICJmb28taGVhZGVyIjogImJhciIsCiAgICAgICJtYXAtaGVhZGVyIjogSlNPTi5zdHJpbmdpZnkoY29uZmlnLmNvbmZpZ19kYXRhLm1hcCksCiAgICAgICJzdHJpbmctaGVhZGVyIjogY29uZmlnLmNvbmZpZ19kYXRhLnN0cmluZywKICAgICAgIm51bS1oZWFkZXIiOiBKU09OLnN0cmluZ2lmeShjb25maWcuY29uZmlnX2RhdGEubnVtKQogICAgfSwKICAgICAgQ29kZTogMjAwCiAgfQogIHJldHVybiBUeWtKc1Jlc3BvbnNlKHJlc3BvbnNlT2JqZWN0LCBzZXNzaW9uLm1ldGFfZGF0YSkKfQ==" + path: /virtual + method: GET + use_session: false + proxy_on_error: false + config_data: + string: "string" + map: + key: 3 + num: 4 +``` + +Decoding the value in `function_source_uri` we can see that the JavaScript code is: + +```javascript expandable +function myVirtualHandler (request, session, config) { + var responseObject = { + Body: "THIS IS A VIRTUAL RESPONSE", + Headers: { + "foo-header": "bar", + "map-header": JSON.stringify(config.config_data.map), + "string-header": config.config_data.string, + "num-header": JSON.stringify(config.config_data.num) + }, + Code: 200 + } + return TykJsResponse(responseObject, session.meta_data) +} +``` + +This function will terminate the request without proxying it to the upstream, returning HTTP 200 as follows: + +```bash expandable +HTTP/1.1 200 OK +Date: Wed, 14 Aug 2024 15:37:46 GMT +Foo-Header: bar +Map-Header: {"key":3} +Num-Header: 4 +Server: tyk +String-Header: string +Content-Length: 27 +Content-Type: text/plain; charset=utf-8 + +THIS IS A VIRTUAL RESPONSE +``` + +If, however, we introduce an error to the JavaScript, such that Tyk fails to process the function, we will receive an HTTP 500 Internal Server Error as follows: + +```bash expandable +HTTP/1.1 500 Internal Server Error +Date: Wed, 14 Aug 2024 15:37:46 GMT +Server: tyk +Content-Type: application/json +Content-Length: 99 + +{ +"error": "Error during virtual endpoint execution. Contact Administrator for more details." +} +``` + +If we set `proxy_on_error` to `true` and keep the error in the Javascript, the request will be forwarded to the upstream and Tyk will return the response received from that service. + +## Examples + +### Accessing Tyk data objects + +In this example, we demonstrate how you can access different [external Tyk objects](/api-management/plugins/javascript#accessing-external-and-dynamic-data) (API request, session key, API definition). + +1. Enable the Virtual Endpoint middleware on an endpoint of your API and paste this JavaScript into the API Designer (or save in a file and reference it from the middleware config): + +```javascript expandable +function myFirstVirtualHandler (request, session, config) { + log("Virtual Test running") + + log("Request Body: " + request.Body) + log("Session: " + JSON.stringify(session.allowance)) + log("Config: " + JSON.stringify(config.APIID)) + log("param-1: " + request.Params["param1"]) // case sensitive + log("auth Header: " + request.Headers["Authorization"]) // case sensitive + + var responseObject = { + Body: "VIRTUAL ENDPOINT EXAMPLE #1", + Headers: { + "x-test": "virtual-header", + "x-test-2": "virtual-header-2" + }, + Code: 200 + } + + return TykJsResponse(responseObject, session.meta_data) +} +log("Virtual Test initialised") +``` + +2. Make a call to your API endpoint passing a request body, a value in the `Authorization` header and a query parameter `param1`. + +3. The virtual endpoint will terminate the request and return this response: + +```bash expandable +HTTP/1.1 200 OK +Date: Thu, 29 Feb 2024 17:39:00 GMT +Server: tyk +X-Test: virtual-header +X-Test-2: virtual-header-2 +Content-Length: 27 +Content-Type: text/plain; charset=utf-8 + +VIRTUAL ENDPOINT EXAMPLE #1 +``` + +4. The gateway logs will include: + +```text expandable +time="" level=info msg="Virtual Test running" prefix=jsvm type=log-msg +time="" level=info msg="Request Body: " prefix=jsvm type=log-msg +time="" level=info msg="Session: " prefix=jsvm type=log-msg +time="" level=info msg="Config: " prefix=jsvm type=log-msg +time="" level=info msg="param-1: " prefix=jsvm type=log-msg +time="" level=info msg="auth Header: " prefix=jsvm type=log-msg +``` + +### Accessing custom attributes in the API Definition + +You can add [custom attributes](/api-management/plugins/javascript#adding-custom-attributes-to-the-api-definition) to the API definition and access these from within your Virtual Endpoint. + +1. Add the following custom attributes to your API definition: + +```json expandable +{ + "string": "string", + "map": { + " key": 3 + }, + "num": 4 +} +``` + +2. Enable the Virtual Endpoint middleware on an endpoint of your API and paste this JavaScript into the API Designer (or save in a file and reference it from the middleware config): + +```js expandable +function mySecondVirtualHandler (request, session, config) { + var responseObject = { + Body: "VIRTUAL ENDPOINT EXAMPLE #2", + Headers: { + "foo-header": "bar", + "map-header": JSON.stringify(config.config_data.map), + "string-header": config.config_data.string, + "num-header": JSON.stringify(config.config_data.num) + }, + Code: 200 + } + return TykJsResponse(responseObject, session.meta_data) +} +``` + +3. Make a call to your API endpoint. + +4. The virtual endpoint will terminate the request and return this response: + +```bash expandable +HTTP/1.1 200 OK +Date: Thu, 29 Feb 2024 17:29:12 GMT +Foo-Header: bar +Map-Header: {" key":3} +Num-Header: 4 +Server: tyk +String-Header: string +Content-Length: 26 +Content-Type: text/plain; charset=utf-8 + +VIRTUAL ENDPOINT EXAMPLE #2 +``` + +### Advanced example + +In this example, every line in the script gives an example of a functionality usage, including: + +- how to get form param +- how to get to a specific key inside a JSON variable +- the structure of the request object +- using `TykMakeHttpRequest` to make an HTTP request from within the virtual endpoint, and the json it returns - `.Code` and `.Body`. + +```js expandable +function myVirtualHandlerGetHeaders (request, session, config) { + rawlog("Virtual Test running") + + //Usage examples: + log("Request Session: " + JSON.stringify(session)) + log("API Config:" + JSON.stringify(config)) + + log("Request object: " + JSON.stringify(request)) + log("Request Body: " + JSON.stringify(request.Body)) + log("Request Headers:" + JSON.stringify(request.Headers)) + log("param-1:" + request.Params["param1"]) + + log("Request header type:" + typeof JSON.stringify(request.Headers)) + log("Request header:" + JSON.stringify(request.Headers.Location)) + + + //Make api call to upstream target + newRequest = { + "Method": "GET", + "Body": "", + "Headers": {"location":JSON.stringify(request.Headers.Location)}, + "Domain": "http://httpbin.org", + "Resource": "/headers", + "FormData": {} + }; + rawlog("--- before get to upstream ---") + response = TykMakeHttpRequest(JSON.stringify(newRequest)); + rawlog("--- After get to upstream ---") + log("response type: " + typeof response); + log("response: " + response); + usableResponse = JSON.parse(response); + var bodyObject = JSON.parse(usableResponse.Body); + + var responseObject = { + //Body: "THIS IS A VIRTUAL RESPONSE", + Body: "yo yo", + Headers: { + "test": "virtual", + "test-2": "virtual", + "location" : bodyObject.headers.Location + }, + Code: usableResponse.Code + } + + rawlog("Virtual Test ended") + return TykJsResponse(responseObject, session.meta_data) +} +``` + +### Running the Advanced example + +You can find a Tyk Classic API definition [here](https://gist.github.com/letzya/5b5edb3f9f59ab8e0c3c614219c40747) that includes the advanced example, with the JS encoded `inline` within the middleware config for the `GET /headers` endpoint. + +Create a new Tyk Classic API using that API definition and then run the following command to send a request to the API endpoint: + +```bash +curl http://tyk-gateway:8080/testvirtualendpoint2/headers -H "location: /get" -v +``` + +This should return the following: + +```bash expandable +Trying 127.0.0.1... +TCP_NODELAY set +Connected to tyk-gateway (127.0.0.1) port 8080 (#0) +GET /testvirtualendpoint2/headers HTTP/1.1 +Host: tyk-gateway:8080 +User-Agent: curl/7.54.0 +Accept: */* +location: /get + +HTTP/1.1 200 OK +Date: Fri, 08 Jun 2018 21:53:57 GMT +**Location: /get** +Server: tyk +Test: virtual +Test-2: virtual +Content-Length: 5 +Content-Type: text/plain; charset=utf-8 + +Connection #0 to host tyk-gateway left intact +yo yo +``` + +### Checking the Tyk Gateway Logs + +The `log` and `rawlog` commands in the JS function write to the Tyk Gateway logs. If you check the logs you should see the following: + +```text expandable +[Jun 13 14:45:21] DEBUG jsvm: Running: myVirtualHandlerGetHeaders +Virtual Test running +[Jun 13 14:45:21] INFO jsvm-logmsg: Request Session: {"access_rights":null,"alias":"","allowance":0,"apply_policies":null,"apply_policy_id":"","basic_auth_data":{"hash_type":"","password":""},"certificate":"","data_expires":0,"enable_detail_recording":false,"expires":0,"hmac_enabled":false,"hmac_string":"","id_extractor_deadline":0,"is_inactive":false,"jwt_data":{"secret":""},"last_check":0,"last_updated":"","meta_data":null,"monitor":{"trigger_limits":null},"oauth_client_id":"","oauth_keys":null,"org_id":"","per":0,"quota_max":0,"quota_remaining":0,"quota_renewal_rate":0,"quota_renews":0,"rate":0,"session_lifetime":0,"tags":null} type=log-msg +[Jun 13 14:45:21] INFO jsvm-logmsg: API Config:{"APIID":"57d72796c5de45e649f22da390d7df43","OrgID":"5afad3a0de0dc60001ffdd07","config_data":{"bar":{"y":3},"foo":4}} type=log-msg +[Jun 13 14:45:21] INFO jsvm-logmsg: Request object: {"Body":"","Headers":{"Accept":["*/*"],"Location":["/get"],"User-Agent":["curl/7.54.0"]},"Params":{"param1":["I-am-param-1"]},"URL":"/testvirtualendpoint2/headers"} type=log-msg +[Jun 13 14:45:21] INFO jsvm-logmsg: Request Body: "" type=log-msg +[Jun 13 14:45:21] INFO jsvm-logmsg: Request Headers:{"Accept":["*/*"],"Location":["/get"],"User-Agent":["curl/7.54.0"]} type=log-msg +[Jun 13 14:45:21] INFO jsvm-logmsg: param-1:I-am-param-1 type=log-msg +[Jun 13 14:45:21] INFO jsvm-logmsg: Request header type:[object Object] type=log-msg +[Jun 13 14:45:21] INFO jsvm-logmsg: Request header: ["/get"] type=log-msg +[Jun 13 14:45:21] INFO jsvm-logmsg: Request location type: object type=log-msg +[Jun 13 14:45:21] INFO jsvm-logmsg: Request location type: string type=log-msg +[Jun 13 14:45:21] INFO jsvm-logmsg: Request location: /get type=log-msg +--- before get to upstream --- +--- After get to upstream --- +[Jun 13 14:45:22] INFO jsvm-logmsg: response type: string type=log-msg +[Jun 13 14:45:22] INFO jsvm-logmsg: response: {"Code":200,"Body":"{\"headers\":{\"Accept-Encoding\":\"gzip\",\"Connection\":\"close\",\"Host\":\"httpbin.org\",\"Location\":\"/get\",\"User-Agent\":\"Go-http-client/1.1\"}}\n","Headers":{"Access-Control-Allow-Credentials":["true"],"Access-Control-Allow-Origin":["*"],"Content-Length":["133"],"Content-Type":["application/json"],"Date":["Wed, 13 Jun 2018 13:45:21 GMT"],"Server":["gunicorn/19.8.1"],"Via":["1.1 vegur"]},"code":200,"body":"{\"headers\":{\"Accept-Encoding\":\"gzip\",\"Connection\":\"close\",\"Host\":\"httpbin.org\",\"Location\":\"/get\",\"User-Agent\":\"Go-http-client/1.1\"}}\n","headers":{"Access-Control-Allow-Credentials":["true"],"Access-Control-Allow-Origin":["*"],"Content-Length":["133"],"Content-Type":["application/json"],"Date":["Wed, 13 Jun 2018 13:45:21 GMT"],"Server":["gunicorn/19.8.1"],"Via":["1.1 vegur"]}} type=log-msg +Virtual Test ended +[Jun 13 14:45:22] DEBUG JSVM Virtual Endpoint execution took: (ns) 191031553 +``` + +### Aggregating upstream calls using batch processing + +One of the most common use cases for virtual endpoints is to provide some form of aggregate data to your users, combining the responses from multiple upstream service calls. This virtual endpoint function will do just that using the batch processing function from the [JavaScript API](/api-management/plugins/javascript#javascript-api) + +```js expandable +function batchTest(request, session, config) { + // Set up a response object + var response = { + Body: "", + Headers: { + "test": "virtual-header-1", + "test-2": "virtual-header-2", + "content-type": "application/json" + }, + Code: 200 + } + + // Batch request + var batch = { + "requests": [ + { + "method": "GET", + "headers": { + "x-tyk-test": "1", + "x-tyk-version": "1.2", + "authorization": "1dbc83b9c431649d7698faa9797e2900f" + }, + "body": "", + "relative_url": "http://httpbin.org/get" + }, + { + "method": "GET", + "headers": {}, + "body": "", + "relative_url": "http://httpbin.org/user-agent" + } + ], + "suppress_parallel_execution": false + } + + log("[Virtual Test] Making Upstream Batch Request") + var newBody = TykBatchRequest(JSON.stringify(batch)) + + // We know that the requests return JSON in their body, lets flatten it + var asJS = JSON.parse(newBody) + for (var i in asJS) { + asJS[i].body = JSON.parse(asJS[i].body) + } + + // We need to send a string object back to Tyk to embed in the response + response.Body = JSON.stringify(asJS) + + return TykJsResponse(response, session.meta_data) + +} +log("Batch Test initialised") +``` + diff --git a/api-management/troubleshooting-debugging.mdx b/api-management/troubleshooting-debugging.mdx new file mode 100644 index 00000000..2b46b700 --- /dev/null +++ b/api-management/troubleshooting-debugging.mdx @@ -0,0 +1,1647 @@ +--- +title: "Troubleshooting and Debugging" +'og:description': "Tyk troubleshooting and debugging gateway, streams, pump, dashboard" +tags: ['troubleshooting', 'debugging', 'Open Source', 'Self-Managed', 'Tyk Cloud', 'API Gateway'] +sidebarTitle: "Troubleshooting" +--- + +## Gateway + +1. ##### Users receive 499 error in the Gateway + + **Cause** + + The Gateway receives closed client responses from the upstream client. There are number of different configuration settings that could bring about this issue. + + **Solution** + + For a standard web app, used by standard HTTP clients 499 errors are not a problem. + ​ + + However, in some specific cases, depending on the service you provide, your clients can have their own fixed constraints. + For example, if you are building an API used by IoT devices, and those devices internally have a strict 2 second timeout for HTTP calls and your service responding with > 2 seconds. In this case a lot of 499 errors may mean that a lot of clients are malfunctioning, and you should investigate this behavior. + + On the other hand, sometimes a client closing the connection before reading the server response is expected functionality. Taking the same example as above, you may have some IoT sensor, which just pushes data to your servers in "fire and forgot" mode, and does not care about the server response. In this case a 499 error is completely expected behavior. + + +2. ##### Users receive 502 error in the Gateway + + **Cause** + + The Gateway received an invalid response from the upstream server. There are number of different configuration settings that could bring about this issue. + + **Solution** + + Try using the following settings in your tyk.conf file: + + ```{.copyWrapper} expandable + enable_detailed_recording: false, + enable_jsvm: false, + ``` + + + And the following key-value pairs should be set in the relevant API definition: + + ```{.copyWrapper} + proxy.service_discovery.use_nested_query = false + proxy.service_discovery.use_target_list = true + proxy.service_discovery.endpoint_returns_list = true + proxy.service_discovery.data_path = ""Address” + proxy.service_discovery.port_data_path = β€œServicePort"" + ``` + + See [Tyk Gateway configuration](/tyk-oss-gateway/configuration) and [Tyk Gateway API](/api-management/gateway-config-tyk-classic) for further information regarding API definition settings. + +3. ##### Gateway proxy error "context canceled" + + In some cases you can see "proxy error: context canceled" error message in the Gateway logs. + The error itself means that the connection was closed unexpectedly. + It can happen for various reasons, and in some cases it is totally fine: for example client can have unstable mobile internet. + + When it happens on the high load, it can be a lot of different reasons. + For example your OS is running out of system limits, like number of opened sockets, and to validate it, you need to try your system limits. + See [this guide](/planning-for-production). + + Additionally, it can be CPU bottleneck: you can't process more than your machine can do. + And note that it is not only about the actual utilization %, it is also about context switches it has to do. + E.g. having one job which consume 100% of your CPU/cores vs having a few thousands jobs, causing CPU constantly switch between them. + Such problems cause internal request processing queues, which cause latency growth (highly recommend measure it). + And in some cases latency can grow so big, that some clients can just disconnect/timeout because of it. + + Additionally, highly recommend read the following blog post https://tyk.io/blog/performance-tuning-your-tyk-api-gateway/. + For example, you can trade memory for performance, and context switch reduction by tuning garbage collector to run less frequently: see `Tuning Tyk’s Garbage Collector` section. + + + Also note that it is not Tyk or Golang specific. + The problem described above will happen with any webserver on high scale. + So in general if you see a lot of "context" errors on high load, use it as a sign that the process is really struggling with the given load, and you need scale it up, either vertically or horizontally. + +4. ##### Invalid memory address or nil pointer dereference error + + **Cause** + + There are a number of reasons, most commonly, an API may have been configured incorrectly in some way (for instance, it may have been set up without an organization). The error itself is a specific to Go language which Tyk was written in and could also suggest that alterations made to the code by the user could also be the culprit. + + **Solution** + + Make sure that API definitions are set up correctly. Information on how to do this with the Tyk Gateway API can be found in the following links: + + * [API Definition Object Details](/api-management/gateway-config-tyk-classic) + * [API Management](/tyk-gateway-api) + +5. ##### Users receive this error message when attempting to make API calls to an existing key. + + **Cause** + When the token was created, most probably it was configured without the `meta_data` key. + + **Solution** + The user will need to add the key-value pair `meta_data: {}` to their key as per the [Tyk Gateway REST API Documentation](/tyk-gateway-api). + +6. ##### There was a problem proxying the request + + **Cause** + + The upstream server may have returned an empty response or cut the response off early so it was unable to complete the proxying process. A proxy error means actual connectivity issues between Tyk and the target host (i.e., a connection-level issue with the downstream server misbehaving for some reason). + + Expired TLS certificates may also cause issues. + + **Solution** + + Users are advised to upgrade to the latest versions of any Tyk packages at their earliest convenience as a patch was released to resolve this issue. Packages are available to download from [Packagecloud.io][1]. See [Upgrading Tyk](/developer-support/upgrading) for details on upgrading to the latest version. It may also be worth checking if any TLS certificates associated with the domain have expired. + + [1]: https://packagecloud.io/tyk + +6. ##### Tyk Gateway Profiling + + In some cases, to identify tricky issues like concurrency or memory related issues, it may be required to get information about the Gateway process runtime. For example, memory or CPU usage details. + The Tyk Gateway is built using Go, and inherits its powerful profiling tools, specifically Google's [`pprof`](https://github.com/google/pprof/). + + The Tyk Gateway can generate various profiles in the `pprof` supported format, which you can analyze by yourself, using the `go tool pprof` command, or you can send the profiles to our support team for analysis. + + There are two way to get profiles: + + 1. Running the process with flags mentioned below which will gather information about the running process for the first 30 seconds, and will generate files containing profiling info: + + * `--memprofile` - memory profile, generates `tyk.mprof` file + * `--cpuprofile` - CPU usage profile, generates `tyk.prof` file + * `--blockprofile` - Blocking profile, generates `tyk.blockprof` file + * `--mutexprofile` - Mutex profile, generates `tyk.mutexprof` file + + 2. Running with the `--httpprofile` flag, or set `enable_http_profiler` to `true` in tyk.conf, which will run a special `/debug/pprof/` public web page, containing dynamic information about the running process, and where you can download various profiles: + + * goroutine - stack traces of all current goroutines + * heap - a sampling of all heap allocations + * threadcreate - stack traces that led to the creation of new OS threads + * block - stack traces that led to blocking on synchronisation primitives + * mutex - stack traces of holders of contended mutexes + +
7. Support Information
+ When contacting support, you may be asked to supply extra information and supply log files, etc, so we can quickly handle your request. Questions may include: + + * "Can you send us your log files" + * "Can you change the logging detail level" + * "What version of Tyk are you on" + * "What profiling information can I get" + + + This page will let you know how to get the above info to us. + + **Log Files** + + **Where do I find my log files?** + + The Gateway will log its output to `stderr` and `stdout`. In a typical installation, these will be handled or redirected by the service manager running the process, and depending on the Linux distribution, will either be output to `/var/log/` or `/var/log/upstart`. + + Tyk will try to output structured logs, and so will include context data around request errors where possible. + + **How do I increase Logging Verbosity?** + + You can set the logging verbosity in two ways: + + 1. Via an Environment Variable to affect all Tyk components + 2. Just for the Gateway via your `tyk.conf` config file + + **Setting via Environment Variable** + + The environment variable is `TYK_LOGLEVEL`. + + By default, the setting is `info`. You also have the following options: + + * `debug` + * `warn` + * `error` + + You will be advised by support which setting to change the logging level to. + + **For the Gateway** + + You can set the logging level in your `tyk.conf` by adding the following: + + ```{.copyWrapper} + "log_level": "info", + ``` + + By default, the setting is `info`. You also have the following options: + + * `debug` + * `warn` + * `error` + + You will be advised by support which setting to change the logging level to. + + **Tyk Version** + + For support requests it is beneficial to provide more information about your Gateway build. These pinpoint the exact Gateway build that is in use. + + - Since Gateway version `5.0.8` or `5.2.3` you can inspect detailed build information by running `tyk version`. The information also includes the Go version it was built with, the operating system and architecture. + + - If you're running an an older version than the above, `tyk --version` prints out the release version for your Gateway binary. + + The binary is installed in `/opt/tyk-gateway/tyk` by default. If your binary is not available in your `PATH` environment, invoke it from there. + + **Profile Information** + + You can provide various profile information for us in [pprof format](https://github.com/google/pprof/). See [Gateway Profiling](#tyk-gateway-profiling) for more details. + +8. ##### API definition URL case sensitive + + For security reasons Tyk lowercases the URL before performing any pattern matching. + +9. ##### Gateway detected 0 APIs + + Tyk Gateway is not able to get API configs from the Tyk Portal. + If you configured your Gateway to be segmented, you would also need to assign tags and you must also tag the APIs in the API Designer to make sure that they load. + + * In the Pro edition that is a connectivity or misconfiguration issue + * In the Community edition, since you are not using the Dashboard we + assume that you use file-based APIs , so in this case it's because + API definition files are missing. + +10. ##### How to import existing keys into Tyk CE + + You can use an API to import existing keys that were not created in Tyk into Tyk's Gateway. + This doc explains how to do that with the Gateway's APIs directly. + + This example uses standard `authorization` header authentication, and assumes that the Gateway is located at `127.0.0.1:8080` and the Tyk secret is `352d20ee67be67f6340b4c0605b044b7` - update these as necessary to match your environment. + + To import a key called `mycustomkey`, save the JSON contents as `token.json` (see example below), then run the following Curl command: + + The Example `token.json` file + + ```{.json} + { + "allowance": 1000, + "rate": 1000, + "per": 60, + "expires": -1, + "quota_max": -1, + "quota_renews": 1406121006, + "quota_remaining": 0, + "quota_renewal_rate": 60, + "access_rights": { + "3": { + "api_name": "Tyk Test API", + "api_id": "3" + } + }, + "org_id": "53ac07777cbb8c2d53000002", + "basic_auth_data": { + "password": "", + "hash_type": "" + }, + "hmac_enabled": false, + "hmac_string": "", + "is_inactive": false, + "apply_policy_id": "", + "apply_policies": [ + "59672779fa4387000129507d", + "53222349fa4387004324324e", + "543534s9fa4387004324324d" + ], + "monitor": { + "trigger_limits": [] + } + } + ``` + + The import of the key to Tyk: + + ``` + curl http://127.0.0.1:8080/tyk/keys/mycustomkey -H 'x-tyk-authorization: 352d20ee67be67f6340b4c0605b044b7' -H 'Content-Type: application/json' -d @token.json + ``` + + Test the key after the import: + + ``` + curl http://127.0.0.1:8080/quickstart/headers -H 'Authorization: mycustomkey' + ``` + + See also the Keys section of the [Tyk Gateway API documentation](/tyk-gateway-api). + + +10. ##### Redis persistence using containers + + Use case: Keep my data persistent at Docker container restart + + The Multi-Cloud Redis container is ephemeral, it isn't configured for persistence because it would very quickly get very large (Docker containers in general should really be considered as ephemeral). + + If using Redis with Multi-Cloud we strongly recommend using an external Redis database. + + There are no settings for Redis available via environment variable, you would need to mount a new `redis.conf` into the container to customize the configuration, but again, we don't recommend it. + +11. ##### DRL not ready, skipping this notification + + **Description** + + You see the following `Log Warning:` + + `DRL not ready, skipping this notification` + + + **Cause** + + There can be a couple of reasons for seeing this error about the [Distributed Rate Limiter](/api-management/rate-limit#rate-limiting-layers): + + 1. When you have more than one installation of the Gateway with one configured to use DRL, and others not. + 2. When the Gateway is started and the DRL receives an event before it has finished initialising. + + **Solution** + + For cause **1**, ensure that all instances of the Tyk Gateway are configured to use DRL. + + For cause **2**, the error will disappear when the DRL has initialised. + +12. ##### "Index out of rangeβ€œ error in logs + + **Description** + + Redis cluster users receive the aforementioned error message in their logs. The log stack may resemble the following: + + ``` + 2016/06/22 09:58:41 http: panic serving 10.0.0.1:37196: runtime error: index out of range + 2016/06/22 09:58:41 http: panic serving 10.0.0.1:37898: runtime error: index out of range + 2016/06/22 09:58:41 http: panic serving 10.0.0.1:38013: runtime error: index out of range + 2016/06/22 09:58:42 http: panic serving 10.0.0.1:39753: runtime error: index out of range + 2016/06/22 10:01:07 http: panic serving 10.0.0.1:34657: runtime error: invalid memory address or nil pointer dereference + 2016/06/22 10:01:07 http: panic serving 10.0.0.1:36801: runtime error: invalid memory address or nil pointer dereference + ``` + + **Cause** + + This is due to a bug that prevents the driver from picking up a random redis handle in single-instance connections such as pub/sub. The issue affects later patch releases of Tyk 2.2 and the first release of Tyk 2.3. + + **Solution** + + Users are advised to upgrade to the latest versions of any Tyk packages as a patch was released to resolve this issue. Packages are available to download from [Packagecloud.io](https://packagecloud.io/tyk) and further details on how to upgrade can be found [here](/developer-support/upgrading). + +13. ##### Hot restart a Tyk Gateway Process + + It is possible to hot-restart a Tyk Gateway process without dropping any connections. This can be useful if you need to load up a new configuration or change a configuration on a production server without losing any traffic. + + To hot-restart a Tyk Gateway process, you simply need to send a `SIGUSR2` signal to the process, for example: + + ```bash + > sudo kill -SIGUSR2 {gateway-pid} + ``` + + This will fork and load a new process, passing all open handles to the new server and wait to drain the old ones. + +14. ##### How to add Custom Certificates to Trusted Storage of Docker Images + + To add your custom Certificate Authority(CA) to your docker containers. You can mount your CA certificate directly into `/etc/ssl/certs` folder. + + Docker: + ```{.copyWrapper} + docker run -it tykio/tyk-gateway:latest \ + -v $(pwd)/myCA.pem:/etc/ssl/certs/myCA.pem + ``` + + Kubernetes - using Helm Chart and secrets: + ```yaml + extraVolumes: + - name: self-signed-ca + secret: + secretName: self-signed-ca-secret + extraVolumeMounts: + - name: self-signed-ca + mountPath: "/etc/ssl/certs/myCA.pem" + subPath: myCA.pem + ``` + +15. ##### How to change the logging output location + + It's not possible to segregate out the error locations in the `tyk.conf`, but you can modify the actual initialisation files to specify the log location, we supply initialisation scripts for `SysV`, `systemd` and `upstart`. + +16. ##### How to clear / invalidate API cache + + Use the REST API to clear the cache + + **OSS** + + ``` + DELETE /tyk/cache/{api-id} + ``` + + **Tyk Dashboard** + + ``` + DELETE /api/cache/{api-id} + ``` + +17. ##### How to find the Gateway logging output + + You are able to see a more detailed output in your Gateway log `/var/log` or `/var/log/upstart`. + +## Gateway Error Response Status Codes + +Tyk Gateway responses include HTTP status codes that follow the [HTTP status code standard](https://datatracker.ietf.org/doc/html/rfc9110). They have three digits that describe the result of the request and the semantics of the response. +The first digit defines the class of response as shown in the [list](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes) below: +- 1xx (Informational): The request was received, continuing process +- 2xx (Successful): The request was successfully received, understood, and accepted +- 3xx (Redirection): Further action needs to be taken in order to complete the request +- 4xx (Client Error): The request contains bad syntax or cannot be fulfilled +- 5xx (Server Error): The server failed to fulfill an apparently valid request + +Here we provide a list of all the error status codes (4xx and 5xx) that may be returned by the Tyk Gateway along with their corresponding messages and some guidance on the likely cause of the error. +Tyk supports [error templating](/api-management/gateway-events#error-templates), allowing you to configure the Gateway to return customised messages for certain HTTP error codes. + +We also support limited customisation of the error codes and messages returned by custom authentication middleware through the use of [override messages](/tyk-oss-gateway/configuration#override_messages). + +| Code | Text | Recommended action | +| :--- | :-------------------------------------------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 400 | Access to this API has been disallowed | Check if the key has access to the right API version or definition. Check if the authentication key used is still valid. Check if the certificate used for authentication is present. Check if the authentication key is created and present in the database. You can use Gateway Keys APIs for confirmation. Check if API definition is using JWT auth and if auth header key and or value is empty or missing.| +| 400 | API is not OAuth2 | Check if OAuth2 is integrated into the API by auth tokens or using Tyk OAuth flow. | +| 400 | Attempted access with malformed header | Values not in basic auth format or auth data not encoded correctly. | +| 400 | Authorization Field Missing | Check if the authorization field is missing. Check if the OAuth authorization field is missing. | +| 400 | Batch request creation failed, request structure malformed | Attempted to construct unsafe requests. Check if request structure is in correct format. | +| 400 | Batch request malformed | Attempted to decode request but failed. Check if request structure is in correct format. | +| 400 | Bearer token malformed | Check if the OAuth authorization field is malformed. | +| 400 | Body do not contain password or username | Check if body contains both password and username. If not, then insert the correct login credentials. | +| 400 | Cannot parse form. Form malformed | Attempted to revoke token but could not parse the request form. Check if the request form is malformed. | +| 400 | Content length is not a valid Integer | Check the value provided in the Content-Length field in the header. | +| 400 | Couldn’t decode instruction | Attempted to decode policy record from an update request. Check if the request body is malformed and is valid. | +| 400 | Couldn’t decode OAS object | Attempted to import OAS Tyk API but failed to retrieve object from request. Check if request body is valid. | +| 400 | Error API not migrated | The supplied API definition is in OAS format. Please use the Tyk native format for this API. | +| 400 | Failed to create key, keys must have at least one Access Rights record set | Attempted to create a key with master keys disabled in configurations. | +| 400 | Failed to remove the key | Failed to delete requested key. Make sure orgID and keyname are correct. | +| 400 | Health checks are not enabled for this node | Enable health checks for the gateway. | +| 400 | Key not authorized | Check if OAuth key is present. Check if the OAuth client is not deleted. Check if there is a valid policy associated with the key/token used. Check if the policy associated with the key is not expired or if the owner is valid. Check if JWT default policies exist. | +| 400 | Key cannot be used without a certificate | Check if key contains a certificate. If not, add a certificate to the key. | +| 400 | Key must be used with an existent certificate | Check if the certificate on the key exist within the system. | +| 400 | Missing parameter api_id | Check if API_ID is missing. If so, fill in the api_ID field with the correct value. | +| 400 | OAuth client doesn’t exist | Check if API_ID is missing. If so, fill in the api_ID field with the correct value. | +| 400 | OAuth client ID is empty | Check if OAuth client ID field is empty. If so, fill in with the correct client ID value. | +| 400 | OAuth is not enabled for this API | Check if OAuth is enabled for the API. | +| 400 | Policy access rights doesn’t contain API this OAuth client belongs to | Check if the policy rights contains the proper api_ID for the API. | +| 400 | Request apiID does not match that in Definition! For Update operations these must match | Attempted a PUT operation using different api_ID's. Make sure the api_ID's are the same. | +| 400 | Request field is missing | Check if the request field is missing. If so, fill in the request field. | +| 400 | Request ID does not match that in policy! For Update operations these must match | Attempted a PUT operation using different policy ID's. Make sure both policy ID's are the same. | +| 400 | Request is too large | The request body exceeds the configured size limit for the API endpoint. | +| 400 | Request with empty authorization header | Fill in authorization header for the request. | +| 400 | Spec field is missing | Attempted to trace a request but spec field is missing. Fill in the spec field. | +| 400 | The provided request is empty | Check if request in the GraphQL playground is correct. | +| 401 | Authorization Field Missing | Check if the authorization field is missing. Check if the OAuth authorization field is missing. | +| 401 | Header missing | Check if header field exist when making request. | +| 401 | Key has expired, please renew | Current key has expired. Please request for a new key. | +| 401 | OAuth Client Id Empty | Fill in the Client ID field. | +| 401 | OAuth Client Secret Empty | Client secret is empty. Insert the required client secret. | +| 401 | Request signature verification failed | Possible empty signature header or validation failed. | +| 401 | Wrong Password | Enter the correct password. Contact an administrator if further help is needed. | +| 403 | Access to this API has been disallowed | Request access to the API from an administrator. | +| 403 | Access to this resource has been disallowed | Request access to the resource from an administrator. | +| 403 | Attempted access with non-existent cert | Check if authentication certificate exist. | +| 403 | Attempted administrative access with invalid or missing key! | Check if there is correct security credentials of the Tyk API. | +| 403 | Certificate with SHA256 $certID not allowed | Certificate ID is nil or invalid. Please have a valid certificate. | +| 403 | Client authorize request in with invalid redirect URI | Check if Auth Redirect URI is malformed or use a valid redirect URI. | +| 403 | Client TLS certificate is required | Check if theres multiple APIs on the same domain with no certificates. | +| 403 | Certificate has expired | Please update the certificate with one that is currently valid and has not expired. | +| 403 | Depth limit exceeded | Exceeded the depth limit that has been applied. Check the key/policy global limits and quota section or the API limits and quota section. | +| 403 | Empty Signature Header | Fill in a signature for auth keys. | +| 403 | Empty Signature Path | Check if path for signature is empty. | +| 403 | Failed with 403 after $x-amount of requests over quota | Process request off thread with quota or process request live with rate limit or process request off thread with rate limit. | +| 403 | Found an empty user ID in predefined base field claim user_id | Request with valid JWT/RSA or signature/empty user_id/sub claim, or signature/no base field or no sub or no id claim. | +| 403 | GraphQL Depth Limit Exceeded | Exceeded the depth limit that has been applied. Check the key/policy global limits and quota section or the API limits and quota section. | +| 403 | Invalid Token | Check if JWT token is valid and not malformed. | +| 403 | Invalid Signature Header | Insert correct signature header value. | +| 403 | Invalid Signature Path | Make sure signature path is correct and valid. | +| 403 | Key is not active, please renew | Create a new key. | +| 403 | Key not authorised: Unexpected signing method | Invalid JWT signature, JWT access with non-existent key. | +| 403 | Key not authorised: OAuth client access was revoked | Check if OAuth client exists. | +| 403 | Key not authorised: no matching policy | Request with invalid policy in JWT, or checking session and identity for valid key for openID. | +| 403 | No matching policy found in scope claim | Check if scope is wrong for JWT request. | +| 403 | Quota Exceeded | Quota limit has been exceeded. Check quota limit settings. | +| 403 | Run Go-plugin auth failed | Used an invalid token for authentication. Please use a valid token to authenticate. | +| 403 | This API version does not seem to exist | Attempted to extract version data from a request. Version does not exist when loading version data. | +| 403 | This organisation access has been disabled, please contact your API administrator | Organisation session is inactive. Contact API administrator. | +| 403 | This organisation quota has been exceeded, please contact your API administrator | Organisation's quota limit has been exceeded. Contact API administrator. | +| 403 | This organisation rate limit has been exceeded, please contact your API administrator | Organisation's rate limit has been exceeded. Contact API administrator. | +| 403 | TLS: bad certificate | Check if the certificates exist and have valid ID's. | +| 403 | Version Information not found | Checking version data from request. No default version has been set or found. | +| 404 | API doesn’t exist | Checking if API exists when rotating OauthClient or if ApiSpec value is nil. | +| 404 | API for this refresh token not found | When invalidating OAuth refresh or if ApiSpec value is nil. | +| 404 | API ID not found | Check if API ID exists in the Gateway. | +| 404 | API not found | Check if API exists. | +| 404 | Bundle not found | No bundles found within the Gateway. | +| 404 | Certificate with given SHA256 fingerprint not found | No certificates exist in the certificate manager list. | +| 404 | Couldn't find organisation session in active API list | Attempted to update session object. However, spec for organisation is nil. Make sure to have the correct organisation ID. | +| 404 | Error getting oauth client | See if OAuth client id exists in the system. | +| 404 | Key not found | Failed to update hashed key. | +| 404 | No such organisation found in Active API list | Make sure organisation ID is correct. | +| 404 | OAuth client doesn’t exist | Attempted to retrieve APIs for OAuth or client ID. Client ID was not found | +| 404 | OAuth client ID not found | Check if OAuth client ID exist in storage. Check if OAuth tokens or client details are valid. Failed to retrieve OAuth client list. Failed to revoke OAuth client list. | +| 404 | Org not found | Could not retrieve record of org ID or failed to delete org keys. Spec for org is nil, make sure orgID value is correct | +| 404 | Policy not found | Could not retrieve policy data. Make sure policy ID is correct. | +| 404 | There is no such key found | Check if key is already deleted. Check if hashed key has been deleted already. | +| 404 | Version Does Not Exist | Check if version path is filled and correct. | +| 405 | Malformed request body | Attempted a POST request with a malformed request body. Make sure the request body is valid. | +| 405 | Method not supported | Attempting to add a method that is not supported by our system. | +| 411 | Content length is required for this request | You need to provide the `Content-Length` field in the request header. | +| 429 | API Rate Limit Exceeded | Check the rate of the requests on the API level. Check the rate of requests on the API key (Auth token, certs, etc). | +| 499 | Client closed request | Check if the client closed the TCP connection | +| 500 | Cache invalidation failed | Attempted to scan or delete the cache, which failed, causing cache invalidation to fail. | +| 500 | Can't detect loop target | Verify target API exsists. Check if URL scheme is "tyk://". Refer to 404 errors | +| 500 | Could not write key data | Failed to update hashed key. Make sure key name is valid. | +| 500 | Delete failed | Attempted to delete policy with invalid filename. Attempted to delete API with invalid filename. Attempted to delete OAuth Client with incorrect OAuth client ID. | +| 500 | Due to enabled service policy source, please use the Dashboard API | Attempted to add/update a policy and rejected due to Policysource=service. Please use the Dashboard API. | +| 500 | Due to enabled use_dp_app_configs, please use Dashboard API | When trying to import OAS, when Dashboard config is set to true. Please use Dashboard API. | +| 500 | Error writing to key store | Attempted to update session with a new session. Make sure orgID is correct. | +| 500 | Failed to create file | When add/update policy, failed to create a file. Make sure the policy file path is correct | +| 500 | Failed to create key | Check if key already exist or if the key exists with a given certificate. Ensure security settings are correct | +| 500 | Failure in storing client data | Attempted to store data when creating a new OAuth client but failed. Make sure the storageID, or orgID is correct and valid. | +| 500 | Get client tokens failed | Failed to retrieve OAuth tokens. Make sure client ID is valid or keyName is valid. | +| 500 | Marshalling failed | Attempted to import printDef but failed. Marshalling of policy failed. Unmarshal object into the file failed when writing to file. | +| 500 | There was a problem proxying the request | Check if the target URL is unavailable to the Gateway. | +| 500 | Unmarshalling failed | Key creation failed. Failed to create OAuth client. Failed to update OAuth client. | +| 500 | Unsupported schema, unable to validate | Check if GraphQL schema is valid. | +| 500 | Upstreaming host lookup failed | Check if the target URL is not resolvable in DNS. | +| 503 | Service temporarily unavailable | Check if a circuit breaker middleware is enforced. | +| 503 | All hosts are down | Attempted to reverse proxy a URL rewrite to a scheme and host, but all the hosts in hostlist are down. | +| 504 | Upstream service reached hard timeout | Timeout awaiting response headers during a request round trip. | +| 507 | Status Insufficient Storage | Attempted to update an API through a POST request but failed to due insufficient storage. | + + +## Dashboard + +1. ##### Can't update policy. Please ensure at least one access rights setting is set + + **Description** + + Users receive this error when attempting to create a new Policy on the Dashboard. + + **Cause** + + The Access Rights field is a required setting for a policy. + + **Solution** + + Users should first [create a new API](/api-management/gateway-config-managing-classic#create-an-api) and then [create a new policy](/api-management/gateway-config-managing-classic#secure-an-api) with an existing API in the Access Rights. + +2. ##### Dashboard not showing any analytics data + + **Description** + + The user is unable to see analytics data from a particular time period in the Dashboard + + **Cause** + + Missing analytics data could be caused by a number of different reasons: + + * Gateway incorrectly configured + * Pump incorrectly configured + * Pump service not running + * Dashboard incorrectly configured + * MDCB incorrectly configured + * Browser caching stale data + + **Solution** + + **Gateway incorrectly configured** + + Ensure the Gateway `tyk.conf` has: + + * `enable_analytics` set to `true`. This sets the Gateway to record analytics data. + * `analytics_config.storage_expiration_time` set to a value larger than the Pump's `purge_delay`. This allows the analytics data to exist long enough in Redis to be processed by the Pump. + * `analytics_config.ignored_ips` set to `[]`. This ensures the Gateway will create analytics for requests from any IP address. + * `enforce_org_data_age` set to `false`. This prevents the data from being removed based on it reaching a certain age. + + **Pump incorrectly configured** + + Ensure the Pump `pump.conf` has: + + * `analytics_storage_type` set to `redis`. + * `analytics_storage_config` settings are set to the same Redis instance that the Gateway is connected to. + + **Pump service not running** + + Ensure the Pump service is running. + + **Dashboard incorrectly configured** + + Ensure the Dashboard `tyk_analytics.conf` has: + + * `mongo_url` set to the same MongoDB instance that the Pump is connected to. + + **MDCB incorrectly configured** + + For scenarios where MDCB is used, ensure the `sink.conf` has: + + * `analytics.mongo_url` set to the same MongoDB instance that the Dashboard is connected to. + * `forward_analytics_to_pump` set to the correct value for your solution. `false` if MDCB is directly recording the analytics itself, `true` if it is forwarding analytics data for the Pump to process. For the forwarding scenario, set the `storage` settings to the same Redis instance that the Pump is connected to. + + **Browser caching stale data** + + Try restarting your browser, or using a private session. + + You can also try restarting the Dashboard service. + + **Troubleshooting tip** + + Check if MongoDB contains analytics data by running the following query (but update the date parameter first): + + ```{.copyWrapper} + db.getCollection('tyk_analytics_aggregates').find({timestamp: {$gte: new ISODate("2016-09-26T23:59:00Z")}}) + ``` + + The query gets all aggregated analytics data from the date provided, so if you set it to yesterday you will get all data since yesterday. The data must be in the ISO format. + +3. ##### Fatal - Dashboard and portal domains cannot be the same + + **Description** + + The Tyk Dashboard service will not start and displays a fatal error as follows: + + ``` + FATAL Dashboard and portal domains cannot be the same. + Dashboard domain: tyk-dashboard.com, Portal domain: tyk-dashboard.com + ``` + + **Cause** + + Tyk's developer portal UI needs to run on either a different subdomain or different domain name to the dashboard UI. + + Tyk's Dashboard service may be run in a multi-tenant configuration, and each tenant may have their own developer portals. + + The Dashboard service determines which portal to load based on the `Host` header in the request by the browser. If this + conflicts with the hostname of the dashboard UI the dashboard service will not know whether to serve the dashboard or + developer portal. + + **Solution** + + Firstly, we will need to disable hostnames from within the Dashboard configuration file in order to get the dashboard + service started again. + + Change `host_config.enable_host_names` from `true` to `false` + ``` + "host_config": { + "enable_host_names": true, <------ CHANGE TO false + ... + ... + }, + ``` + + You should now be able to start the Dashboard service. + + Navigate to the Dashboard via it's public IP address and log-in. + + Change your portal domain to something different - e.g. `portal.tyk-dashboard.com` + + Edit the Dashboard configuration file to re-enable host names. + + Restart the Dashboard service. + +4. ##### Internal TIB SSO User unable to log in + + **Description** + + After creating an SSO Identity profile in the Tyk Dashboard, a user is unable to log in to the Dashboard or the Developer Portal + + **Cause** + + One potential cause is that the `DashboardCredential` setting has not been populated with the user's Tyk Dashboard API Access Credentials. + You can check this from: + + 1. From the Dashboard menu, select the Identity Management option + 2. Edit the profile you created + 3. Select the Raw Editor + 4. Check to see if the `DashboardCredential` setting is set + + DashboardCredentials + + + + **Workaround Solution** + + If, as above, the `DashboardCredential` setting is empty (`"DashboardCredential": ""`), you can manually add the user's Tyk Dashboard API Access Credentials by performing the following: + + 1. From the System Management > Users menu, select Actions > Edit from the user whose credentials you want to use + 2. Copy the **Tyk Dashboard API Access Credentials** value + + User API Access Credentials + + 3. Paste this into the Raw editor for the `DashboardCredential` setting. For example - `"DashboardCredential": "887dad0de40b4ff05b6b50739b311099"` + 4. Click **Update** + 5. The user should now be able to log in to the Dashboard/Portal + + + + + This issue is due to be fixed in an up coming release + + + +5. ##### Key object validation failed, most likely malformed input error + + **Description** + + The user is getting error as `Key object validation failed, most likely malformed input` when calling the Dashboard API. + + **Cause** + + Issue caused by invalid character passed in the JSON body of the request. + + **Solution** + + Validate the JSON using JSON validator. + + Further, please see [this community forum post](https://community.tyk.io/t/error-creating-new-api-through-dashboard-rest-api/1555/2) for additional guidance. + +6. ##### Port 5000 Errors in the Browser Console + + > **NOTE**: Port 5000 is no longer required from v2.9.3. + + **Description** + + You see a lot of `net::ERR_CONNECTION_REFUSED` errors in the browser console. + + **Cause** + + The Dashboard is trying to connect to `https://:5000/socket.io/?chan=ui_notifications` and you don't have port 5000 open. + + **Solution** + + Port 5000 is used for WebSocket connections for real-time Dashboard notifications. You can change the port by changing the default `notifications_listen_port` in your `tyk_analytics.conf`. Otherwise you can ignore the errors in the browser console. + + + +Port 5000 is only required if you need to enable the Tyk Gateway log viewer. + + + +7. ##### There was a problem updating your CNAMEβ€œ error in the Dashboard + + **Description** + + A user may find that they are unable to update a CNAME from within the Dashboard. The following error will appear in a pop-up: + + ``` + There was a problem updating your CNAME, please contact support + ``` + + **Cause** + + The UI for setting the domain name has a very strict validation, so it may just be rejecting this domain. + + **Solution** + + The best way to set the domain is to use the Tyk Dashboard Admin API, to obtain the organization object via a GET request and then update the object using a PUT request with the relevant CNAME added to the body of the request.[[1](/api-management/dashboard-configuration#organizations-api)] Restarting the process will then set the domain. + +8. ##### runtime error invalid memory address or nil pointer dereference + + **Description** + + When attempting to POST an OAuth Client to a newly generated API, user may receive the following stack trace: + + ``` + 2016/12/08 08:06:16 http: panic serving 172.18.0.4:46304: runtime error: invalid memory address or nil pointer dereference + goroutine 364279 [running]: + net/http.(*conn).serve.func1(0xc420569500) + panic(0xb0e780, 0xc420014040) + /usr/local/go/src/runtime/panic.go:458 +0x243 + main.createOauthClient(0xf58260, 0xc4203a41a0, 0xc4206764b0) + /home/tyk/go/src/github.com/lonelycode/tyk/api.go:1526 +0x64a + main.CheckIsAPIOwner.func1(0xf58260, 0xc4203a41a0, 0xc4206764b0) + /home/tyk/go/src/github.com/lonelycode/tyk/middleware_api_security_handler.go:24 +0x2ae + net/http.HandlerFunc.ServeHTTP(0xc420533e50, 0xf58260, 0xc4203a41a0, 0xc4206764b0) + /usr/local/go/src/net/http/server.go:1726 +0x44 + github.com/gorilla/mux.(*Router).ServeHTTP(0xc42061cdc0, 0xf58260, 0xc4203a41a0, 0xc4206764b0) + /home/tyk/go/src/github.com/gorilla/mux/mux.go:98 +0x255 + net/http.(*ServeMux).ServeHTTP(0xc420667290, 0xf58260, 0xc4203a41a0, 0xc4206764b0) + /usr/local/go/src/net/http/server.go:2022 +0x7f + net/http.serverHandler.ServeHTTP(0xc42000fc80, 0xf58260, 0xc4203a41a0, 0xc4206764b0) + /usr/local/go/src/net/http/server.go:2202 +0x7d + net/http.(*conn).serve(0xc420569500, 0xf58d20, 0xc42068bdc0) + /usr/local/go/src/net/http/server.go:1579 +0x4b7 + created by net/http.(*Server).Serve + /usr/local/go/src/net/http/server.go:2293 +0x44d + ``` + + **Cause** + + The API that the OAuth Client has been POSTed to either doesn't exist or hasn't had a chance to propagate throughout the system. + + **Solution** + + When creating a new OAuth Client, make sure that API it is created under exists. If the API was created recently, please wait a few minutes before attempting to create an OAuth Client under it. + +9. ##### ValueError No JSON object could be decoded" when running Dashboard Bootstrap script + + **Description** + + Users receive the following error message when attempting to run the bootstrap script in their Tyk instance: + + ``` + Traceback (most recent call last): + File """", line 1, in + File ""/usr/lib64/python2.7/json/__init__.py"", line 290, in load + **kw) + File ""/usr/lib64/python2.7/json/__init__.py"", line 338, in loads + return _default_decoder.decode(s) + File ""/usr/lib64/python2.7/json/decoder.py"", line 365, in decode + obj, end = self.raw_decode(s, idx=_w(s, 0).end()) + File ""/usr/lib64/python2.7/json/decoder.py"", line 383, in raw_decode + raise ValueError(""No JSON object could be decoded"") + ValueError: No JSON object could be decoded + ORGID: + Adding new user + Traceback (most recent call last): + File """", line 1, in + File ""/usr/lib64/python2.7/json/__init__.py"", line 290, in load + **kw) + File ""/usr/lib64/python2.7/json/__init__.py"", line 338, in loads + return _default_decoder.decode(s) + File ""/usr/lib64/python2.7/json/decoder.py"", line 365, in decode + obj, end = self.raw_decode(s, idx=_w(s, 0).end()) + File ""/usr/lib64/python2.7/json/decoder.py"", line 383, in raw_decode + raise ValueError(""No JSON object could be decoded"") + ValueError: No JSON object could be decoded + USER AUTH: + Traceback (most recent call last): + File """", line 1, in + File ""/usr/lib64/python2.7/json/__init__.py"", line 290, in load + **kw) + File ""/usr/lib64/python2.7/json/__init__.py"", line 338, in loads + return _default_decoder.decode(s) + File ""/usr/lib64/python2.7/json/decoder.py"", line 365, in decode + obj, end = self.raw_decode(s, idx=_w(s, 0).end()) + File ""/usr/lib64/python2.7/json/decoder.py"", line 383, in raw_decode + raise ValueError(""No JSON object could be decoded"") + ValueError: No JSON object could be decoded + NEW ID: + Setting password + DONE" + ``` + + **Cause** + + The bootstrap script requires a valid hostname and port number to generate a new login user. + + **Solution** + + Make sure that the correct hostname and port number used to run the bootstrap.sh script. An example command would be: `./bootstrap.sh new-tyk-instance.com:3000` + +10. ##### Dashboard bootstrap error + + Make sure you: + + Target the correct domain with your `bootstrap.sh` as it is very specific once you set up a Dashboard service with hostname set + + ```{.copyWrapper} + ./bootstrap.sh my-tyk-instance.com + + ``` + + Have checked the firewall rules in your instance and VPC to allow + port 3000 access. + +11. ##### How to find the policy ID for a created policy + + Open the Active Policies page in the Dashboard (System Management > Policies) and click **Edit** next to the name of the policy you've created. The policy ID should appear in the URL of the edit page that opens up. + +12. ##### How to Connect to DocumentDB with X.509 client cert + + As AWS DocumentDB runs with TLS enabled, we require a way to run it without disabling the TLS verification. + DocumentDB uses self-signed certs for verification, and provides a bundle with root certificates for this purpose, so we need a way to load this bundle. + + Additionally DocumentDB can't be exposed to the local machine outside of the Amazon Virtual Private Cloud (VPC), which means that even if verification is turned on, it will always fail since if we use a SSH tunnel or a similar method, the domain will differ from the original. Also, it can have [Mutual TLS](/basic-config-and-security/security/mutual-tls/client-mtls#why-use-mutual-tls) enabled. + + So, in order to support it, we provide the following variables for both our [Tyk Analytics Dashboard](/tyk-dashboard/configuration) and [Tyk Pump](/api-management/tyk-pump#tyk-pump-configuration): + + * `mongo_ssl_ca_file` - path to the PEM file with trusted root certificates + * `mongo_ssl_pem_keyfile` - path to the PEM file which contains both client certificate and private key. This is required for Mutual TLS. + * `mongo_ssl_allow_invalid_hostnames` - ignore hostname check when it differs from the original (for example with SSH tunneling). The rest of the TLS verification will still be performed. + + + A working DocumentDB configuration looks like this (assuming that there is SSH tunnel, proxying to 27018 port). + + ```{.json} + "mongo_url": "mongodb://testest:testtest@127.0.0.1:27018/tyk_analytics?connect=direct", + "mongo_use_ssl": true, + "mongo_ssl_insecure_skip_verify": false, + "mongo_ssl_ca_file": "/rds-combined-ca-bundle.pem", + "mongo_ssl_allow_invalid_hostnames": true, + ``` + + **Capped Collections** + + If you are using DocumentDB, [capped collections](/api-management/tyk-pump#tyk-pump-capping-analytics-data-storage) are not supported. See [here](https://docs.aws.amazon.com/documentdb/latest/developerguide/mongo-apis.html) for more details. + +13. ##### How to disable an API + + You will need to GET the API from the Dashboard, then set `active` property to `false`, then PUT it back. + See [Dashboard API - API Definitions](/api-management/dashboard-configuration#manage-apis-api-definition) for more details on how to GET and PUT an API definition. + +14. ##### How to Setup CORS + + **Upstream service supports CORS** + If your upstream service supports CORS already then Tyk should ignore **OPTIONS** methods as these are pre-flights sent by the browser. In order to do that you should select **Options passthrough**, and **NOT CHECK** CORS in Tyk. + + - If you not do allow **OPTIONS** to pass through, it will cause Tyk to dump the options request upstream and reply with the service's response so you'll get an error similar to `no 'access-control-allow-origin' header is present on the requested resource`. + + - If you check **CORS** as well you'll get an error similar to this: + ``` + Failed to load https://ORG_NAME.cloud.tyk.io/YOUR_API: The 'Access-Control-Allow-Origin' header + contains multiple values 'http://UPSTREAM', but only one is allowed. Origin 'http://UPSTREAM' + is therefore not allowed access. Have the server send the header with a valid value, or, if an + opaque response serves your needs, set the request's mode to 'no-cors' to fetch the resource with + CORS disabled + ``` + This is because you have enabled CORS on the Api Definition and the upstream **also** supports CORS and so both add the header. + + + **Upstream does not handle CORS** + If your upstream does not handle CORS, you should let Tyk manage all CORS related headers and responses. In order to do that you should **enable CORS** in Tyk and **NOT ENABLE** Options pass through. + + To learn more, look for `CORS.options_passthrough` [here](/api-management/gateway-config-tyk-classic#cross-origin-resource-sharing-cors). + + + **CORS middleware is allowing headers which I did not allow** + This may be the case when you enable CORS but don't provide any headers explicitly (basically providing an empty array). In this case the CORS middleware will use some sensible defaults. + To allow all headers, you will need to provide `*` (although this is not recommended). + + The same can happen with Allowed Origins and Allowed Methods. Read more about it [here](/api-management/gateway-config-tyk-classic#cross-origin-resource-sharing-cors). + + **CORS middleware is blocking my authenticated request** + Please make sure that you did allow the authorization header name (e.g. `Authorization`) or else the request will be blocked by the CORS middleware. If you're having trouble on the developer portal with authenticated requests make sure to also allow the `Content-Type` header. + +15. ##### No Key information on the Dashboard + + Information relating to a given key doesn't automatically appear in the Dashboard for users who have switched from a Self-Managed installation to a Multi-Cloud setup. + + The stats for a key will never update in the Cloud for a Multi-Cloud installation. The Dashboard in this mode only sets the initial β€œmaster” values for a key and those keys are then propagated across the Multi-Cloud instances that are using them (for example, you may have multiple zones with independent Redis DBs) at which point they diverge from each other. + + To see the up to date stats for a token, the key must be queried via the Gateway API. + +16. ##### How to rename or move existing headers in a request + + To rename a header, or to move a value from one header to another (for example, moving an authentication token to a secondary place, or copying a value that gets replaced upstream) is easy with [context variables](/api-management/traffic-transformation/request-context-variables). Here is an example where we move the value of `X-Custom-Header` to a new header called `X-New-Custom-Header` in all requests. + + We do this by setting the following in our API Definition Version section: + ```{.copyWrapper} + "global_headers": { + "X-New-Custom-Header": "$tyk_context.headers_X_Custom_Header" + }, + "global_headers_remove": ["X-Custom-Header"], + ``` + + You can test the header with the following command. This assumes your API Authentication mode is set to open(keyless): + + ```{.copyWrapper} + curl -X GET \ + https://DOMAIN/LISTEN_PATH/get \ + -H 'content-type: application/json' \ + -H 'x-custom-header: Foo' \ + ``` + + + You can also do this via the Dashboard from the Endpoint Designer tab within the API Designer: + + rename header + +17. ##### How to run the Dashboard and portal on different ports + + Unfortunately its not possible to run the Dashboard and Portal on different ports, they must use the same port. + +18. ##### How to run two Gateways with docker-compose + + Managing a second Tyk Gateway with our [Tyk Pro Docker Demo](/deployment-and-operations/tyk-self-managed/tyk-demos-and-pocs/overview#docker-compose-setup) is a case of mounting the `tyk.conf` file into a new volume and declaring a new Gateway service but exposed on a different port. + You will need to make some minor modifications to `docker-compose.yml` and start your services as usual with `docker-compose up`. + + + + + This will only work with an appropriate license. The free license is for development purposes and would allow running Tyk's licensed platform with only one Gateway. If you want to test Tyk with more please contact us by email [info@tyk.io](mailto:info@tyk.io) and we will be happy to discuss your case and PoC requirements as well as providing a short period license. + + + + + **Add the following to `docker-compose.yml` (after the `tyk-gateway` definition)** + + ``` + tyk-gateway2: + image: tykio/tyk-gateway:latest + ports: + - "8081:8080" + networks: + - tyk + depends_on: + - tyk-redis + volumes: + ./confs/tyk.conf:/opt/tyk-gateway/tyk.conf + ``` + +19. ##### β€œPayload signature is invalid!β€œ error + + **Description** + + Users receive the error "Payload signature is invalid!” in their logs. + + **Cause** + + Users may not have enabled payload signatures in their settings after an upgrade. + + **Solution** + + See [System Payloads](/api-management/security-best-practices#sign-payloads) for more details. + +## Pump + +1. ##### Capturing detailed logs + + If you've seen the documentation for Tyk Dashboard's [log browser](/api-management/dashboard-configuration#activity-logs), then you'll also be wondering how to set up your Tyk configuration to enable detailed request logging. + + **What is detailed request logging?** + + When [detailed request logging](/api-management/logs-metrics#capturing-detailed-logs) is enabled, Tyk will record the request and response in wire-format in the analytics database. This can be very useful when trying to debug API requests to see what went wrong for a user or client. + + This mode is configured in the gateway and can be enabled at the [system](/api-management/logs-metrics#configure-at-gateway-level), [API](/api-management/logs-metrics#configure-at-api-level) or [access key](/api-management/logs-metrics#configure-at-key-level) level. + + You will also need your Tyk Pump configured to move data into your preferred data store. + + **Disabling detailed recording for a particular pump** + + In some cases, you don't want to send the detailed request and response to a particular data store. + In order to do that, you can set `omit_detailed_recording` in your Tyk Pump configuration file to `true`. This will disable the detailed logging for a specific pump. + + For example, if we have an ElasticSearch, Kafka and CSV stores, and you want to save the detailed recording in all of them except Kafka you can use the following configuration: + + Enable detailed analytics on the Gateway `tyk.conf` using: + ```{.copyWrapper} + "enable_analytics" : true, + "analytics_config": { + "enable_detailed_recording": true + } + ``` + - Configure each pump on `pump.conf`. + - Add the `omit_detailed_recording` variable to the Kafka pump: + ```{.copyWrapper} + "pumps": { + "kafka": { + "type": "kafka", + "omit_detailed_recording":"true" + "meta": { + ... + } + }, + ... + }, + ``` + +2. ##### Connection dropped, connecting... + + **Description** + + Users may notice the following message in their logs for the Tyk Pump: + + ``` + [Jun 3 22:48:02] INFO elasticsearch-pump: Elasticsearch Index: tyk_analytics + [Jun 3 22:48:02] INFO main: Init Pump: Elasticsearch Pump + [Jun 3 22:48:02] INFO main: Starting purge loop @10(s) + [Jun 3 22:48:12] WARN redis: Connection dropped, connecting.. + [Jun 3 22:48:23] INFO elasticsearch-pump: Writing 1386 records + [Jun 3 22:50:11] INFO elasticsearch-pump: Writing 13956 records + ``` + + **Cause** + + This is normal behavior for the Tyk Pump. + + **Solution** + + N/A + +3. ##### Data Seen in Log Browser but No Reports + + **Description** + + You can see data in the log browser but the rest of the reports display nothing. + + **Solution** + + If your Pump is configured to use `mongo_selective_pump` (e.g. store data in a collection per organization), ensure that the [Dashboard configuration setting](/tyk-dashboard/configuration) `use_sharded_analytics` is set to `true`. + + The same applies in the reverse direction. If you are using `mongo-pump-aggregate` in your [pump configuration](/api-management/tyk-pump#tyk-pump-configuration), set `use_sharded_analytics` to false. + + This is because you've enabled `use_sharded_analytics` as per above and you're using the `mongo-pump-aggregate`, but you now also have to add a `mongo-pump-selective` in order to save individual requests to Mongo, which the Dashboard can read into the Log Browser. + +4. ##### No Elasticsearch node available + + **Description** + + Tyk Pump is configured to use Elasticsearch, but it does not work and shows `no Elasticsearch node available` message in log. + + ``` + tyk-pump[68354]: time="Aug 30 15:19:36" level=error msg="Elasticsearch connection failed: no Elasticsearch node available" + ``` + + **Cause** + + The `elasticsearch_url` configuration property in the `pump.conf` is missing the HTTP prefix e.g. + + ``` + "elasticsearch_url": "127.0.0.1:9200" + ``` + + **Solution** + + Ensure the HTTP prefix is present in the `elasticsearch_url` configuration property e.g. + + ``` + "elasticsearch_url": "http://127.0.0.1:9200" + ``` + +5. ##### Tyk Pump Panic β€œstack exceeds 1000000000-byte limitβ€œ + + **Description** + + Users receive a the aforementioned error message in a stack trace in the Pump. + + **Cause** + + Users receive a the aforementioned error message in a stack trace in the Pump. + + **Solution** + + Users are advised to upgrade to the latest version of Tyk. They must also ensure that their Pump is configured with a `purge_delay` and an `optimisation_max_active` value that's greater than 0. Packages are available to download from [Packagecloud.io](https://packagecloud.io/tyk) and further details on how to upgrade can be found [here](/developer-support/upgrading) + +6. ##### Pump overloaded + + **Description** + + The Tyk Pump cannot deal with amount of analytics data generated by the Gateway. This means the Pump is unable to process all the analytics data within the purge period. + + **Cause** + + If there is excessive analytics data, the pump may become overwhelmed and not able to move the data from Redis to the target data store. + + **Solution** + + There are many ways to approach solving this problem. + + **Scale the Pump** + + Scale the Pump by either increasing the CPU capacity of the Pump host or by adding more Pump instances. + + By adding more instances you are spreading the load of processing analytics records across multiple hosts, which will increase processing capacity. + + **Disable detailed analytics recording** + + Set `analytics_config.enable_detailed_recording` to `false` in the Gateway configuration file `tyk.conf`. Detailed analytics records contain much more data and are more expensive to process, by disabling detailed analytics recording the Pump will be able to process higher volumes of data. + + **Reduce the Pump purge delay** + + Set `purge_delay` to a low value e.g. `1` in the Pump configuration file `pump.conf`. This value is the number of seconds the Pump waits between checking for analytics data. Setting it to a low value will prevent the analytics data set from growing too large as the pump will purge the records more frequently. + + **Reduce analytics record expiry time** + + Set `analytics_config.storage_expiration_time` to a low value e.g. `5` in the Gateway configuration file `tyk.conf`. This value is the number of seconds beyond which analytics records will be deleted from the database. The value must be higher than the `purge_delay` set for the Pump. This will allow for analytics records to be discarded in the scenario that the system is becoming overwhelmed. Note that this results in analytics record loss, but will help prevent degraded system performance. + +7. ##### Tyk Pump Graceful Shutdown + + From version 1.5.0, Tyk Pump has a new mechanism which is called Pump’s Graceful Shutdown. + + **What does Pump’s Graceful Shutdown mean exactly?** + + That means that now, when Tyk Pump gets stopped or restarts, it is going to wait until the current purge cycle finishes, determined by `purge_delay` configuration. It is also going to flush all the data of the pumps that have an inner buffer. + + **Why are we doing that?** + + We are adding this mechanism since we were losing data on Kubernetes environments or ephemeral containers, whenever a pump instance finishes. Now, with this change, we aren't going to lose analytics records anymore. + + **When is it triggered?** + + This is triggered when Tyk Pump catches one of the following signals from the OS: + + - `os.Interrupt, syscall.SIGINT` + - `syscall.SIGTERM` + + When the signal is `SIGKILL`, it will not gracefully stop its execution. + + **What pumps are affected?** + + The main affected pumps are: + - `ElasticSearch` + - `dogstatd` + - `Influx2` + + As they all have some kind of in-memory buffering before sending the data to the storage. With this new mechanism, all those pumps are going to flush their data before finishing the Tyk Pump process. + Furthermore, in a certain way, this new feature affects all the rest of the pumps since Tyk Pump is now going to wait to finish the purge cycle per se, like the writing of the records in all the configured storages. + +## Streams + +1. ##### Failure to connect to the event broker + + If Tyk Gateway is unable to establish a connection to the configured event broker (e.g., Kafka, MQTT), check the following: + - Verify that the broker connection details in the Tyk Dashboard are correct, including the hostname, port, and any required credentials. + - Ensure that the event broker is running and accessible from the Tyk Gateway instance. + - Check the network connectivity between the Tyk Gateway and the event broker. Use tools like telnet or nc to validate the connection. + +2. ##### Messages are not being published or consumed + + If messages are not being successfully published to or consumed from the event broker, consider the following: + - Verify that the topic or queue names are correctly configured in the Tyk Dashboard and match the expected values in the event broker. + - Check the Tyk Gateway logs for any error messages related to message publishing or consumption. Adjust the log level to "debug" for more detailed information. + - Validate that the message format and schema match the expectations of the consumer or producer. Inspect the message payloads and ensure compatibility. + +3. ##### Async API performance is poor or connections are being throttled + + If you observe performance issues or connection throttling with async APIs, consider the following: + - Review the configured rate limits and quotas for the async API. Adjust the limits if necessary to accommodate the expected traffic. + - Monitor the resource utilization of the Tyk Gateway instances and the event broker. Ensure that there is sufficient capacity to handle the load. + - Consider scaling the Tyk Gateway horizontally by adding more instances to distribute the traffic load. + +4. ##### What are best practices of using Tyk Streams + + - Use meaningful and descriptive names for your async APIs, topics, and subscriptions to improve readability and maintainability. + - Implement proper security measures, such as authentication and authorization, to protect your async APIs and restrict access to authorized clients only. + - Set appropriate rate limits and quotas to prevent abuse and ensure fair usage of the async API resources. + - Monitor the performance and health of your async APIs using Tyk's built-in analytics and monitoring capabilities. Set up alerts and notifications for critical events. + - Version your async APIs to manage compatibility and enable seamless updates without disrupting existing clients. + - Provide comprehensive documentation for your async APIs, including details on message formats, schemas and example payloads, to assist developers in integrating with your APIs effectively. + + +## Debugging Series + +### MongoDB + +Tyk uses Mongo as a database to store much of its analytical data. This means if you have a dashboard instance that is down, there’s a high chance that this is because of either Mongo being down or an issue with your dashboard connecting to Mongo. + +Here, we'll outline the following: + + - How to isolate Mongo as the root of the error + - The steps to take to help stop your system from going down. + +1. ##### Isolating Mongo as the fault + + Here are a few ways to identify Mongo as the source of the problem: + + 1. Analytics is not showing up on the dashboard + 2. When hitting the `/hello` endpoint, the dashboard is down + 3. The Mongo database size is hitting hardware resource limits. + +2. ##### Mongo status + + Similarly to Tyk, Mongo has a health check that we can run to get the status of our Mongo instance. This should be a starting point for debugging Mongo (depending on which system): + + - `Sudo systemctl status mongod` or `sudo service mongodb status` + - Logs under `/var/log/mongo/mongo.log` should also outline any outage + +3. ##### Mongo version + + Does Tyk support the version of Mongo that you’re using? Read more about that [here](/planning-for-production/database-settings#mongodb). + +4. ##### Capped collections + + Suppose a Mongo instance runs over a long period in addition to a lot of traffic in a Tyk system. In that case, the chances of the collections growing out of control are very real - especially the `tyk_analytics` collections. + + In some cases, `enable_detailed_logging: true` adds fuel to the fire, as this parameter should only be set temporarily during debugging. This configuration exists on the gateway and the API levels, so ensure this is off after debugging. + + We advise everyone to cap every collection in Mongo, as this prevents collections from growing out of control and bringing your dashboard down by hitting resource limits. + + You can determine each collection's cap size by visiting our [MongoDB sizing calculator](/planning-for-production/database-settings#mongodb-sizing-guidelines). + + Here’s more information on how and why you want to [cap your collections](https://www.mongodb.com/docs/manual/core/capped-collections/). + +5. ##### Size caps versus TTL-capped collections + + Are you trying to decide between capping your collections or by size? It depends on a couple of factors. Ultimately, both settings will get rid of older data, so it’s based on how far back you need to view it. + + Assuming you only need data for a few days, then using a TTL will be the best route, as it will only allow your collections to grow that wild over a short period. + + Alternatively, if you care about how big the collections grow and want to see longer-lived data, then capping by size is your best direction. This will limit the collection to developing within a controlled resource limit. And in the context of aggregate analytics, this collection will hold data for long periods. + + One thing to note here is that if you head down the TTL route, and if your environment has A LOT of traffic, then your collections can grow wild and fast, while a size-capped collection will always stay within a known size limit. + +6. ##### Handling overgrown, uncapped collections + + There are three ways to do this: + + 1. The first method is to delete (drop) the collection and create a new collection with a cap (commands below). + + ```bash + # This will drop a collection. When using this, cached data will not be deleted. + db..drop() + ``` + + ```bash + # Can use the below call. Drops the collection and removes any cache data + db..remove() + ``` + + 2. The second method is to rename the collection to a random name and then create a new collection with a cap. Then restart Mongo with a larger size (we do this because the overgrown collections still exist). This is to confirm that the collection size grew too large and dropped the Mongo connection. The renaming also helps conserve the existing data if you still need it (but it will be useless in the background unless you attempt the third method). + + 3. The third method is to delete (deleteMany() call below) the old data to trim down their collection size. Then, you can restart your instance to see if the connection goes up again. + + ```bash + # Will delete data off a collection that does NOT have a cap. Otherwise, it will throw an error. + db..deleteMany() + ``` + +7. ##### Secure Mongo connection + + You will use a secured connection to your Mongo instance in most production cases. Here are a few things to consider: + + - Verify there isn’t a network issue that stops your dashboard from connecting to Mongo. You can do this by hitting the dashboard server from your Mongo server (or vice versa) + + - Validate certificate and `.pem` files + + - Connect (command below) to Mongo with certificates + + ```bash + # Replace the above files with the correct parameters (proper file paths and host). + mongo --ssl --sslCAFile /opt/mongodb/ssl/ca.pem --sslPEMKeyFile /opt/mongodb/ssl/mongodb.pem --host 127.0.0.1 + ``` + - Verify Pump has the correct parameters to include your certificates + + - Verify your dashboard has the correct parameters relative to your environment: + + ```json + "mongo_url": "mongodb://localhost/tyk_analytics", + "mongo_use_ssl": true, + "mongo_ssl_ca_file": "/opt/mongodb/ssl/ca.pem", + "mongo_ssl_pem_keyfile": "/opt/mongodb/ssl/mongodb.pem", + "mongo_ssl_insecure_skip_verify": true + ``` + +8. ##### How to Cap analytics data storage + + What methods are available to enable me to manage my MongoDB analytics storage? + + [Time Based Caps](/api-management/tyk-pump#time-based-cap-in-single-tenant-environments) + + [Size Based Caps](/api-management/tyk-pump#size-based-cap) + + + + + Time based caps (TTL indexes) are incompatible with already configured size based caps. + + + + + +If you are using DocumentDB, capped collections are not supported. See [here](https://docs.aws.amazon.com/documentdb/latest/developerguide/mongo-apis.html) for more details. + + + +9. ##### MongoDB X.509 Client Authentication + + You can use the *MongoDB X509 Certificate* flow to authenticate the *Tyk Dashboard*, *Tyk Pump*, and *Tyk MDCB* with your *MongoDB* install. This is slightly different from [AWS DocumentDB setup instructions](/api-management/troubleshooting-debugging#how-to-connect-to-documentdb-with-x509-client-cert). + + Before we get into the configuration, we need to understand the two key components: connection strings and certificates. + + 1. **Connection Strings** + + 1) You must specify a username (and password if needed) in the connection string. [Why do you need a username at all?](https://docs.mongodb.com/manual/tutorial/configure-x509-client-authentication/) + + 2) We must specify the following parameters: `?authSource=$external&authMechanism=MONGODB-X509"` + + **An example of a connection string would be:** + + ```bash + "mongodb://CN=tyk-mongo-client,OU=TykTest@:/?authSource=$external&authMechanism=MONGODB-X509" + ``` + + ##### Passwords + If you have to include a password, you can do it after the username in basic auth format: + + ```bash + "mongodb://CN=tyk-mongo-client,OU=TykTest,O=TykTest:mypassword@:/?authSource=$external&authMechanism=MONGODB-X509" + ``` + + ##### URL Encoding Protected Characters + Note that you must URL encode the `:` character into `%40`. So replace any `:` in the username field into the URL encoded version. + + 2. **Certificates** + + You'll need to provide two certificates to complete the X509 Client Authentication: + + **CA Cert** containing just the public key of the Certificate Authority (CA). + + **Client Cert** containing both the public and private keys of the client. + + ##### Configuration + + Here's what it looks like all put together: + + 1. **Tyk Dashboard** + + Your `tyk_analytics.conf` should include these fields at the root level: + + ```json + { + ... + "mongo_url": "mongodb://@:/?authSource=$external&authMechanism=MONGODB-X509", + "mongo_use_ssl": true, + "mongo_ssl_ca_file": "ca.pem", + "mongo_ssl_pem_keyfile": "client.pem" + } + ``` + + | Config File | Environment Variable | Type | Examples + | --- | -- | ---- | ---- | + | "mongo_url" | TYK_DB_MONGOURL | string | "mongodb://{username}@{host}:{port}/{db}?authSource=$external&authMechanism=MONGODB-X509" | + | "mongo_use_ssl" | TYK_DB_MONGOUSESSL | bool | true, false | + | "mongo_ssl_ca_file" | TYK_DB_MONGOSSLCAFILE | string | "certificates/ca.pem" | + | "mongo_ssl_pem_keyfile" | TYK_DB_MONGOSSLPEMKEYFILE | string | "certificates/key.pem" | + | "mongo_ssl_insecure_skip_verify" | TYK_DB_MONGOSSLINSECURESKIPVERIFY | bool | true, false | + | "mongo_ssl_allow_invalid_hostnames" | TYK_DB_MONGOSSLALLOWINVALIDHOSTNAMES | bool | true, false | + | "mongo_session_consistency" | TYK_DB_MONGOSESSIONCONSISTENCY | string | "strong", "eventual", or "monotonic". default is "strong" | + | "mongo_batch_size" | TYK_DB_MONGOBATCHSIZE | int | Default "2000", min "100" | + + 2. **Tyk Pump** + + Tyk offers three different MongoDB pumps (`mongo`, `mongo_aggregate`, and `mongo_selective`), each of which must be separately configured for X509 certificate authentication. + + The following fields must be set under the `meta` section of each pump (or set as environment variable): + + ```yaml + { + ... + "pumps": { + "mongo": { + "type": "mongo", + "meta": { + "collection_name": "tyk_analytics", + "mongo_url": "mongodb://CN=tyk-mongo-client,OU=TykTest@:/?authSource=$external&authMechanism=MONGODB-X509", + "mongo_use_ssl": true, + "mongo_ssl_ca_file": "ca.pem", + "mongo_ssl_pem_keyfile": "client.pem" + } + } + } + } + ``` + + In addition to the other configs, these are the ones related to MongoDB: + + | Config File | Type | Examples + | -- | -- | -- + "mongo_url" | string | "mongodb://{username}@{host}:{port}/{db}?authSource=$external&authMechanism=MONGODB-X509" | + "mongo_use_ssl" | bool | true, false | + "mongo_ssl_ca_file" | string | "certificates/ca.pem" | + β€œmongo_ssl_pem_keyfile" | string | "certificates/key.pem" | + "mongo_ssl_insecure_skip_verify" | bool | true, false | + "mongo_ssl_allow_invalid_hostnames" | bool | true, false | + + 3. **Tyk MDCB** + + As of Tyk MDCB v1.8.0, you have been able to secure Tyk MDCB with MongoDB using X509 Certificate Authentication flow. + + The config settings are exactly the same as the Tyk Dashboard steps, just nested one level deeper: + + **Example Config:** + ```json + { + ... + "analytics": { + "mongo_url": "mongodb://CN=tyk-mongo-client,OU=TykTest@:/?authSource=$external&authMechanism=MONGODB-X509", + "mongo_use_ssl": true, + "mongo_ssl_ca_file": "ca.pem", + "mongo_ssl_pem_keyfile": "client.pem" + } + } + ``` + | Config File | Environment Variable | Type | Examples + | --- | -- | ---- | ---- | + "analytics.mongo_url" | TYK_MDCB_ANALYTICSCONFIG_MONGOURL | string | "mongodb://{username}@{host}:{port}/{db}?authSource=$external&authMechanism=MONGODB-X509" + "analytics.mongo_use_ssl" | TYK_MDCB_ANALYTICSCONFIG_MONGOUSESSL | bool | true, false | + "analytics.mongo_ssl_ca_file" | TYK_MDCB_ANALYTICSCONFIG_MONGOSSLCAFILE | string | "certificates/ca.pem" | + "analytics.mongo_ssl_pem_keyfile" | TYK_MDCB_ANALYTICSCONFIG_MONGOSSLPEMKEYFILE | string | "certificates/key.pem" | + "analytics.mongo_ssl_insecure_skip_verify" | TYK_MDCB_ANALYTICSCONFIG_MONGOSSLINSECURESKIPVERIFY | bool | true, false | + "analytics.mongo_ssl_allow_invalid_hostnames" | TYK_MDCB_ANALYTICSCONFIG_MONGOSSLALLOWINVALIDHOSTNAMES | bool | true, false | + "analytics.mongo_session_consistency" | TYK_MDCB_ANALYTICSCONFIG_MONGOSESSIONCONSISTENCY | string | "strong", "eventual", or "monotonic". default is "strong" | + "analytics.mongo_batch_size" | TYK_MDCB_ANALYTICSCONFIG_MONGOBATCHSIZE | int | Default "2000", min "100" | + +### Tyk Self-Managed + +This guide should help a user of Tyk Self-Managed in debugging common issues. A helpful way to go about this is by: + +1. Isolating your components to see where the error is coming from +2. Enabling debug logs to ensure you get all the information you need + +1. ##### Gateway `/hello` endpoint + + Querying the gateway's `/hello` health endpoint is the quickest way to determine the status of your Tyk instance. You can find more information in our docs about the [Gateway Liveness health check](/planning-for-production/ensure-high-availability/health-check). + + This endpoint is important as it allows the user to isolate the problem's origin. At a glance, the `/hello` endpoint reports the Gateways connectivity to Redis, and the control plane components eg. Tyk Dashboard, Tyk Multi-Data Center Bridge (MDCB), and Tyk Cloud. + + ```json + { + "status": "pass", + "version": "v5.0", + "description": "Tyk GW", + "details":{ + "dashboard":{ + "status": "pass", + "componentType": "system", + "time": "2023-01-13T14:45:00Z" + }, + "redis":{ + "status": "pass", + "componentType": "datastore", + "time": "2023-01-13T14:45:00Z" + } + }, + "rpc": { + "status": "pass", + "componentType": "system", + "time": "2023-01-13T14:45:00Z" + } + } + ``` + + If the Dashboard or RPC connectivity fails (control plane components), the Gateway will still function based on the last received configurations from those components. However, if Redis fails, Gateway will go down since it is a hard dependency. + +#### Debug Logs + +Setting the log level to debug will allow for more descriptive logs that will give a better context around any issue you might be facing. For example, here are the different outputs you receive when calling an Open Keyless API with `info` and `debug` log-level modes. + +Here is the output when using `info` as the log level: + +```bash +tyk-pump | time="Jan 24 14:39:19" level=info msg="Purged 1 records..." prefix=mongo-pump +tyk-pump | time="Jan 24 14:39:19" level=info msg="Purged 1 records..." prefix=mongo-pump-selective +tyk-mongo | 2023-01-24T14:39:19.228+0000 I NETWORK [listener] connection accepted from 172.20.0.2:51028 #19 (19 connections now open) +tyk-pump | time="Jan 24 14:39:19" level=info msg="Completed upserting" collection="tyk_analytics_aggregates" prefix=mongo-pump-aggregate +tyk-pump | time="Jan 24 14:39:19" level=info msg="Purged 1 records..." prefix=mongo-pump-aggregate +``` + +Here is a more detailed output of the same call when using `debug` as the log level: + +```bash +tyk-gateway | time="Jan 24 14:32:19" level=debug msg="Started proxy" +tyk-gateway | time="Jan 24 14:32:19" level=debug msg="Stripping proxy listen path: /api1/" +tyk-gateway | time="Jan 24 14:32:19" level=debug msg="Upstream path is: /get" +tyk-gateway | time="Jan 24 14:32:19" level=debug msg=Started api_id=63666619de884d0563ee3ccc67d57929 api_name=api1 mw=ReverseProxy org_id=63ca963f6888c7000191890e ts=1674570739659369736 +tyk-gateway | time="Jan 24 14:32:19" level=debug msg="Upstream request URL: /get" api_id=63666619de884d0563ee3ccc67d57929 api_name=api1 mw=ReverseProxy org_id=63ca963f6888c7000191890e +tyk-gateway | time="Jan 24 14:32:19" level=debug msg="Outbound request URL: http://httpbin.org/get" api_id=63666619de884d0563ee3ccc67d57929 api_name=api1 mw=ReverseProxy org_id=63ca963f6888c7000191890e +tyk-gateway | time="Jan 24 14:32:19" level=debug msg="Creating new transport" api_id=63666619de884d0563ee3ccc67d57929 api_name=api1 mw=ReverseProxy org_id=63ca963f6888c7000191890e +tyk-gateway | time="Jan 24 14:32:19" level=debug msg="Out request url: http://httpbin.org/get" api_id=63666619de884d0563ee3ccc67d57929 api_name=api1 mw=ReverseProxy org_id=63ca963f6888c7000191890e +tyk-gateway | time="Jan 24 14:32:19" level=debug msg="Request is not cacheable" mw=ResponseCacheMiddleware +tyk-gateway | time="Jan 24 14:32:19" level=debug msg=Finished api_id=63666619de884d0563ee3ccc67d57929 api_name=api1 mw=ReverseProxy ns=316559477 org_id=63ca963f6888c7000191890e +tyk-gateway | time="Jan 24 14:32:19" level=debug msg="Upstream request took (ms): 316.639871" +tyk-gateway | time="Jan 24 14:32:19" level=debug msg="Checking: 63ca963f6888c7000191890e" api_id=63666619de884d0563ee3ccc67d57929 api_name=api1 org_id=63ca963f6888c7000191890e +tyk-gateway | time="Jan 24 14:32:19" level=debug msg="no cached entry found, returning 7 days" api_id=63666619de884d0563ee3ccc67d57929 api_name=api1 org_id=63ca963f6888c7000191890e +tyk-gateway | time="Jan 24 14:32:19" level=debug msg="Done proxy" +tyk-pump | time="Jan 24 14:32:20" level=info msg="Purged 0 records..." prefix=mongo-pump-aggregate +tyk-pump | time="Jan 24 14:32:20" level=info msg="Purged 1 records..." prefix=mongo-pump-selective +tyk-pump | time="Jan 24 14:32:20" level=info msg="Completed purging the records" collection="tyk_analytics" number of records=1 prefix=mongo-pump +tyk-pump | time="Jan 24 14:32:20" level=info msg="Purged 1 records..." prefix=mongo-pump +tyk-mongo | 2023-01-24T14:32:20.398+0000 I NETWORK [listener] connection accepted from 172.20.0.3:54712 #19 (19 connections now open) +tyk-pump | time="Jan 24 14:32:20" level=info msg="Completed upserting" collection="tyk_analytics_aggregates" prefix=mongo-pump-aggregate +tyk-pump | time="Jan 24 14:32:20" level=info msg="Purged 1 records..." prefix=mongo-pump-aggregate + +``` + +As shown above, the `debug` log level mode provides more information which will help during your debugging stage, i.e when the API call was started, when it was finished, how long it took for the call to finish, the endpoint that was called, the upstream that was called, the organization that the API belongs to, and more. + +1. ##### Gateway Debug Settings + + If you’re using a `*.conf` for your configuration parameters: + + ```json + "log_level": "debug" + ``` + + If you’re using environment variables for your configuration: + + ```bash + TYK_GW_LOGLEVEL=debug + ``` + + If you're using Tyk Helm Charts. Add the following items to your `values.yaml`: + + ```yaml + extraEnvs: + - name: TYK_LOGLEVEL + value: debug + ``` + +2. ##### Dashboard Debug Settings + + If you’re using a `*.conf` for your configuration parameters: + + ```json + "log_level": "debug" + ``` + + If you’re using environment variables for your configuration: + + ``` + TYK_DB_LOGLEVEL=debug + ``` + + If you're using Tyk Helm Charts. Add the following items to your `values.yaml`: + + ```yaml + extraEnvs: + - name: TYK_LOGLEVEL + value: debug + ``` + + You can find the full [log levels](/api-management/logs-metrics#system-logs) in our documentation. + +#### Versions + +You can access all Tyk release information on the [release notes](/developer-support/release-notes/overview) overview page. + +We recommend always using the [Long-Term Support (LTS) release](/developer-support/release-types/long-term-support) for stability and long term support. + +##### Non-LTS versions +Tyk is backwards compatible, upgrading to newer versions won't turn on new features or change the behavior of your existing environment. + +For the best experience when experimenting with Tyk and exploring its latest capabilities, you can use our latest version. You can access all Tyk releases on the [release notes summary](/developer-support/release-notes/overview) page. + +#### Dashboard + +The Dashboard front-end (GUI included) uses [Tyk Dashboard API](/tyk-dashboard-api) to retrieve data to display or update. This means you can use the [developer tools on your browser](https://developer.mozilla.org/en-US/docs/Learn/Common_questions/Tools_and_setup/What_are_browser_developer_tools) to access the API and its information. Looking into the API details, the URL, the headers, the payload and the response can help you investigate the source of the issue and replicate it with API calls using an HTTP client such as [cURL](https://curl.se/) or [Postman](https://www.postman.com/). +As a next step to this investigation, if that specific endpoint exists also in [Tyk Gateway API](/tyk-gateway-api), you can compare the responses from both gateway and dashboard requests. + +##### Isolating + +As mentioned above, errors can happen in any of the components of your Tyk deployment, and as such, one of the critical things you'll pick up during your debugging phase is isolating these environments. + +##### Dashboard Level + +When debugging an issue, in order to isolate the gateway from the Dashboard, try to call the same API ednpoint on both Tyk Dashboard and Tyk Gateway +If it works with the gateway API only, then the issue is likely to be in the Dashboard. It could be that you need to set in the Dashboard some [configuration parameters](/tyk-dashboard/configuration) (using the config file or via environment variables). + +##### Gateway or API level + +Are you making calls against your gateway or API, and it's not working? Try isolating the gateway from everything else. Often you'll see that the gateway or API aren't at fault and that it's something else; it can be the load balancer you have in your environment blocking the call from ever reaching it. + +In the case of the API error-ing out, you can also isolate it by: + +- Creating a generic Httpbin API and calling it + - If this works, then the API configuration or the backend is at fault +- Changing the target URL of the API + - The upstream API can be at fault +- Assuming your API has a plugin, take away the plugin and test the API + - The error most likely exists in the plugin +- If the error exists in your plugin, try taking out certain parts of the code and testing it with minimal logic + - This means that part of your code with integrated logic is incorrect +- Is the target URL the same in another one of your APIs? + - The gateway sees the API as duplicated and changes the new target URL causing the gateway to error. + +You will eventually hit the point of error by further isolating parts of your API. + diff --git a/api-management/tyk-pump.mdx b/api-management/tyk-pump.mdx new file mode 100644 index 00000000..79b1d655 --- /dev/null +++ b/api-management/tyk-pump.mdx @@ -0,0 +1,1835 @@ +--- +title: "Tyk Pump - Export Metrics to Persistent Datastore" +'og:description': "How to configure Tyk Pump" +tags: ['Pump', 'CSV', 'Datadog', 'Elasticsearch', 'Logzio', 'Moesif', 'Splunk', 'Prometheus', 'Analytics Storage', 'Monitoring', 'Observability'] +sidebarTitle: "Metrics Exporter" +--- + +## Introduction + +Traffic analytics are captured by the Gateway nodes and then temporarily stored in Redis. The Tyk Pump is responsible for moving those analytics into a persistent data store, such as MongoDB, where the traffic can be analyzed. + +## What is the Tyk Pump? + +The Tyk Pump is our [open source](https://github.com/TykTechnologies/tyk-pump) analytics purger that moves the data generated by your Tyk nodes to any back-end. It is primarily used to display your analytics data in the Tyk Dashboard. + + + +The Tyk Pump is not currently configurable in our Tyk Cloud solution. + + + +### Tyk Pump Data Flow + +Here's the architecture depending on your deployment model: + + + + +Tyk Enterprise Pump Architecture + + + + +Tyk Open Source Pump Architecture + + + + +Tyk-Pump is both extensible, and flexible- meaning it is possible to configure Tyk-Pump to send data to multiple different backends at the same time as depicted by Pump Backends (i) and (ii), MongoDB and Elasticsearch respectively in Figure 1. Tyk-Pump is scalable, both horizontally and vertically, as indicated by Instances "1", "2", and "n". Additionally, it is possible to apply filters that dictate WHAT analytics go WHERE, please see the [docs on sharded analytics configuration here](/api-management/tyk-pump#configuring-the-sharded-analytics). + +| Configuration and Scaling of Tyk Pump | +| :-- | +| Figure 1: An architecture diagram illustrating horizontal scaling of "n" Instances of Tyk-Pump each with two different backends. | + +### Other Supported Backend Services + +We list our [supported backends here](/api-management/tyk-pump#external-data-stores). + +### Configuring your Tyk Pump + +See [Tyk Pump Configuration](/api-management/tyk-pump#tyk-pump-configuration) for more details on setting up your Tyk Pump. + +Tyk Pump can be horizontally scaled without causing duplicate data, please see the following Table for the supported permutations of Tyk Pump scaling. + +| Supported | Summary | +| :-- | :-- | +| βœ… | Single Pump Instance, Single Backend | +| βœ… | Single Pump Instance, Multiple Backend(s) | +| βœ… | Multiple Pump Instances, Same Backend(s)| +| ❌ | Multiple Pump Instances, Different Backend(s) | + +## Getting Started + +### Tyk Pump Configuration + +The Tyk Pump is our Open Source analytics purger that moves the data generated by your Tyk nodes to any back-end. By moving the analytics into your supported database, it allows the Tyk Dashboard to display traffic analytics across all your Tyk Gateways. + +#### Tyk Dashboard + +##### MongoDB + +The Tyk Dashboard uses the `mongo-pump-aggregate` collection to display analytics. This is different than the standard `mongo` pump plugin that will store individual analytic items into MongoDB. The aggregate functionality was built to be fast, as querying raw analytics is expensive in large data sets. See [Pump Dashboard Config](/api-management/tyk-pump#setup-dashboard-analytics) for more details. + +##### SQL + + + +Tyk no longer supports SQLite as of Tyk 5.7.0. To avoid disruption, please transition to [PostgreSQL](/planning-for-production/database-settings#postgresql), [MongoDB](/planning-for-production/database-settings#mongodb), or one of the listed compatible alternatives. + + + +In v4.0 of the Tyk Dashboard, we added support for the following SQL platforms: +- PostgreSQL +- SQLite + +Within your Dashboard configuration file (`tyk-analytics.conf`) there is now a `storage` section. + +```{.shell} expandable +{ + ... + "storage": { + "main":{}, + "analytics":{}, + "logs":{}, + "uptime": {} + } +} +``` +###### Field description + +- `main` - Main storage (APIs, Policies, Users, User Groups, etc.) +- `analytics` - Analytics storage (used for display all the charts and for all analytics screens) +- `logs` - Logs storage (log browser page) +- `uptime` - uptime tests analytics data + +###### Common settings + +For every `storage` section, you must populate the following fields: +```{.shell} expandable +{ +... + "storage": { + ... + "main": { + "type": "postgres", + "connection_string": "user=root password=admin database=tyk-demo-db host=tyk-db port=5432", + } + } +} +``` +- `type` use this field to define your SQL platform (currently SQLite or PostgreSQL are supported) +- `connection_string` the specific connection settings for your platform + +The pump needed for storing logs data in the database, is very similar to other pumps as well as the storage setting in your Tyk Dashboard config. It just requires the `sql` name and database specific configuration options. + +####### SQL example + +```{.shell} expandable +"sql": { + "name": "sql", + "meta": { + "type": "postgres", + "connection_string": "user=laurentiughiur password=test123 database=tyk-demo-db host=127.0.0.1 port=5432" + } +}, +``` + +#### Capping analytics data + +Tyk Gateways can generate a lot of analytics data. Be sure to read about [capping your Dashboard analytics](/api-management/tyk-pump#tyk-pump-capping-analytics-data-storage) + +#### Omitting the configuration file + +From Tyk Pump 1.5.1+, you can configure an environment variable to omit the configuration file with the `TYK_PMP_OMITCONFIGFILE` variable. +This is specially useful when using Docker, since by default, the Tyk Pump has a default configuration file with pre-loaded pumps. + +#### Sharding analytics to different data sinks + +In a multi-organization deployment, each organization, team, or environment might have their preferred analytics tooling. This capability allows the Tyk Pump to send analytics for different organizations or various APIs to different destinations. +E.g. Org A can send their analytics to MongoDB + DataDog +while Org B can send their analytics to DataDog + expose the Prometheus metrics endpoint. + +##### Configuring the sharded analytics + +You can achieve the sharding by setting both an allowlistt and a blocklist, meaning that some data sinks can receive information for all orgs, whereas other data sinks will not receive certain organization's analytics if it was block listed. + +This feature makes use of the field called `filters`, which can be defined per pump. This is its structure: +``` expandable +"filters":{ + "api_ids":[], + "org_ids":[], + "skip_api_ids":[], + "skip_org_ids":[] + } +``` +- `api_ids` and `org_ids` works as allow list (APIs and orgs where we want to send the analytic records). +- `skip_api_ids` and `skip_org_ids` works as block list (APIs and orgs where we want to filter out and not send their the analytic records). + +The priority is always a blocklist over a allowlist. + +An example of configuration would be: + ``` expandable +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +}, +"elasticsearch": { + "type": "elasticsearch", + "filters": { + "skip_api_ids": ["api_id_1"], + }, + "meta": { + "index_name": "tyk_analytics", + "elasticsearch_url": "https://elasticurl:9243", + "enable_sniffing": false, + "document_type": "tyk_analytics", + "rolling_index": false, + "extended_stats": false, + "version": "6" + } +} +``` +With this configuration, all the analytics records related to `org1` or `org2` will go to the `csv` backend and everything but analytics records from `api_id_1` to `elasticsearch`. + + +### Setup Dashboard Analytics + +To enable [Dashboard Analytics](/api-management/dashboard-configuration#traffic-analytics), you would need to configure Tyk Pump to send analytic data to the Dashboard storage MongoDB / SQL. + +These are the different pumps that handle different kinds of analytic data. + +| Analytics | Activities Graph | Log Browser | Uptime Analytics | +| :---------------------------- | :-------------------- | :-------------------- | :---------------- | +| Mongo (Multi organization) | Mongo Aggregate Pump | Mongo Selective Pump | Uptime Pump | +| Mongo (Single organization) | Mongo Aggregate Pump | Mongo Pump | Uptime Pump | +| SQL | SQL Aggregate Pump | SQL Pump | Uptime Pump | + +See below details about these pumps, their configs, matching collections and relevant dashboard setting, to view this data. + +#### MongoDB + +##### Mongo Pump + +**`mongo`** Pump simply saves all individual requests across every organization to a collection called **`tyk_analytics`**. Each request will be stored as a single document. + +###### Pump Config + +```yaml expandable +{ + ... + "pumps": { + "mongo": { + "type": "mongo", + "meta": { + "collection_name": "tyk_analytics", + "mongo_url": "mongodb://username:password@{hostname:port},{hostname:port}/{db_name}" + } + } +} +``` + +###### Capping +This collection [should be capped](/api-management/tyk-pump#capping-analytics-data) due to the number of individual documents. This is especially important if the `detailed_recording` in the Gateway is turned on which means that the Gateway records the full payload of the request and response. + +###### Omitting Indexes +From Pump 1.6+, the Mongo Pumps indexes default behavior is changed and the new configuration option `omit_index_creation` is available. This option is applicable to the following Pumps: `Mongo Pump`,`Mongo Aggregate Pump` and `Mongo Selective Pump`. + +The behavior now depends upon the value of 'omit_index_creation' and the Pump in use, as follows: + +- If `omit_index_creation` is set to `true`, tyk-pump will not create any indexes (for Mongo pumps). +- If `omit_index_creation` is set to `false` (default) and you are using `DocumentDB`, tyk-pump will create the Mongo indexes. +- If `omit_index_creation` is set to `false` (default) and you are using `MongoDB`, the behavior of tyk-pump depends upon whether the collection already exists: + - If the collection exists, tyk-pump will not create the indexes again. + - If the collection does not already exist, tyk-pump will create the indexes. + +###### Dashboard Setting + +In **API Usage Data > Log Browser** screen you will see all the individual requests that the Gateway has recorded and saved in `tyk_analytics` collection using the `mongo` pump. + +Because you have the option to store and display analytics of every organization or separately per organization, you need to configure the Tyk Dashboard with the matching setting according to the way you set the pump to store the data in MongoDB. +The field [use_sharded_analytics](/tyk-dashboard/configuration#use_sharded_analytics) controls the collection that the dashboard will query. +- If `use_sharded_analytics: false` - the dashboard will query the collection `tyk_analytics` that mongo pump populated +- If `use_sharded_analytics: true` - the dashboard will query the collection that `mongo-pump-selective` pump populated + + + +##### Mongo Aggregate Pump + +**`mongo-pump-aggregate`** pump stores data in a collection called `**z_tyk_analyticz_aggregate_{ORG ID}**`. + +###### Pump Config + +```yaml expandable +{ + ... + "pumps": { + "mongo-pump-aggregate": { + "name": "mongo-pump-aggregate", + "meta": { + "mongo_url": "mongodb://username:password@{hostname:port},{hostname:port}/{db_name}", + "use_mixed_collection": true + } + } + } +} +``` + +- `use_mixed_collection: true` - will store analytics to **both** your organization defined collections `z_tyk_analyticz_aggregate_{ORG ID}` and your org-less `tyk_analytics_aggregates` collection. +- `use_mixed_collection: false`- your pump will only store analytics to your org defined collection. + +`tyk_analytics_aggregates` collection is used to query analytics across your whole Tyk setup. This can be used, for example, by a superuser role that is not attached to an organization. When set to `true`, you also need to set [use_sharded_analytics](/tyk-dashboard/configuration#use_sharded_analytics) to true in your Dashboard config. + + +###### Dashboard Setting + +This pump supplies the data for the following sub categories **`API Usage Data`**: + +* Activity by API screen +* Activity by Key screen +* Errors screen + +As with the regular analytics, because Tyk gives you the option to store and display aggregated analytics across all organizations or separately per organization, you need to configure the Tyk Dashboard with the matching setting according to the way to set the pump to store the data in MongoDB, otherwise, you won't see the data in the Dashboard. + +1. The [enable_aggregate_lookups: true](/tyk-dashboard/configuration#enable_aggregate_lookups) field must be set in the Dashboard configuration file, in order for the Dashboard to query and display the aggregated data that `mongo-pump-aggregate` saved to MongoDB. + +###### Capping +As a minimal number of documents get stored, you don't need to worry about capping this. The documents contain aggregate info across an individual API, such as total requests, errors, tags and more. + +####### High Traffic Environment Settings + +If you have a high traffic environment, and you want to ignore aggregations to avoid Mongo overloading and/or reduce aggregation documents size, you can do it using the `ignore_aggregations` configuration option. The possible values are: +* APIID +* Errors +* Versions +* APIKeys +* OauthIDs +* Geo +* Tags +* Endpoints +* KeyEndpoint +* OauthEndpoint +* ApiEndpoint + +For example, if you want to ignore the API Keys aggregations: +```yaml expandable +pump.conf: + +{ + ... + "pumps": { + "mongo-pump-aggregate": { + "name": "mongo-pump-aggregate", + "meta": { + "mongo_url": "mongodb://username:password@{hostname:port},{hostname:port}/{db_name}", + "use_mixed_collection": true, + "ignore_aggregations": ["APIKeys"] + } + } + } +} +``` + +####### Unique Aggregation Points + +In case you set your API definition in the Tyk Gateway to tag unique headers (like `request_id` or timestamp), this collection can grow a lot since aggregation of unique values simply creates a record/document for every single value with a counter of 1. To mitigate this, avoid tagging unique headers as the first option. If you can't change the API definition quickly, you can add the tag to the ignore list `"ignore_aggregations": ["request_id"]`. This ensures that Tyk pump does not aggregate per `request_id`. +Also, if you are not sure what's causing the growth of the collection, you can also set time capping on these collections and monitor them. + + +##### Mongo Selective Pump + +**`mongo-pump-selective`** pump stores individual requests per organization in collections called **`z_tyk_analyticz_{ORG ID}`**. +Similar to the regular `mongo` pump, Each request will be stored as a single document. + +###### Pump Config + +This collection [should be capped](/api-management/tyk-pump#tyk-pump-capping-analytics-data-storage) due to the number of individual documents. +```yaml expandable +{ + ... + "pumps": { + "mongo-pump-selective": { + "name": "mongo-pump-selective", + "meta": { + "mongo_url": "mongodb://username:password@{hostname:port},{hostname:port}/{db_name}", + "use_mixed_collection": true + } + } + } +} +``` + +###### Capping + +This collection [should be capped](/api-management/tyk-pump#tyk-pump-capping-analytics-data-storage) due to the number of individual documents. + +###### Dashboard Setting + +As with the regular analytics, if you are using the Selective pump, you need to set `use_sharded_keys: true` in the dashboard config file so it will query `z_tyk_analyticz_{ORG ID}` collections to populate the `Log Browser`. + +##### Uptime Tests Analytics + +###### Pump Configuration + +```yaml +"uptime_pump_config": { + "collection_name": "tyk_uptime_analytics", + "mongo_url": "mongodb://tyk-mongo:27017/tyk_analytics", + }, +``` + +###### Tyk Dashboard Configuration + +```yaml expandable + β€œstorage” : { + ... + β€œuptime”: { + "type": "postgres", + "connection_string": "user=root password=admin database=tyk-demo-db host=tyk-db port=5432", + } + } +} +``` + +###### Tyk Gateway Setting + +To enable Uptime Pump, modify gateway configuration [enable_uptime_analytics](/tyk-oss-gateway/configuration#uptime_testsconfigenable_uptime_analytics) to true. + +#### SQL + +When using one of our [supported SQL platforms](/api-management/dashboard-configuration#supported-database), Tyk offers 3 types of SQL pumps: + +1. Aggregated Analytics: `sql_aggregate` +2. Raw Logs Analytics: `sql` +3. Uptime Tests Analytics + +In a production environment, we recommend sharding. You can configure your analytics in the following ways: + +* Sharding **raw logs** +* Sharding **aggregated analytics** +* Sharding **uptime tests** + +##### SQL Pump + +While aggregated analytics offer a decent amount of details, there are use cases when you’d like to have access to all request details in your analytics. For that you can generate analytics based on raw logs. This is especially helpful when, once you have all the analytics generated based on raw logs stored in your SQL database, you can then build your own custom metrics, charts etc. outside of your Tyk Dashboard, which may bring more value to your product. + +The pump needed for storing log data in the database is very similar to other pumps as well as the storage setting in the Tyk Dashboard config. It just requires the SQL name and database-specific configuration options. + +###### SQL Pump Configuration + +For storing logs into the `tyk_analytics` database table. + +```yaml expandable +"sql": { + "name": "sql", + "meta": { + "type": "postgres", + "connection_string": "host=localhost port=5432 user=admin dbname=postgres_test password=test", + "table_sharding": false + } +} +``` +`type` - The supported types are `sqlite` and `postgres`. + +`connection_string` - Specifies the connection string to the database. For example, for `sqlite` it will be the path/name of the database, and for `postgres`, specifying the host, port, user, password, and dbname. + +`log_level` - Specifies the SQL log verbosity. The possible values are: `info`,`error` and `warning`. By default, the value is `silent`, which means that it won't log any SQL query. + +`table_sharding` - Specifies if all the analytics records are going to be stored in one table or in multiple tables (one per day). By default, it is set to `false`. + +If `table_sharding` is `false`, all the records are going to be stored in the `tyk_analytics` table. If set to `true`, daily records are stored in a `tyk_analytics_YYYYMMDD` date formatted table. + +###### Dashboard Setting + +In the **API Usage Data > Log Browser** screen you will see all the individual requests that the Gateway has recorded and saved in `tyk_analytics` collection using the `sql` pump. + +Make sure you have configured the dashboard with your SQL database connection settings: + +```yaml expandable +{ + ... + "storage" : { + ... + "analytics": { + "type": "postgres", + "connection_string": "user=root password=admin host=tyk-db database=tyk-demo-db port=5432", + } + } +} +``` + +##### SQL Aggregate Pump + +This is the default option offered by Tyk, because it is configured to store the most important analytics details which will satisfy the needs of most of our clients. This allows your system to save database space and reporting is faster, consuming fewer resources. + +###### SQL Aggregate Pump Configuration + +For storing logs into the `tyk_aggregated` database table. + +```yaml expandable +"sql_aggregate": { + "name": "sql_aggregate", + "meta": { + "type": "postgres", + "connection_string": "host=localhost port=5432 user=admin dbname=postgres_test password=test", + "table_sharding": true + } +} +``` + +`type` - The supported types are `sqlite` and `postgres`. + +`connection_string` - Specifies the connection string to the database. For example, for `sqlite` it will be the path/name of the database, and for `postgres`, specifying the host, port, user, password, and dbname. + +`log_level` - Specifies the SQL log verbosity. The possible values are: `info`, `error`, and `warning`. By default, the value is `silent`, which means that it won't log any SQL query. + +`track_all_paths` - Specifies if it should store aggregated data for all the endpoints. By default, it is set to `false`, which means that it only stores aggregated data for `tracked endpoints`. + +`ignore_tag_prefix_list` - Specifies prefixes of tags that should be ignored. + +`table_sharding` - Specifies if all the analytics records are going to be stored in one table or in multiple tables (one per day). By default, it is set to `false`. + +If `table_sharding` is `false`, all the records are going to be stored in the `tyk_aggregated` table. If set to `true`, daily records are stored in a `tyk_aggregated_YYYYMMDD` date formatted table. + +###### Dashboard Setting + +This pump supplies the data for the following sub categories **`API Usage Data`**: + +* Activity by API screen +* Activity by Key screen +* Errors screen + +As with the regular analytics, because Tyk gives you the option to store and display aggregated analytics across all organizations or separately per organization, you need to configure the Tyk Dashboard with the matching set according to the way to set the pump to store the data in SQL, otherwise, you won't see the data in the Dashboard. + +1. The [enable_aggregate_lookups: true](/tyk-dashboard/configuration#enable_aggregate_lookups) field must be set in the Dashboard configuration file, in order for the Dashboard to query and display the aggregated data that `sql-aggregate` saved to the database. + +2. Make sure you have configured the dashboard with your SQL database connection settings: + +```yaml expandable +{ + ... + "storage": { + ... + "analytics": { + "type": "postgres", + "connection_string": "user=root password=admin host=tyk-db database=tyk-demo-db port=5432", + } + } +} +``` + +##### SQL Uptime Pump + +In an `uptime_pump_config` section, you can configure a SQL uptime pump. To do that, you need to add the field `uptime_type` with `sql` value. + +```yaml expandable +"uptime_pump_config": { + "uptime_type": "sql", + "type": "postgres", + "connection_string": "host=sql_host port=sql_port user=sql_usr dbname=dbname password=sql_pw", + "table_sharding": false +}, +``` +`type` - The supported types are `sqlite` and `postgres`. + +`connection_string` - Specifies the connection string to the database. For example, for `sqlite` it will be the path/name of the database, and for `postgres`, specifying the host, port, user, password, and dbname. + +`table_sharding` - Specifies if all the analytics records will be stored in one table or multiple tables (one per day). By default, it is set to `false`. + +If `table_sharding` is `false`, all the records will be stored in the `tyk_analytics` table. If set to `true`, daily records are stored in a `tyk_analytics_YYYYMMDD` date formatted table. + +###### Tyk Dashboard Configuration + +You need to set `enable_aggregate_lookups` to `false` + +Then add your SQL database connection settings: + +```yaml expandable +{ + ... + β€œstorage” : { + ... + β€œanalytics”: { + "type": "postgres", + "connection_string": "user=root password=admin host=tyk-db database=tyk-demo-db port=5432", + } + } +} +``` + +##### Uptime Tests Analytics + +###### Tyk Pump Configuration + +For storing logs into the `tyk_aggregated` database table. + +```yaml +"uptime_pump_config": { + "uptime_type": "sql", + "type": "postgres", + "connection_string": "host=sql_host port=sql_port user=sql_usr database=tyk-demo-db password=sql_pw", +}, +``` + +###### Tyk Dashboard Configuration + +```{.shell} expandable + β€œstorage” : { + ... + β€œuptime”: { + "type": "postgres", + "connection_string": "user=root password=admin database=tyk-demo-db host=tyk-db port=5432", + } + } +} +``` + +###### Tyk Gateway Setting + +To enable Uptime Pump, modify gateway configuration [enable_uptime_analytics](/tyk-oss-gateway/configuration#uptime_testsconfigenable_uptime_analytics) to true. + +##### Sharding + +In a production environment, we recommend the following setup: + +By default, all logs/analytics are stored in one database table, making it hard and less performant to execute CRUD operations on the dataset when it grows significantly. + +To improve the data maintenance processes, as querying or removing data from one single table is slow, we have added a new option (`table_sharding`), so that the data can be stored daily (one table of data per day), which will automatically make querying or removing sets of data easier, whether dropping tables for removing logs/analytics, or reading multiple tables based on the selected period. + +###### Tyk Pump Configuration + +```yaml expandable +"sql": { + ... + "meta": { + ... + "table_sharding": true + } +}, +"sql_aggregate" : { + ... + "meta": { + ... + "table_sharding": true + } +}, +"uptime_pump_config": { + ... + "table_sharding": true +}, +``` + +###### Tyk Dashboard Configuration + +```yaml expandable + "storage": { + "main": { + ... + "table_sharding": true + }, + "analytics": { + ... + "table_sharding": true + }, + "logs": { + ... + "table_sharding": true + }, + "uptime": { + ... + "table_sharding": true + } + }, +``` + +### Graph Pump setup + +#### MongoDB + +Starting with version `1.7.0` of Tyk Pump and version `4.3.0` of Tyk Gateway it is possible to configure Graph MongoDB Pump. Once configured, the pump enables support for Graphql-specific metrics. The Graphql-specific metrics currently supported include (more to be added in future versions ): + +* Types Requested. +* Fields requested for each type. +* Error Information (not limited to HTTP status codes). + +##### Setting up Graph MongoDB Pump + +1. Set `enable_analytics` to `true` in your `tyk.conf`. +2. Enable Detailed recording by setting `enable_detailed_recording` in your `tyk.conf` to `true`. This is needed so that the GraphQL information can be parsed from the request body and response. + + + + + This will enable detailed recording globally, across all APIs. This means that the behavior of individual APIs that have this configuration parameter set will be overridden. The Gateway must be restarted after updating this configuration parameter. + + + +3. Set up your Mongo `collection_name`. +4. Add your Graph MongoDB Pump configuration to the list of pumps in your `pump.conf` (pump configuration file). + +Sample setup: + +``` expandable +{ + ... + "pumps": { + ... + "mongo-graph": { + "meta": { + "collection_name": "tyk_graph_analytics", + "mongo_url": "mongodb://mongo/tyk_graph_analytics" + } + }, + } +} +``` + +##### Current limitations + +The Graph MongoDB Pump is being improved upon regularly and as such there are a few things to note about the Graph MongoDB Pump current behavior: + +* Size of your records - due to the detailed recording being needed for this Pump’s to function correctly, it is important to note that your records and consequently, your MongoDB storage could increase in size rather quickly. +* Subgraph requests are not recorded - Requests to tyk-controlled subgraphs from supergraphs in federation setting are currently not recorded by the Graph MongoDB Pump, just the supergraph requests are handled by the Graph MongoDB Pump. +* UDG requests are recorded but subsequent requests to data sources are currently ignored. +* Currently, Graph MongoDB Pump data can not be used in Tyk Dashboard yet, the data is only stored for recording purposes at the moment and can be exported to external tools for further analysis. + +#### SQL + +Starting with Version `1.8.0` of Tyk Pump and version `5.0.0` of the Tyk Gateway; It is possible to export GraphQL analytics to an SQL database. + +##### Setting up Graph SQL Pump + +With the Graph SQL pump currently includes information (per request) like: +- Types Requested +- Fields requested for each type +- Error Information +- Root Operations Requested. + + Setup steps include: +1. Set `enable_anaytics` to `true` in your `tyk.conf`. +2. Enable Detailed recording by setting `enable_detailed_recording` in your `tyk.conf` to `true`. This is needed so that the GraphQL information can be parsed from the request body and response. + + + + + This will enable detailed recording globally, across all APIs. This means that the behavior of individual APIs that have this configuration parameter set will be overridden. The Gateway must be restarted after updating this configuration parameter. + + + +3. Configure your `pump.conf` using this sample configuration: +``` expandable +"sql-graph": { + "meta": { + "type": "postgres", + "table_name": "tyk_analytics_graph", + "connection_string": "host=localhost user=postgres password=password dbname=postgres", + "table_sharding": false + } +}, +``` +The Graph SQL pump currently supports `postgres`, `sqlite` and `mysql` databases. The `table_name` refers to the table that will be created in the case of unsharded setups, and the prefix that will be used for sharded setups +e.g `tyk_analytics_graph_20230327`. + +>The Graph SQL pump currently has the same limitations as the Graph Mongo Pump. + +##### Setting up Graph SQL Aggregate Pump +The `sql-graph-aggregate` can be configured similar to the Graph SQL pump: +``` expandable + "sql-graph-aggregate": { + "meta": { + "type": "postgres", + "connection_string": "host=localhost port=5432 user=postgres dbname=postgres password=password", + "table_sharding": false + } +} +``` + + +## External Data Stores + +The Tyk Pump component takes all of the analytics in Tyk and moves the data from the Gateway into your Dashboard. It is possible to set it up to send the analytics data it finds to other data stores. Currently we support the following: + +- MongoDB or SQL (Used by the Tyk Dashboard) +- [CSV](/api-management/tyk-pump#csv) +- [Elasticsearch (2.0 - 7.x)](/api-management/tyk-pump#elasticsearch) +- Graylog +- Resurface.io +- InfluxDB +- [Moesif](/api-management/tyk-pump#moesif) +- [Splunk](/api-management/tyk-pump#splunk) +- StatsD +- DogStatsD +- Hybrid (Tyk RPC) +- [Prometheus](/api-management/tyk-pump#monitor-your-apis-with-prometheus) +- [Logz.io](/api-management/tyk-pump#logzio) +- Kafka +- Syslog (FluentD) + +See the [Tyk Pump Configuration](/tyk-pump) for more details. + +### CSV + +Tyk Pump can be configured to create or modify a CSV file to track API Analytics. + +#### JSON / Conf file + +Add the following configuration fields to the pumps section within your `pump.conf` file: + +```json expandable +{ + "csv": + { + "type": "csv", + "meta": { + "csv_dir": "./your_directory_here" + } + } +} +``` + +#### Environment variables +```bash +TYK_PMP_PUMPS_CSV_TYPE=csv +TYK_PMP_PUMPS_CSV_META_CSVDIR=./your_directory_here +``` + +### Datadog + +The Tyk Pump can be configured to send your API traffic analytics to [Datadog](https://www.datadoghq.com/) with which you can build a [dashboards](https://docs.datadoghq.com/integrations/tyk/#dashboards) with various metrics based on your API traffic in Tyk. + +#### Datadog dashboard example + +We ceated a defaulkt Tyk dashboard canvat to give our users an easier starting point. You can find it in Datadog portal, under the `Dashboards --> lists` section, (https://app.datadoghq.com/dashboard/lists)[https://app.datadoghq.com/dashboard/lists], and it is called `Tyk Analytics Canvas`. To use this dashboard you will need to make sure that your datadog agent deployment has the following tag `env:tyk-demo-env` and that your Tyk Pump configuration has `dogstatsd.meta.namespace` set to `pump`. You can also import it from [Datadog official GH repo](https://github.com/DataDog/integrations-extras/blob/master/tyk/assets/dashboards/tyk_analytics_canvas.json) and change those values in the dashboard itself to visualize your analytics data as it flows into Datadog. + +Sample Datadog dashboard + + +#### Prerequisites + +- A working Datadog agent installed on your Environment. See the [Datadog Tyk integration docs](https://docs.datadoghq.com/integrations/tyk/) for more information. +- Either a [Tyk Pro install](/tyk-self-managed/install) or [Tyk OSS Gateway install](/apim/open-source/installation) along with a [Tyk Pump](/tyk-pump) install. + +#### How it works + +When running the Datadog Agent, [DogstatsD](https://github.com/TykTechnologies/tyk-pump#dogstatsd) gets the [request_time](https://docs.datadoghq.com/integrations/tyk/#data-collected) metric from your Tyk Pump in real time, per request, so you can understand the usage of your APIs and get the flexibility of aggregating by various parameters such as date, version, returned code, method etc. + +#### Tyk Pump configuration + +Below is a sample DogstatD section from a Tyk `pump.conf` file + +```json expandable +"dogstatsd": { + "type": "dogstatsd", + "meta": { + "address": "dd-agent:8126", + "namespace": "tyk", + "async_uds": true, + "async_uds_write_timeout_seconds": 2, + "buffered": true, + "buffered_max_messages": 32, + "sample_rate": 0.9999999999, + "tags": [ + "method", + "response_code", + "api_version", + "api_name", + "api_id", + "org_id", + "tracked", + "path", + "oauth_id" + ] + } +}, +``` + +##### Field descriptions + +- `address`: address of the datadog agent including host & port +- `namespace`: prefix for your metrics to datadog +- `async_uds`: Enable async [UDS over UDP](https://github.com/Datadog/datadog-go#unix-domain-sockets-client) +- `async_uds_write_timeout_seconds`: Integer write timeout in seconds if `async_uds: true` +- `buffered`: Enable buffering of messages +- `buffered_max_messages`: Max messages in single datagram if `buffered: true`. Default 16 +- `sample_rate`: default 1 which equates to 100% of requests. To sample at 50%, set to 0.5 +- `tags`: List of tags to be added to the metric. The possible options are listed in the below example + +If no tag is specified the fallback behavior is to use the below tags: +- `path` +- `method` +- `response_code` +- `api_version` +- `api_name` +- `api_id` +- `org_id` +- `tracked` +- `oauth_id` + +Note that this configuration can generate significant data due to the unbound nature of the `path` tag. + + +On startup, you should see the loaded configs when initialising the DogstatsD pump +```console +[May 10 15:23:44] INFO dogstatsd: initializing pump +[May 10 15:23:44] INFO dogstatsd: namespace: pump. +[May 10 15:23:44] INFO dogstatsd: sample_rate: 50% +[May 10 15:23:44] INFO dogstatsd: buffered: true, max_messages: 32 +[May 10 15:23:44] INFO dogstatsd: async_uds: true, write_timeout: 2s +``` + + +### Elasticsearch + +[Elasticsearch](https://www.elastic.co/) is a highly scalable and distributed search engine that is designed to handle large amounts of data. + + +#### JSON / Conf + +Add the following configuration fields to the pumps section within your `pump.conf` file: + +```json expandable +{ + "pumps": { + "elasticsearch": { + "type": "elasticsearch", + "meta": { + "index_name": "tyk_analytics", + "elasticsearch_url": "http://localhost:9200", + "enable_sniffing": false, + "document_type": "tyk_analytics", + "rolling_index": false, + "extended_stats": false, + "version": "6" + } + } + } +} +``` + +#### Configuration fields +- `index_name`: The name of the index that all the analytics data will be placed in. Defaults to `tyk_analytics` +- `elasticsearch_url`: If sniffing is disabled, the URL that all data will be sent to. Defaults to `http://localhost:9200` +- `enable_sniffing`: If sniffing is enabled, the `elasticsearch_url` will be used to make a request to get a list of all the nodes in the cluster, the returned addresses will then be used. Defaults to `false` +- `document_type`: The type of the document that is created in Elasticsearch. Defaults to `tyk_analytics` +- `rolling_index`: Appends the date to the end of the index name, so each days data is split into a different index name. For example, `tyk_analytics-2016.02.28`. Defaults to `false`. +- `extended_stats`: If set to true will include the following additional fields: `Raw Request`, `Raw Response` and `User Agent`. +- `version`: Specifies the ES version. Use `3` for ES 3.X, `5` for ES 5.X, `6` for ES 6.X, `7` for ES 7.X . Defaults to `3`. +- `disable_bulk`: Disable batch writing. Defaults to `false`. +- `bulk_config`: Batch writing trigger configuration. Each option is an OR with each other: + - `workers`: Number of workers. Defaults to `1`. + - `flush_interval`: Specifies the time in seconds to flush the data and send it to ES. Default is disabled. + - `bulk_actions`: Specifies the number of requests needed to flush the data and send it to ES. Defaults to 1000 requests. If it is needed, can be disabled with `-1`. + - `bulk_size`: Specifies the size (in bytes) needed to flush the data and send it to ES. Defaults to 5MB. Can be disabled with `-1`. + + +#### Environment variables +```bash expandable +TYK_PMP_PUMPS_ELASTICSEARCH_TYPE=elasticsearch +TYK_PMP_PUMPS_ELASTICSEARCH_META_INDEXNAME=tyk_analytics +TYK_PMP_PUMPS_ELASTICSEARCH_META_ELASTICSEARCHURL=http://localhost:9200 +TYK_PMP_PUMPS_ELASTICSEARCH_META_ENABLESNIFFING=false +TYK_PMP_PUMPS_ELASTICSEARCH_META_DOCUMENTTYPE=tyk_analytics +TYK_PMP_PUMPS_ELASTICSEARCH_META_ROLLINGINDEX=false +TYK_PMP_PUMPS_ELASTICSEARCH_META_EXTENDEDSTATISTICS=false +TYK_PMP_PUMPS_ELASTICSEARCH_META_VERSION=5 +TYK_PMP_PUMPS_ELASTICSEARCH_META_BULKCONFIG_WORKERS=2 +TYK_PMP_PUMPS_ELASTICSEARCH_META_BULKCONFIG_FLUSHINTERVAL=60 +``` + +### Moesif + +This is a step by step guide to setting up [Moesif API Analytics and Monetization platform](https://www.moesif.com/solutions/track-api-program?language=tyk-api-gateway&utm_medium=docs&utm_campaign=partners&utm_source=tyk) to understand [customer API usage](https://www.moesif.com/features/api-analytics?utm_medium=docs&utm_campaign=partners&utm_source=tyk) and [setup usage-based billing](https://www.moesif.com/solutions/metered-api-billing?utm_medium=docs&utm_campaign=partners&utm_source=tyk). + +We also have a [blog post](https://tyk.io/tyk-moesif-the-perfect-pairing/) which highlights how Tyk and Moesif work together. + +The assumptions are that you have Docker installed and Tyk Self-Managed already running. +See the [Tyk Pump Configuration](/tyk-pump) for more details. + + +#### Overview + +With the Moesif Tyk plugin, your API logs are sent to Moesif asynchronously to provide analytics on customer API usage along with your API payloads like JSON and XML. This plugin also enables you to monetize your API with [billing meters](https://www.moesif.com/solutions/metered-api-billing?utm_medium=docs&utm_campaign=partners&utm_source=tyk) and provide a self-service onboarding experience. Moesif also collects information such as the authenticated user (AliasId or OAuthId) to identify customers using your API. An overview on how Moesif and Tyk works together is [available here](https://tyk.io/tyk-moesif-the-perfect-pairing/). + +#### Steps for Configuration + +1. **Get a Moesif Application Id** + + Go to [www.moesif.com](https://www.moesif.com/?language=tyk-api-gateway) and sign up for a free account. + Application Ids are write-only API keys specific to an application in Moesif such as β€œDevelopment” or β€œProduction”. You can always create more applications in Moesif. + +2. **Enable Moesif backend in Tyk Pump** + + Add Moesif as an analytics backend along with your Moesif Application Id you obtained in the last step to your [Tyk Pump](https://github.com/TykTechnologies/tyk-pump) Configuration + +####### JSON / Conf File +```json expandable +{ + "pumps": { + "moesif": { + "name": "moesif", + "meta": { + "application_id": "Your Moesif Application Id" + } + } + } +} +``` + +####### Env Variables: +``` +TYK_PMP_PUMPS_MOESIF_TYPE=moesif +TYK_PMP_PUMPS_MOESIF_META_APPLICATIONID=your_moesif_application_id +``` + +3. **Ensure analytics is enabled** + +If you want to log HTTP headers and body, ensure that [detailed analytics recording](/api-management/logs-metrics#capturing-detailed-logs) is enabled true in your [Tyk Gateway Conf](/tyk-oss-gateway/configuration) + +####### JSON / Conf File + +```json expandable +{ + "enable_analytics" : true, + "analytics_config": { + "enable_detailed_recording": true + } +} +``` + +####### Env Variables: +```conf +TYK_GW_ENABLEANALYTICS=true +TYK_GW_ANALYTICSCONFIG_ENABLEDETAILEDRECORDING=true +``` + + +This will enable detailed recording globally, across all APIs. This means that the behavior of individual APIs that have this configuration parameter set will be overridden. The Gateway must be restarted after updating this configuration parameter. + + + +4. **Restart Tyk Pump to pickup the Moesif config** + +Once your config changes are done, you need to restart your Tyk Pump and Tyk Gateway instances (if you've modified Tyk gateway config). +If you are running Tyk Pump in Docker: + +`$ docker restart tyk-pump` + +5. **PROFIT!** + +You can now make a few API calls and verify they show up in Moesif. + +```bash +$ curl localhost:8080 +``` +Step5 + +The Moesif Tyk integration automatically maps a [Tyk Token Alias](https://tyk.io/blog/simpler-usage-tracking-token-aliases-tyk-cloud/) to a user id in Moesif. With a Moesif SDK, you can store additional customer demographics to break down API usage by customer email, company industry, and more. + +#### Configuration options + +The Tyk Pump for Moesif has a few configuration options that can be set in your `pump.env`: + +|Parameter|Required|Description|Environment Variable| +| :--------- | :--------- | :----------- | :----------- | +|application_id|required|Moesif Application Id. Multiple Tyk api_id's will be logged under the same app id.|TYK_PMP_PUMPS_MOESIF_META_APPLICATIONID| +|request_header_masks|optional|Mask a specific request header field. Type: String Array [] string|TYK_PMP_PUMPS_MOESIF_META_REQUESTHEADERMASKS| +|request_body_masks|optional|Mask a specific - request body field. Type: String Array [] string| TYK_PMP_PUMPS_MOESIF_META_REQUESTBODYMASKS | +|response_header_masks|optional|Mask a specific response header field. Type: String Array [] string|TYK_PMP_PUMPS_MOESIF_META_RESPONSEHEADERMASKS| +|response_body_masks|optional|Mask a specific response body field. Type: String Array [] string|TYK_PMP_PUMPS_MOESIF_META_RESPONSEBODYMASKS| +|disable_capture_request_body|optional|Disable logging of request body. Type: Boolean. Default value is false.|TYK_PMP_PUMPS_MOESIF_META_DISABLECAPTUREREQUESTBODY| +|disable_capture_response_body|optional|Disable logging of response body. Type: Boolean. Default value is false.|TYK_PMP_PUMPS_MOESIF_META_DISABLECAPTURERESPONSEBODY| +|user_id_header|optional|Field name to identify User from a request or response header. Type: String. Default maps to the token alias|TYK_PMP_PUMPS_MOESIF_META_USERIDHEADER| +|company_id_header|optional|Field name to identify Company (Account) from a request or response header. Type: String|TYK_PMP_PUMPS_MOESIF_META_COMPANYIDHEADER| + +#### Identifying users +By default, the plugin will collect the authenticated user (AliasId or OAuthId) to identify the customer. This can be overridden by setting the `user_id_header` to a header that contains your API user/consumer id such as `X-Consumer-Id`. You can also set the `company_id_header` which contains the company to link the user to. [See Moesif docs on identifying customers](https://www.moesif.com/docs/getting-started/identify-customers/?utm_medium=docs&utm_campaign=partners&utm_source=tyk) + + +### Splunk + +This is a step by step guide to setting Splunk to receive logs from the Tyk Pump. + +The assumptions are that you have Docker installed and Tyk Pro Self-Managed already running. + +#### Steps for Configuration + +1. **Run Splunk using Docker** + + Assuming you have Docker installed locally, run the following from a terminal: + + ```{.copyWrapper} expandable + $ docker run \ + -p 8000:8000 \ + -p 8088:8088 \ + -v splunk-data:/opt/splunk/var \ + -v splunk-data:/opt/splunk/etc \ + -e SPLUNK_START_ARGS=--accept-license \ + -e SPLUNK_PASSWORD=mypassword \ + splunk/splunk:latest + ``` + +2. **Setup a collector in Splunk** + + A) Visit http://localhost:8000 and log into the Splunk Dashboard using the username `admin` and the password we set in the Docker run command, `mypassword` + + B) Create a new Data input + Step1 + + C) Select `HTTP Event Collector -> Add New` + Step2 + + D) Set the name to "tyk" and then leave everything else as default + Step2b + + Grab your token at the end page: + Step3 + +3. **Add the Splunk bit to pump.conf** + + Edit your pump's `pump.conf` and add this bit to the "Pumps" section, like so, adding the token from step #1: + + Make sure to add your token from the previous step into the `collector_token` field above + + ```json + { + "pumps": { + "splunk": { + "type": "splunk", + "meta": { + "collector_token": "", + "collector_url": "https://localhost:8088/services/collector/event", + "ssl_insecure_skip_verify": true + } + } + } + } + ``` + + +Make sure that the `localhost` value matches with your setup. Head on over to our [community forum](https://community.tyk.io/) to ask for help if you are stuck here. + + + +4. **Restart Tyk Pump to pickup the Splunk config** + + If you are running Tyk Pump in Docker: + + `$ docker restart tyk-pump` + +5. **PROFIT!** + + Let's make a few API calls against Tyk, and see if they flow into Splunk + + ```bash + $ curl localhost:8080/loan-service-api/ + + { + "error": "Key not authorized" + }% + ``` + + Success: + Step4 + +### Logzio + +[Logz.io](https://logz.io/) is a cloud-based log management and analytics platform that provides log management built on [Elasticsearch](https://www.elastic.co/), [Logstash](https://www.elastic.co/guide/en/logstash/current/index.html) and [Kibana](https://www.elastic.co/kibana/). + + +#### JSON / Conf file + +Add the following configuration fields to the pumps section within your `pump.conf` file: + +```json +{ + "pumps" + { + "logzio": { + "type": "logzio", + "meta": { + "token": "" + } + } + } +} +``` + +#### Environment variables +```bash +TYK_PMP_PUMPS_LOGZIO_TYPE=logzio +TYK_PMP_PUMPS_LOGZIO_META_TOKEN="{YOUR-LOGZIO-TOKEN}" +``` expandable + +#### Advanced configuration fields +- `meta.url`: Use if you do not want to use the default Logz.io URL, for example when using a proxy. The default url is `https://listener.logz.io:8071`. +- `meta.queue_dir`: The directory for the queue. +- `meta.drain_duration`: This sets the drain duration (when to flush logs on the disk). The default value is `3s`. +- `meta.disk_threshold`: Set the disk queue threshold. Once the threshold is crossed the sender will not enqueue the received logs. The default value is `98` (percentage of disk). +- `meta.check_disk_space`: Set the sender to check if it crosses the maximum allowed disk usage. The default value is `true`. + +## Tyk Analytics Record Fields + +Below is a detailed list of each field contained within our Tyk Analytics Record that is sent from Tyk Pump. + +### Method +Request method. + +**Example:** `GET`, `POST`. + +### Host +Request `Host` header. + +**Remarks:** Includes host and optional port number of the server to which the request was sent. +**Example:** `tyk.io`, or `tyk.io:8080` if port is included. + +### Path +Request path. + +**Remarks:** Displayed in decoded form.
+**Example:** `/foo/bar` for `/foo%2Fbar` or `/foo/bar`. + +### RawPath +Request path. + +**Remarks:** Original request path without changes just decoded.
+**Example:** `/foo/bar` for `/foo%2Fbar` or `/foo/bar`. + +### ContentLength +Request `Content-Length` header. + +**Remarks:** The number of bytes in the request body.
+**Example:** `10` for request body `0123456789`. + +### UserAgent +Request `User-Agent` header. + +**Example:** `curl/7.86.0`. + +### Day +Request day. + +**Remarks:** Based on `TimeStamp` field.
+**Example:** `16` for `2022-11-16T03:01:54Z`. + +### Month +Request month. + +**Remarks:** Based on `TimeStamp` field.
+**Example:** `11` for `2022-11-16T03:01:54Z`. + +### Year +Request year. + +**Remarks:** Based on `TimeStamp` field. +**Example:** `2022` for `2022-11-16T03:01:54Z`. + +### Hour +Request hour. + +**Remarks:** Based on `TimeStamp` field.
+**Example:** `3` for `2022-11-16T03:01:54Z`. + +### ResponseCode +Response code. + +**Remarks:** Only contains the integer element of the response code. Can be generated by either the gateway or upstream server, depending on how the request is handled.
+**Example:** `200` for `200 OK`. + +### APIKey +Request authentication key. + +**Remarks:** OAuthentication key, as provided in request. If no API key is provided then gateway will substitute a default value.
+**Example:** Unhashed `auth_key`, hashed `6129dc1e8b64c6b4`, or `00000000` if no authentication provided. + +### TimeStamp +Request timestamp. + +**Remarks:** Generated by the gateway, based on the time it receives the request from the client.
+**Example:** `2022-11-16T03:01:54.648+00:00`. + +### APIVersion +Version of API Definition requested. + +**Remarks:** Based on version configuration of context API definition. If API is unversioned then value is "Not Versioned".
+**Example:** Could be an alphanumeric value such as `1` or `b`. Is `Not Versioned` if not versioned. + +### APIName +Name of API Definition requested. + +**Example:** `Foo API`. + +### APIID +Id of API Definition requested. + +**Example:** `727dad853a8a45f64ab981154d1ffdad`. + +### OrgID +Organization Id of API Definition requested. + +**Example:** `5e9d9544a1dcd60001d0ed20`. + +### OauthID +Id of OAuth client. + +**Remarks:** Value is empty string if not using OAuth, or OAuth client not present.
+**Example:** `my-oauth-client-id`. + +### RequestTime +Duration of upstream roundtrip. + +**Remarks:** Equal to value of `Latency.Total` field. +**Example:** `3` for a 3ms roundtrip. + +### RawRequest +Raw HTTP request. + +**Remarks:** Base64 encoded copy of the request sent from the gateway to the upstream server.
+**Example:** `R0VUIC9nZXQgSFRUUC8xLjEKSG9zdDogdHlrLmlv`. + +### RawResponse +Raw HTTP response. + +**Remarks:** Base64 encoded copy of the response sent from the gateway to the client.
+**Example:** `SFRUUC8xLjEgMjAwIE9LCkNvbnRlbnQtTGVuZ3RoOiAxOQpEYXRlOiBXZWQsIDE2IE5vdiAyMDIyIDA2OjIxOjE2IEdNVApTZXJ2ZXI6IGd1bmljb3JuLzE5LjkuMAoKewogICJmb28iOiAiYmFyIgp9Cg==`. + +### IPAddress +Client IP address. + +**Remarks:** Taken from either `X-Real-IP` or `X-Forwarded-For` request headers, if set. Otherwise, determined by gateway based on request.
+**Example:** `172.18.0.1`. + +### Geo +Client geolocation data. + +**Remarks:** Calculated using MaxMind database, based on client IP address.
+**Example:** `{"country":{"isocode":"SG"},"city":{"geonameid":0,"names":{}},"location":{"latitude":0,"longitude":0,"timezone":""}}`. + +### Network +Network statistics. + +**Remarks:** Not currently used. + +### Latency +Latency statistics + +**Remarks:** Contains two fields; `upstream` is the roundtrip duration between the gateway sending the request to the upstream server and it receiving a response. `total` is the `upstream` value plus additional gateway-side functionality such as processing analytics data.
+**Example:** `{"total":3,"upstream":3}`. + + + +We record the round trip time of the call from the gateways reverse proxy. So what you get is the sum of `leaving Tyk -> upstream -> response received back at Tyk`. + + + +### Tags +Session context tags. + +**Remarks:** Can contain many tags which refer to many things, such as the gateway, API key, organization, API definition etc.
+**Example:** `["key-00000000","org-5e9d9544a1dcd60001d0ed20","api-accbdd1b89e84ec97f4f16d4e3197d5c"]`. + +### Alias +Session alias. + +**Remarks:** Alias of the context authenticated identity. Blank if no alias set or request is unauthenticated.
+**Example:** `my-key-alias`. + +### TrackPath +Tracked endpoint flag. + +**Remarks:** Value is `true` if the requested endpoint is configured to be tracked, otherwise `false`.
+**Example:** `true` or `false`. + +### ExpireAt +Future expiry date. + +**Remarks:** Can be used to implement automated data expiry, if supported by storage.
+**Example:** `2022-11-23T07:26:25.762+00:00`. + +## Monitor your APIs with Prometheus + +Your Tyk Pump can expose Prometheus metrics for the requests served by your Tyk Gateway. This is helpful if you want to track how often your APIs are being called and how they are performing. Tyk collects latency data of how long your services take to respond to requests, how often your services are being called and what status code they return. + +We have created a [demo project in GitHub](https://github.com/TykTechnologies/demo-slo-prometheus-grafana) if you want to see this setup in action. + +### Prerequisites + +- A Tyk installation (either Self-Managed or Open Source Gateway) +- Tyk Pump 1.6 or higher + +### Configure Tyk Pump to expose Prometheus metrics + +Prometheus collects metrics from targets by scraping metrics HTTP endpoints. To expose Tyk’s metrics in the Prometheus format, you need to add the following lines to your Tyk Pump configuration file `pump.conf`: + +#### Host + +```.copyWriter +"prometheus": { + "type": "prometheus", + "meta": { + "listen_address": ":9090", + "path": "/metrics", + "custom_metrics":[ + { + "name":"tyk_http_requests_total", + "description":"Total of API requests", + "metric_type":"counter", + "labels":["response_code","api_name","method","api_key","alias","path"] + }, + { + "name":"tyk_http_latency", + "description":"Latency of API requests", + "metric_type":"histogram", + "labels":["type","response_code","api_name","method","api_key","alias","path"] + } + ] + } +} +``` + +Replace `` with your host name or IP address. + +#### Docker + +```.copyWrapper +"prometheus": { + "type": "prometheus", + "meta": { + "listen_address": ":9090", + "path": "/metrics", + "custom_metrics":[ + { + "name":"tyk_http_requests_total", + "description":"Total of API requests", + "metric_type":"counter", + "labels":["response_code","api_name","method","api_key","alias","path"] + }, + { + "name":"tyk_http_latency", + "description":"Latency of API requests", + "metric_type":"histogram", + "labels":["type","response_code","api_name","method","api_key","alias","path"] + } + ] + } +} +``` + +Port 9090 needs also to be exported by Docker in addition to the port used for health check (here 8083), e.g. with Docker compose: + +```.copyWrapper +tyk-pump: + image: tykio/tyk-pump-docker-pub:${PUMP_VERSION} + ports: + - 8083:8083 + - 9090:9090 +``` expandable + +Restart your Pump to apply to configuration change. + +Verify that the metrics are being exposed by calling the metrics endpoint `http://:9090` from your browser. + +### Configure Prometheus to scrape the metrics endpoint + +Prometheus is configured via a [configuration file](https://prometheus.io/docs/prometheus/latest/configuration/configuration/) where you can define the metrics endpoint Prometheus will scrape periodically. + +Here’s an example configuration scraping Tyk Pump metric endpoints: + +#### Host + +```.copyWrapper +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: tyk + static_configs: + - targets: ['tyk-pump:9090'] +``` +#### Docker + +```.copyWrapper +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: tyk + static_configs: + - targets: ['host.docker.internal:9090'] +``` expandable +1. Then restart your Prometheus instance after any configuration change +2. In Prometheus under β€œStatus” / β€œTargets”, we can see that Prometheus is able to scrape the metrics successfully: state is UP. + +Prometheus status + +### Exploring your metrics in Grafana + +Before trying out, make sure to generate traffic by calling your APIs. You will find a [couple of useful queries](https://github.com/TykTechnologies/tyk-pump#prometheus) in our Tyk Pump GitHub repo based on the metrics exposed by Tyk. These will demonstrate which metric types are exported and how you can customize them. + +You also need to make sure that Grafana is connected to your Prometheus server. This can be configured under [Configuration / Data sources](https://grafana.com/docs/grafana/latest/datasources/add-a-data-source/) + +Grafana Configuration with Prometheus + +### Useful queries + +Here are some useful queries to help you monitor the health of your APIs: + +#### Upstream time across all services + +Tyk collects latency data of how long your upstream services take to respond to requests. This data can be used to configure an alert if the latency goes beyond a certain threshold. This query calculated the 95th percentile of the total request latency of all the upstream services. To run the query: + +``` +histogram_quantile(0.95, sum(rate(tyk_http_latency_bucket[1m])) by (le)) +``` +Upstream Time Query output + +#### Upstream time per API + +This query calculated the 95th percentile of the request latency of upstream services for the selected API. To run this query: + +``` +histogram_quantile(0.90, sum(rate(tyk_http_latency_bucket{api_name=""}[1m])) by (le,api_name)) +``` +Replace `` with the name of your API for this query. + +#### Request rate + +Track the request rate of your services: + +``` +sum (rate(tyk_http_requests_total[1m])) +``` + +#### Request Rate per API + +Track the request rate of your services for the selected API: + +``` +sum (rate(tyk_http_requests_total{api_name=""}[1m])) +``` +Replace `` with the name of your API for this query. + +#### Error Rates + +Track the error rate your services are serving: + +``` +sum (rate(tyk_http_requests_total{response_code =~"5.."}[1m])) +``` + +#### Error rates per API + +Track the error rate your services are serving for the selected API: + +``` +sum (rate(tyk_http_requests_total{response_code =~"5..", api_name="httpbin - HTTP Request & Response Service"}[1m])) +``` expandable +Replace `` with the name of your API for this query. + +## Setup Prometheus Pump + +We'll show you how to setup Tyk Pump for Prometheus Service Discovery. + +pump-prometheus + +### Integrate with Prometheus using Prometheus Operator + +**Steps for Configuration:** + +1. **Setup Prometheus** + + *Using the prometheus-community/kube-prometheus-stack chart* + + In this example, we use [kube-prometheus-stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack), which installs a collection of Kubernetes manifests, [Grafana](http://grafana.com/) dashboards, and [Prometheus rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/) combined with documentation and scripts to provide easy to operate end-to-end Kubernetes cluster monitoring with [Prometheus](https://prometheus.io/) using the [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator). + + ```bash + helm install prometheus-stack prometheus-community/kube-prometheus-stack -n monitoring --create-namespace + ``` + + This is a useful stack where you can get Prometheus, the Prometheus Operator, and Grafana all deployed and configured in one go. + +2. **Install Tyk Pump with PodMonitor** + + If you have Prometheus Operator enabled on the cluster, it would look for β€œPodMonitor” or β€œServiceMonitor” resources and scrap from specified port. The only thing you would need to modify here is the helm release name for Prometheus Operator. + + Also you can customize Prometheus Custom Metrics based on your analytics needs. We are using `tyk_http_requests_total` and `tyk_http_latency` described [here](/api-management/tyk-pump#monitor-your-apis-with-prometheus) for illustration: + + ```bash + NAMESPACE=tyk-oss + APISecret=foo + REDIS_BITNAMI_CHART_VERSION=19.0.2 + PromOperator_Release=prometheus-stack + Prometheus_Custom_Metrics='[{"name":"tyk_http_requests_total"\,"description":"Total of API requests"\,"metric_type":"counter"\,"labels":["response_code"\,"api_name"\,"method"\,"api_key"\,"alias"\,"path"]}\, { "name":"tyk_http_latency"\, "description":"Latency of API requests"\, "metric_type":"histogram"\, "labels":["type"\,"response_code"\,"api_name"\,"method"\,"api_key"\,"alias"\,"path"] }]' + + helm upgrade tyk-redis oci://registry-1.docker.io/bitnamicharts/redis -n $NAMESPACE --create-namespace --install --version $REDIS_BITNAMI_CHART_VERSION + + helm upgrade tyk-oss tyk-helm/tyk-oss -n $NAMESPACE --create-namespace \ + --install \ + --set global.secrets.APISecret="$APISecret" \ + --set global.redis.addrs="{tyk-redis-master.$NAMESPACE.svc.cluster.local:6379}" \ + --set global.redis.passSecret.name=tyk-redis \ + --set global.redis.passSecret.keyName=redis-password \ + --set global.components.pump=true \ + --set "tyk-pump.pump.backend={prometheus}" \ + --set tyk-pump.pump.prometheusPump.customMetrics=$Prometheus_Custom_Metrics \ + --set tyk-pump.pump.prometheusPump.prometheusOperator.enabled=true \ + --set tyk-pump.pump.prometheusPump.prometheusOperator.podMonitorSelector.release=$PromOperator_Release + ``` + + + +Please make sure you are installing Redis versions that are supported by Tyk. Please refer to Tyk docs to get list of [supported versions](/tyk-self-managed#redis). + + + + + +For Custom Metrics, commas are escaped to be used in helm --set command. You can remove the backslashes in front of the commas if you are to set it in values.yaml. We have included an example in the default values.yaml comments section. + + + +3. **Verification** + + When successfully configured, you could see the following messages in pump log: + + ```console + β”‚ time="Jun 26 13:11:01" level=info msg="Starting prometheus listener on::9090" prefix=prometheus-pump β”‚ + β”‚ time="Jun 26 13:11:01" level=info msg="Prometheus Pump Initialized" prefix=prometheus-pump β”‚ + β”‚ time="Jun 26 13:11:01" level=info msg="Init Pump: PROMETHEUS" prefix=main + ``` + + On Prometheus Dashboard, you can see the Pump is listed as one of the target and Prometheus is successfully scrapped from it. + + pump-prometheus + + You can check our [Guide on Monitoring API with Prometheus](/api-management/tyk-pump#useful-queries) for a list of useful queries you can setup and use. + + e.g. The custom metrics tyk_http_requests_total can be retrieved: + + pump-prometheus + + pump-prometheus + +### Integrate with Prometheus using annotations + +**Steps for Configuration:** + +1. **Setup Prometheus** + + *Using the prometheus-community/prometheus chart* + + Alternatively, if you are not using Prometheus Operator, please check how your Prometheus can support service discovery. Let say you’re using the [prometheus-community/prometheus](https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus#scraping-pod-metrics-via-annotations) chart, which configures Prometheus to scrape from any Pods with following annotations: + + ```yaml + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/path: /metrics + prometheus.io/port: "9090" + ``` + + To install Prometheus, run + + ```bash + helm install prometheus prometheus-community/prometheus -n monitoring --create-namespace + ``` + +2. **Install Tyk Pump with prometheus annotations** + + ```bash + NAMESPACE=tyk-oss + APISecret=foo + REDIS_BITNAMI_CHART_VERSION=19.0.2 + PromOperator_Release=prometheus-stack + Prometheus_Custom_Metrics='[{"name":"tyk_http_requests_total"\,"description":"Total of API requests"\,"metric_type":"counter"\,"labels":["response_code"\,"api_name"\,"method"\,"api_key"\,"alias"\,"path"]}\, { "name":"tyk_http_latency"\, "description":"Latency of API requests"\, "metric_type":"histogram"\, "labels":["type"\,"response_code"\,"api_name"\,"method"\,"api_key"\,"alias"\,"path"] }]' + + helm upgrade tyk-redis oci://registry-1.docker.io/bitnamicharts/redis -n $NAMESPACE --create-namespace --install --version $REDIS_BITNAMI_CHART_VERSION + + helm upgrade tyk-oss tyk-helm/tyk-oss -n $NAMESPACE --create-namespace \ + --install \ + --set global.secrets.APISecret="$APISecret" \ + --set global.redis.addrs="{tyk-redis-master.$NAMESPACE.svc.cluster.local:6379}" \ + --set global.redis.passSecret.name=tyk-redis \ + --set global.redis.passSecret.keyName=redis-password \ + --set global.components.pump=true \ + --set "tyk-pump.pump.backend={prometheus}" \ + --set tyk-pump.pump.prometheusPump.customMetrics=$Prometheus_Custom_Metrics \ + --set-string tyk-pump.pump.podAnnotations."prometheus\.io/scrape"=true \ + --set-string tyk-pump.pump.podAnnotations."prometheus\.io/port"=9090 \ + --set-string tyk-pump.pump.podAnnotations."prometheus\.io/path"=/metrics + ``` + + + +Please make sure you are installing Redis versions that are supported by Tyk. Please refer to Tyk docs to get list of [supported versions](/tyk-self-managed#redis). + + + +3. **Verification** + + After some time, you can see that Prometheus is successfully scraping from Tyk Pump: + + pump-prometheus + +### Expose a service for Prometheus to scrape + +You can expose Pump as a service so that Prometheus can access the `/metrics` endpoint for scraping. Just enable service in `tyk-pump.pump.service`: + +```yaml + service: + # Tyk Pump svc is disabled by default. Set it to true to enable it. + enabled: true +``` expandable + +## Tyk Pump Capping Analytics Data Storage + +Tyk Gateways can generate a lot of analytics data. A guideline is that for every 3 million requests that your Gateway processes it will generate roughly 1GB of data. + +If you have Tyk Pump set up with the aggregate pump as well as the regular MongoDB pump, then you can make the `tyk_analytics` collection a [capped collection](https://docs.mongodb.com/manual/core/capped-collections/). Capping a collection guarantees that analytics data is rolling within a size limit, acting like a FIFO buffer which means that when it reaches a specific size, instead of continuing to grow, it will replace old records with new ones. + + + +If you are using DocumentDB, capped collections are not supported. See [here](https://docs.aws.amazon.com/documentdb/latest/developerguide/mongo-apis.html) for more details. + + + +The `tyk_analytics` collection contains granular log data, which is why it can grow rapidly. The aggregate pump will convert this data into a aggregate format and store it in a separate collection. The aggregate collection is used for processing reporting requests as it is much more efficient. + +If you've got an existing collection which you want to convert to be capped you can use the `convertToCapped` [MongoDB command](https://docs.mongodb.com/manual/reference/command/convertToCapped/). + +If you wish to configure the pump to cap the collections for you upon creating the collection, you may add the following +configurations to your `uptime_pump_config` and / or `mongo.meta` objects in `pump.conf`. + +``` +"collection_cap_max_size_bytes": 1048577, +"collection_cap_enable": true +``` expandable + +`collection_cap_max_size_bytes` sets the maximum size of the capped collection. +`collection_cap_enable` enables capped collections. + +If capped collections are enabled and a max size is not set, a default cap size of `5Gib` is applied. +Existing collections will never be modified. + + + +An alternative to capped collections is MongoDB's **Time To Live** indexing (TTL). TTL indexes are incompatible with capped collections. If you have set a capped collection, a TTL index will not get created, and you will see error messages in the MongoDB logs. See [MongoDB TTL Docs](https://docs.mongodb.com/manual/tutorial/expire-data/) for more details on TTL indexes. + + + + +### Time Based Cap in single tenant environments + +If you wish to reduce or manage the amount of data in your MongoDB, you can add an TTL expire index to the collection, so older records will be evicted automatically. + + + +Time based caps (TTL indexes) are incompatible with already configured size based caps. + + + + +Run the following command in your preferred MongoDB tool (2592000 in our example is 30 days): + +```{.copyWrapper} +db.tyk_analytics.createIndex( { "timestamp": 1 }, { expireAfterSeconds: 2592000 } ) +``` +This [command](https://docs.mongodb.com/manual/tutorial/expire-data/#expire-documents-at-a-specific-clock-time) sets expiration rule to evict all the record from the collection which `timestamp` field is older then specified expiration time. + +### Time Based Cap in multi-tenant environments +When you have multiple organizations, you can control analytics expiration on per organization basis. +This technique also use TTL indexes, as described above, but index should look like: + +```{.copyWrapper} +db.tyk_analytics.createIndex( { "expireAt": 1 }, { expireAfterSeconds: 0 } ) +``` + +This [command](https://docs.mongodb.com/manual/tutorial/expire-data/#expire-documents-at-a-specific-clock-time) sets the value of `expireAt` to correspond to the time the document should expire. MongoDB will automatically delete documents from the `tyk_analytics` collection 0 seconds after the `expireAt` time in the document. The `expireAt` will be calculated and created by Tyk in the following step. + +#### Create an Organization Quota + +```{.copyWrapper} +curl --header "x-tyk-authorization: {tyk-gateway-secret}" --header "content-type: application/json" --data @expiry.txt http://{tyk-gateway-ip}:{port}/tyk/org/keys/{org-id} +``` + +Where context of expiry.txt is: + +```{.json} +{ + "org_id": "{your-org-id}", + "data_expires": 86400 +} +``` expandable + +`data_expires` - Sets the data expires to a time in seconds for it to expire. Tyk will calculate the expiry date for you. + + +### Size Based Cap + +#### Add the Size Cap + + + +The size value should be in bytes, and we recommend using a value just under the amount of RAM on your machine. + + + + + +Run this [command](https://docs.mongodb.com/manual/reference/command/convertToCapped/) in your MongoDB shell: + + +```{.copyWrapper} +use tyk_analytics +db.runCommand({"convertToCapped": "tyk_analytics", size: 100000}); +``` + +#### Adding the Size Cap if using a mongo_selective Pump + +The `mongo_selective` pump stores data on a per organization basis. You will have to run the following command in your MongoDB shell for an individual organization as follows. + + +```{.copyWrapper} +db.runCommand({"convertToCapped": "z_tyk_analyticz_", size: 100000}); +``` + +## Separated Analytics Storage + +For high-traffic systems that make heavy use of analytics, it makes sense to separate out the Redis analytics server from the Redis configuration server that supplies auth tokens and handles rate limiting configuration. + +To enable a separate analytics server, update your `tyk.conf` with the following section: + +```{.copyWrapper} +"enable_separate_analytics_store": true, +"analytics_storage": { + "type": "redis", + "host": "", + "port": 0, + "addrs": [ + "localhost:6379" + ], + "username": "", + "password": "", + "database": 0, + "optimisation_max_idle": 3000, + "optimisation_max_active": 5000, + "enable_cluster": false +}, +``` + + + +`addrs` is new in v2.9.3, and replaces `hosts` which is now deprecated. + + + +If you set `enable_cluster` to `false`, you only need to set one entry in `addrs`: + +The configuration is the same (and uses the same underlying driver) as the regular configuration, so Redis Cluster is fully supported. + diff --git a/api-management/upstream-authentication.mdx b/api-management/upstream-authentication.mdx new file mode 100644 index 00000000..3e0988dd --- /dev/null +++ b/api-management/upstream-authentication.mdx @@ -0,0 +1,36 @@ +--- +title: "Upstream Authentication" +'og:description': "Authenticating Tyk Gateway with upstream services" +tags: ['security', 'upstream authentication', 'gateway to upstream', 'OAuth', 'mTLS', 'Basic Auth'] +sidebarTitle: "Overview" +--- + +## Introduction + +Tyk Gateway sits between your clients and your services, securely routing requests and responses. For each API proxy that you expose on Tyk, you can configure a range of different methods that clients must use to identify (authenticate) themselves to Tyk Gateway. These are described in detail in the [Client Authentication](/api-management/client-authentication) section. + +In the same way as you use Client Authentication to securely confirm the identity of the API clients, your upstream services probably need to securely confirm the identity of their client - namely Tyk. This is where Tyk's flexible **Upstream Authentication** capability comes in. + +When using Tyk, you can choose from a range of authentication methods for each upstream API: +- [Mutual TLS](/api-management/upstream-authentication/mtls) +- [Token-based authentication](/api-management/upstream-authentication/auth-token) +- [Request signing](/api-management/upstream-authentication/request-signing) +- [Basic Authentication](/api-management/upstream-authentication/basic-auth) +- [OAuth 2.0](/api-management/upstream-authentication/oauth) + - [OAuth 2.0 Client Credentials](/api-management/upstream-authentication/oauth#oauth-client-credentials) + - [OAuth 2.0 Password Grant](/api-management/upstream-authentication/oauth#oauth-resource-owner-password-credentials) + + + + + Upstream Basic Authentication and Oauth 2.0 support are only available to licensed users, via the Tyk Dashboard. These features are not available to open source users. + + + + + +Note that OAuth 2.0 Password Grant is prohibited in the [OAuth 2.0 Security Best Practice](https://datatracker.ietf.org/doc/html/draft-ietf-oauth-security-topics-13#section-3.4") but is supported by Tyk for use with legacy upstream services. + + + + diff --git a/api-management/upstream-authentication/auth-token.mdx b/api-management/upstream-authentication/auth-token.mdx new file mode 100644 index 00000000..cb43c914 --- /dev/null +++ b/api-management/upstream-authentication/auth-token.mdx @@ -0,0 +1,24 @@ +--- +title: "Upstream Authentication using Auth Token" +'og:description': "How to authenticate upstream service using auth token" +tags: ['security', 'upstream authentication', 'gateway to upstream', 'auth token'] +sidebarTitle: "Auth Token" +--- + +## Token-based authentication + +Token-based authentication (also referred to as Auth Token) is a method whereby the client is identified and authenticated by the server based on a key/token they present as a credential with each request. Typically the token is issued by the server to a specific client. + +The server determines how the key should be provided - typically in a request header, cookie or query parameter. + +Tyk supports [Auth Token](/api-management/authentication/bearer-token) as a method for authenticating **clients** with the **Gateway** - you can use Tyk Gateway or Dashboard to generate access *keys* for an Auth Token protected API as explained in the [documentation](/api-management/policies). The client must then provide the *key* in the appropriate parameter for each request. + +If your **upstream service** is protected using Auth Token then similarly, Tyk will need to provide a token, issued by the upstream, in the request. + +### How to use Upstream Token-based Authentication +Typically Auth Token uses the `Authorization` header to pass the token in the request. + +Tyk's [Request Header Transform](/api-management/traffic-transformation/request-headers) middleware can be configured to add this header to the request prior to it being proxied to the upstream. To enhance security by restricting visibility of the access token, the key/token can be stored in a [key-value store](/tyk-self-managed/install), with only the reference included in the middleware configuration. + + + diff --git a/api-management/upstream-authentication/basic-auth.mdx b/api-management/upstream-authentication/basic-auth.mdx new file mode 100644 index 00000000..1a9c4f26 --- /dev/null +++ b/api-management/upstream-authentication/basic-auth.mdx @@ -0,0 +1,121 @@ +--- +title: "Upstream Authentication using Basic Auth" +'og:description': "How to authenticate upstream service basic authentication" +tags: ['security', 'upstream authentication', 'gateway to upstream', 'basic auth'] +sidebarTitle: "Basic Auth" +--- + +## Basic Authentication + +Basic Authentication is a standard authentication mechanism implemented by HTTP servers, clients and web browsers. This makes it an excellent access control method for smaller APIs. + +An API request made using Basic Authentication will have an `Authorization` header that contains the client's credentials in the form: `Basic `. + +The `` are a base64 encoded concatenation of a client username and password, joined by a single colon `:`. + +Tyk supports Basic Authentication as a method for authenticating **clients** with the **Gateway** - you can use Tyk Gateway or Dashboard to create Basic Auth users, as explained in the [documentation](/api-management/authentication/basic-authentication#registering-basic-authentication-user-credentials-with-tyk). + +If your **upstream service** is protected using Basic Authentication then similarly, Tyk will need to provide user credentials, registered with the upstream, in the request. + +### How to use Upstream Basic Authentication + +If your upstream service requires that Tyk authenticates using Basic Authentication, you will first need to obtain a valid username and password from the server. To enhance security by restricting visibility of the credentials, these can be stored in a [key-value store](/tyk-self-managed/install), with only references included in the API definition. + +If the incoming request from the client already has credentials in the `Authorization` header, then Tyk will replace those with the basic auth credentials before proxying onwards to the upstream. + +Sometimes a non-standard upstream server might require the authentication credentials to be provided in a different header (i.e. not `Authorization`). With Tyk, you can easily configure a custom header to be used for the credentials if required. + +Upstream Basic Authentication is only supported by Tyk OAS APIs. If you are using Tyk Classic APIs, you could create the client credential offline and add the `Authorization` header using the [Request Header Transform](/api-management/traffic-transformation/request-headers) middleware. + +#### Configuring Upstream Basic Auth in the Tyk OAS API definition + +Upstream Authentication is configured per-API in the Tyk extension (`x-tyk-api-gateway`) within the Tyk OAS API definition by adding the `authentication` section within the `upstream` section. + +Set `upstream.authentication.enabled` to `true` to enable upstream authentication. + +For Basic Authentication, you will need to add the `basicAuth` section within `upstream.authentication`. + +This has the following parameters: +- `enabled` set this to `true` to enable upstream basic authentication +- `username` is the username to be used in the request *credentials* +- `password` is the password to be used in the request *credentials* +- `header.enabled` must be set to `true` if your upstream expects the *credentials* to be in a custom header, otherwise it can be omitted to use `Authorization` header +- `header.name` is the custom header to be used if `header.enabled` is set to `true` + +Note that if you use the [Tyk API Designer](#configuring-upstream-basic-auth-using-the-api-designer) in Tyk Dashboard it will always configure the `header` parameter - even if you are using the default `Authorization` value. + +For example: + +```json {hl_lines=["43-54"],linenos=true, linenostart=1} expandable +{ + "info": { + "title": "example-upstream-basic-auth", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "servers": [ + { + "url": "http://localhost:8181/example-upstream-basic-auth/" + } + ], + "security": [], + "paths": { + "/anything": { + "get": { + "operationId": "anythingget", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "components": { + "securitySchemes": {} + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-upstream-basic-auth", + "state": { + "active": true + } + }, + "server": { + "listenPath": { + "strip": true, + "value": "/example-upstream-basic-auth/" + } + }, + "upstream": { + "url": "https://httpbin.org/basic-auth/myUsername/mySecret", + "authentication": { + "enabled": true, + "basicAuth": { + "password": "mySecret", + "username": "myUsername", + "enabled": true, + "header": { + "enabled": true, + "name": "Authorization" + } + } + } + } + } +} +``` + +In this example upstream authentication has been enabled (line 44). Requests will be proxied to the `GET /basic-auth` endpoint at httpbin.org using the credentials in lines 46 and 47 (username: myUsername, password: mySecret). These credentials will be combined, base64 encoded and then provided in the `Authorization` header, as required by the httpbin.org [documentation](https://httpbin.org/#/Auth/get_basic_auth__user___passwd_"). + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the Upstream Basic Authentication feature. + +#### Configuring Upstream Basic Auth using the API Designer + +Upstream Authentication is configured from the **Settings** tab of the Tyk OAS API Designer, where there is a dedicated section within the **Upstream** section. + +Select **Basic Auth** from the choice in the **Authentication Method** drop-down, then you can provide the client credentials and header name. + +Tyk OAS API Designer showing Upstream Basic Auth configuration options + +
diff --git a/api-management/upstream-authentication/mtls.mdx b/api-management/upstream-authentication/mtls.mdx new file mode 100644 index 00000000..8aec71d5 --- /dev/null +++ b/api-management/upstream-authentication/mtls.mdx @@ -0,0 +1,456 @@ +--- +title: "Upstream Authentication using Mutual TLS" +'og:description': "How to authenticate upstream service using mutual tls" +tags: ['security', 'upstream authentication', 'gateway to upstream', 'mTLS', 'mutual tls'] +sidebarTitle: "Mutual TLS" +--- + +## Mutual TLS (mTLS) + +If your upstream API is protected with [mutual TLS](/basic-config-and-security/security/mutual-tls/client-mtls#why-use-mutual-tls) then Tyk must provide a certificate when connecting to the upstream service and also will need to verify the certificate presented by the upstream. This ensures secure communication between Tyk and your upstream services. + +When Tyk performs an mTLS handshake with an upstream, it needs to know: + +- which client certificate Tyk should use to identify itself +- which public key (certificate) that Tyk should use to verify the identity of the upstream + +We use a system of [mapping certificates](#mapping-certificates-to-domains) to upstreams based on their host domain. This is used for both the [client certificate](#upstream-client-certificates) and, optionally, for the [upstream public key](#upstream-server-certificates) if we want to use specific certificates to protect against compromised certificate authorities (CAs). + +#### Upstream mTLS for Tyk middleware and plugins + +If upstream mTLS certificates are configured for an API, they will not be used for direct proxies to the upstream and will also automatically be used for any HTTP requests made from the [JavaScript Virtual Endpoint](/api-management/traffic-transformation/virtual-endpoints) middleware. They will **not** be used for HTTP requests from custom plugins. + + +#### Upstream mTLS for Tyk Cloud + +All Tyk Cloud users can secure their upstream services with mTLS + +### Mapping certificates to domains + +Tyk maintains mappings of certificates to domains (which can include the port if a non-standard HTTP port is used). Separate maps can be declared globally, to be applied to all APIs, and at the API level for more granular control. The granular API level mapping takes precedence if both are configured. Within each mapping, both default and specific maps can be defined, giving ultimate flexibility. + +When Tyk performs an mTLS handshake with an upstream, it will check if there are certificates mapped to the domain: + +- first it will check in the API definition for a specific certificate +- then it will check in the API definition if there is a default certificate +- then it will check at the Gateway level for a specific certificate +- then it will check at the Gateway level for a default certificate + +Certificates are identified in the mapping using the [Certificate Id](/api-management/certificates#certificate-management) assigned by the Tyk Certificate Store, for example: `{"": ""}`. + +When mapping a certificate to a domain: + +- do not include the protocol (e.g. `https://`) +- include the port if a non-standard HTTP port is in use +- you can use the `*` wildcard - either in place of the whole domain or as part of the domain name + +For example, to map a certificate with Id `certId` to an upstream service located at `https://api.production.myservice.com:8443` you could map the certificate as: + +- `{"api.production.myservice.com:8443": "certId"}` +- `{"*.production.myservice.com:8443": "certId"}` +- `{"api.*.myservice.com:8443": "certId"}` + +Note that when using the wildcard (`*`) to replace part of the domain name, it can only represent one fragment so, using our example, you would not achieve the same mapping using `{"*.myservice.com:8443": "certId"}`. + + +A *default* certificate to be used for all upstream requests can be mapped by replacing the specific domain with the wildcard, for example `{"*", "certId"}`. + + +### Upstream client certificates + +Tyk can be configured to proxy requests to a single API on to different upstream hosts (for example via load balancing, API versions or URL rewrite middleware). You can configure Tyk to present specific client certificates to specific hosts, and you can specify a default certificate to be usedfor all upstream hosts. + +The upstream service uses the public key (from the certificate presented by Tyk) to verify the signed data, confirming that Tyk possesses the corresponding private key. + +All certificates are retrieved from the [Tyk Certificate Store](/api-management/certificates#certificate-management) when the proxy occurs. + +#### Mapping client certificates at the Gateway level + +You can map certificates to domains using the [security.certificates.upstream](/tyk-oss-gateway/configuration#securitycertificatesupstream) field in your Gateway configuration file. + +Mapping a certificate to domain `*` will ensure that this certificate will be used in all upstream requests where no other certificate is mapped (at Gateway or API level). + +#### Mapping client certificates at the API level + +You can map certificates to domains using the [upstream.mutualTLS](/api-management/gateway-config-tyk-oas#mutualtls) object (Tyk Classic: `upstream_certificates`) in your API definition. + +Mapping a certificate to domain `*` will ensure that this certificate will be used in all upstream requests where no other certificate is mapped in the API definition. + + +### Upstream server certificates + +Tyk will verify the certificate received from the upstream by performing the following checks: + +- Check that it's issued by a trusted CA +- Check that the certificate hasn't expired +- Verify the certificate's digital signature using the public key from the certificate + + + + + Tyk will look in the system trust store for the server that is running Tyk Gateway (typically `/etc/ssl/certs`). If you are using self-signed certificates, store them here so that Tyk can verify the upstream service. + + + +If you want to restrict the public keys that can be used by the upstream service, then you can use [certificate pinning](/api-management/upstream-authentication/mtls#certificate-pinning) to store a list of certificates that Tyk will use to verify the upstream. + +#### Certificate Pinning + +Tyk provides the facility to allow only certificates generated from specific public keys to be accepted from the upstream services during the mTLS exchange. This is called "certificate pinning" because you *pin* a specific public certificate to an upstream service (domain) and Tyk will only use this to verify connections to that domain. This helps to protect against compromised certificate authorities. You can pin one or more public keys per domain. + +The public keys must be stored in PEM format in the [Tyk Certificate Store](/api-management/certificates#certificate-management). + +##### Configuring Certificate Pinning at the Gateway level + +If you want to lock down the public certificates that can be used in mTLS handshakes for specific upstream domains across all APIs, you can pin public certificates to domains using the [security.pinned_public_keys](/tyk-oss-gateway/configuration#securitypinned_public_keys) field in your Gateway configuration file. + +This accepts a map of domain addresses to certificates in the same way as for the client certificates. Wildcards are supported in the domain addresses. Pinning one or more certificates to domain `*` will ensure that only these certificates will be used to verify the upstream service during the mTLS handshake. + +##### Configuring Certificate Pinning at the API level + +Restricting the certificates that can be used by the upstream for specific APIs is simply a matter of registering a map of domain addresses to certificates in the [upstream.certificatePinning](/api-management/gateway-config-tyk-oas#certificatepinning) object in the API definition (Tyk Classic: `pinned_public_keys`). + + +### Overriding mTLS for non-production environments + +When you are developing or testing an API, your upstream might not have the correct certificates that are deployed for your production service. This could cause problems when integrating with Tyk. + +You can use the [proxy.transport.insecureSkipVerify](/api-management/gateway-config-tyk-oas#tlstransport) option in the API definition (Tyk Classic: `proxy.transport.ssl_insecure_skip_verify`) to instruct Tyk to ignore the certificate verification stage for a specific API. + +If you want to ignore upstream certificate verification for all APIs deployed on Tyk, you can use the [proxy_ssl_insecure_skip_verify](/tyk-oss-gateway/configuration#proxy_ssl_insecure_skip_verify) option in the Tyk Gateway configuration. + +These are labelled *insecure* with good reason and should never be configured in production. + + +### Using Tyk Dashboard to configure upstream mTLS + +Using the Tyk Dashboard, you can enable upstream mTLS from the **Upstream** section in the API Designer: + +Enable upstream mTLS + +Click on **Attach Certificate** to open the certificate attachment window: + +Attach a certificate to an API + +This is where you can define the upstream **Domain Name** and either select an existing certificate from the Tyk Certificate Store, or upload a new certificate to the store. + +If you want to [pin the public certificates](/api-management/upstream-authentication/mtls#certificate-pinning) that can be used by Tyk when verifying the upstream service, then you should enable **Public certificates** and attach certificates in the same manner as for the client certificates: + +Enable public key pinning + +For details on managing certificates with Tyk, please see the [certificate management](/api-management/certificates#certificate-management) documentation. + +For Tyk Classic APIs, the **Upstream Certificates** controls are on the **Advanced Options** tab of the Tyk Classic API Designer. + + +### Using Tyk Operator to configure mTLS + + + + +Configure upstream mTLS client certificates using the `mutualTLS` field in the `TykOasApiDefinition` object when using Tyk Operator, for example: + +```yaml{hl_lines=["12-18"],linenos=false} expandable +apiVersion: tyk.tyk.io/v1alpha1 + kind: TykOasApiDefinition + metadata: + name: petstore + namespace: default + spec: + tykOAS: + configmapRef: + name: petstore + namespace: default + keyName: petstore.json + mutualTLS: + enabled: true + domainToCertificateMapping: + - domain: "petstore.com" + certificateRef: petstore-domain + - domain: "petstore.co.uk" + certificateRef: petstore-uk-domain +``` + + + + +Tyk Operator supports certificate pinning in the Tyk OAS custom resource, allowing you to secure your API by pinning a public key stored in a secret to a specific domain. + +Example of public keys pinning + +```yaml expandable +apiVersion: v1 +kind: ConfigMap +metadata: + name: cm + namespace: default +data: + test_oas.json: |- + { + "info": { + "title": "httpbin with certificate pinning", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "components": {}, + "paths": {}, + "x-tyk-api-gateway": { + "info": { + "name": "httpbin with certificate pinning", + "state": { + "active": true + } + }, + "upstream": { + "url": "https://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/httpbin/", + "strip": true + } + } + } + } +--- +apiVersion: v1 +kind: Secret +metadata: + name: domain-secret +type: kubernetes.io/tls # The secret needs to be a type of kubernetes.io/tls +data: + tls.crt: + tls.key: "" +--- +apiVersion: tyk.tyk.io/v1alpha1 +kind: TykOasApiDefinition +metadata: + name: "oas-pinned-public-keys" +spec: + tykOAS: + configmapRef: + keyName: test_oas.json + name: cm + certificatePinning: + enabled: true + domainToPublicKeysMapping: + - domain: "httpbin.org" + publicKeyRefs: + - domain-secret +``` + +This example demonstrates how to enable certificate pinning for the domain `httpbin.org` using a public key stored in a Kubernetes secret (`domain-secret`). + + + + +### Using Tyk Operator to configure mTLS for Tyk Classic APIs + + + +When using Tyk Classic APIs with Tyk Operator, you can configure upstream client certificates for mTLS using one of the following fields within the ApiDefinition object: + +- **upstream_certificate_refs**: Configure using certificates stored within Kubernetes secret objects. +- **upstream_certificates**: Configure using certificates stored within Tyk Dashboard's certificate store. + +**upstream_certificate_refs** + +The `upstream_certificate_refs` field can be used to configure certificates for different domains. References can be held to multiple secrets which are used for the domain mentioned in the key. Currently "*" is used as a wildcard for all the domains + +The example listed below shows that the certificate in the secret, *my-test-tls*, is used for all domains. + +```yaml expandable +# First apply this manifest using the command +# "kubectl apply -f config/samples/httpbin_upstream_cert.yaml" +# +# The operator will try to create the ApiDefinition and will succeed but will log an error that a certificate is missing +# in the cluster for an upstream +# +# Generate your public-private key pair , for test you can use the following command to obtain one fast: +# "openssl req -new -newkey rsa:4096 -x509 -sha256 -days 365 -nodes -out tls.crt -keyout tls.key" +# +# Run the following command to obtain the values that must be put inside the yaml that contians the secret resource: +# "kubectl create secret tls my-test-tls --key="tls.key" --cert="tls.crt" -n default -o yaml --dry-run=client" +# +# Apply your TLS certificate using the following command: (we already have an example one in our repo) +# "kubectl apply -f config/sample/simple_tls_secret.yaml" +# +# NOTE: the upstream_certificate_refs can hold references to multiple secrets which are used for the domain +# mentioned in the key (currently "*" is used as a wildcard for all the domains) +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin +spec: + name: httpbin + use_keyless: true + upstream_certificate_refs: + "*": my-test-tls + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default +``` + +A secret can be created and output in yaml format using the following command: + +```bash +kubectl create secret tls my-test-tls --key="keyfile.key" --cert="certfile.crt" -n default -o yaml --dry-run=client +kubectl apply -f path/to/your/tls_secret.yaml +``` + +**upstream_certificates** + +The `upstream_certificates` field allows certificates uploaded to the certificate store in Tyk Dashboard to be referenced in the Api Definition: + +```yaml expandable +# Skip the concatenation and .pem file creation if you already have a certificate in the correct format + +# First generate your public-private key pair , for test use you can use the following command to obtain one fast: +# "openssl req -new -newkey rsa:4096 -x509 -sha256 -days 365 -nodes -out tls.crt -keyout tls.key" + +# Concatenate the above files to obtain a .pem file which we will upload using the dashboard UI +# "cat tls.crt tls.key > cert.pem" + +# Upload it to the tyk certificate store using the dashboard + +# Fill in the manifest with the certificate id (the long hash) that you see is given to it in the dashboard +# (in place of "INSERT UPLOADED CERTIFICATE NAME FROM DASHBOARD HERE") +# Optional: Change the domain from "*" to something more specific if you need to use different +# upstream certificates for different domains + +# Then apply this manifest using the command +# "kubectl apply -f config/samples/httpbin_upstream_cert_manual.yaml" + +# The operator will try create the ApiDefinition and will succeed and it will have the requested domain upstream certificate +# in the cluster for an upstream + +# NOTE: the upstream_certificate can hold multiple domain-certificateName pairs +# (currently "*" is used as a wildcard for all the domains) + +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin +spec: + name: httpbin + use_keyless: true + upstream_certificates: + "*": #INSERT UPLOADED CERTIFICATE NAME FROM DASHBOARD HERE# + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default +``` + + + + +When using Tyk Classic APIs with Tyk Operator you can configure certificate pinning using one of the following fields within the ApiDefinition object: + +- **pinned_public_keys**: Use public keys uploaded via the Certificate API. +- **pinned_public_keys_refs**: Uses public keys configured from Kubernetes secret objects. + +###### pinned_public_keys + +Use the `pinned_public_keys` mapping to pin public keys to specific domains, referencing public keys that have been uploaded to Tyk Certificate storage via the Certificate API. + +```yaml +pinned_public_keys: + "foo.com": "", + "*": "," +``` + +Each `key-id` value should be set to the ID returned from uploading the public key via the Certificate API. Multiple public keys can be specified by separating their IDs by a comma. + +
+ +###### pinned_public_keys_refs + +The `pinned_public_keys_refs` mapping should be used to configure pinning of public keys sourced from Kubernetes secret objects for different domains. + +Each key should be set to the name of the domain and the value should refer to the name of a Kuberenetes secret object that holds the corresponding public key for that domain. + +Wildcard domains are supported and "*" can be used to denote all domains. + + + +**Caveats** + +- Only *kubernetes.io/tls* secret objects are allowed. +- Please use the *tls.crt* field for the public key. +- The secret that includes a public key must be in the same namespace as the ApiDefinition object. + + + +The example below illustrates a scenario where the public key from the Kubernetes secret object, *httpbin-secret*, is used for all domains, denoted by the wildcard character '*'. In this example the *tls.crt* field of the secret is set to the actual public key of *httpbin.org*. Subsequently, if you any URL other than https://httpbin.org is targetted (e.g. https://github.com/) a *public key pinning error* will be raised for that particular domain. This is because the public key of *httpbin.org* has been configured for all domains. + +```yaml expandable +# ApiDefinition object 'pinned_public_keys_refs' field uses the following format: +# spec: +# pinned_public_keys_refs: +# "domain.org": # the name of the Kubernetes Secret Object that holds the public key for the 'domain.org'. +# +# In this way, you can refer to Kubernetes Secret Objects through 'pinned_public_keys_refs' field. +# +# In this example, we have an HTTPS upstream target as `https://httpbin.org`. The public key of httpbin.org is obtained +# with the following command: +# $ openssl s_client -connect httpbin.org:443 -servername httpbin.org 2>/dev/null | openssl x509 -pubkey -noout +# +# Note: Please set tls.crt field of your secret to actual public key of httpbin.org. +# +# We are creating a secret called 'httpbin-secret'. In the 'tls.crt' field of the secret, we are specifying the public key of the +# httpbin.org obtained through above `openssl` command, in the decoded manner. +# +apiVersion: v1 +kind: Secret +metadata: + name: httpbin-secret +type: kubernetes.io/tls +data: + tls.crt: # Use tls.crt field for the public key. + tls.key: "" +--- +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-certificate-pinning +spec: + name: httpbin - Certificate Pinning + use_keyless: true + protocol: http + active: true + pinned_public_keys_refs: + "*": httpbin-secret + proxy: + target_url: https://httpbin.org + listen_path: /pinning + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default +``` + + + +
+ diff --git a/api-management/upstream-authentication/oauth.mdx b/api-management/upstream-authentication/oauth.mdx new file mode 100644 index 00000000..b23aa41a --- /dev/null +++ b/api-management/upstream-authentication/oauth.mdx @@ -0,0 +1,258 @@ +--- +title: "Upstream Authentication using OAuth" +'og:description': "How to authenticate upstream service using oauth" +tags: ['security', 'upstream authentication', 'gateway to upstream', 'oauth'] +sidebarTitle: "OAuth 2.0" +--- + +## Upstream OAuth 2.0 + +OAuth 2.0 is an open standard authorization protocol that allows services to provide delegated and regulated access to their APIs; critically the user credentials are not shared with the upstream service, instead the client authenticates with a separate Authentication Server which issues a time-limited token that the client can then present to the upstream (Resource Server). The upstream service validates the token against the Authentication Server before granting access to the client. + +The Authentication Server (auth server) has the concept of an OAuth Client - this is equivalent to the client's account on the auth server. There are different ways that a client can authenticate with the auth server, each with their own advantages and disadvantages for different use cases. + +The auth server is often managed by a trusted third party Identity Provider (IdP) such as Okta or Auth0. + +Tyk supports OAuth 2.0 as a method for authenticating **clients** with the **Gateway** - you can use Tyk's own auth server functionality via the [Tyk OAuth 2.0](/api-management/authentication/oauth-2) auth method or obtain the access token via a third party auth server and use the [JWT Auth](/basic-config-and-security/security/authentication-authorization/json-web-tokens) method. + +If your **upstream service** is protected using OAuth 2.0 then similarly, Tyk will need to obtain a valid access token to provide in the request to the upstream. + +Tyk supports two different OAuth grant types for connecting to upstream services: +- [Client credentials](#oauth-client-credentials) +- [Resource owner password credentials](#oauth-resource-owner-password-credentials) + +#### OAuth client credentials + +The client credentials grant relies upon the client providing an id and secret (the *client credentials*) to the auth server. These are checked against the list of OAuth Clients that it holds and, if there is a match, it will issue an access token that instructs the Resource Server which resources that client is authorized to access. For details on configuring Tyk to use Upstream Client Credentials see [below](#configuring-upstream-oauth-20-client-credentials-in-the-tyk-oas-api-definition). + +#### OAuth resource owner password credentials + +The resource owner password credentials grant (also known simply as **Password Grant**) is a flow where the client must provide both their own credentials (client Id and secret) and a username and password identifying the resource owner to the auth server to obtain an access token. Thus the (upstream) resource owner must share credentials directly with the client. This method is considered unsafe and is prohibited in the [OAuth 2.0 Security Best Practice](https://datatracker.ietf.org/doc/html/draft-ietf-oauth-security-topics-13#section-3.4") but is supported by Tyk for use with legacy upstream services. For details on configuring Tyk to use Upstream Password Grant see [below](#configuring-upstream-oauth-20-password-grant-in-the-tyk-oas-api-definition). + +### How to use Upstream OAuth 2.0 for Authentication + +If your upstream service requires that Tyk authenticates via an OAuth auth server, you will first need to obtain credentials for the OAuth Client created in the auth server. You select which grant type to use and provide the required credentials in the API definition. + +To enhance security by restricting visibility of the credentials, these can be stored in a [key-value store](/tyk-self-managed/install), with only references included in the API definition. + +Some auth servers will return *additional metadata* with the access token (for example, the URL of the upstream server that should be addressed using the token if this can vary per client). Tyk can accommodate this using the optional `extraMetadata` field in the API definition. The response from the auth server will be parsed for any fields defined in `extraMetadata`; any matches will be saved to the request context where they can be accessed from other middleware (for our example, the [URL rewrite](/transform-traffic/url-rewriting#url-rewrite-middleware) middleware could be used to modify the upstream target URL). + +#### Configuring Upstream OAuth 2.0 Client Credentials in the Tyk OAS API definition + +Upstream Authentication is configured per-API in the Tyk extension (`x-tyk-api-gateway`) within the Tyk OAS API definition by adding the `authentication` section within the `upstream` section. + +Set `upstream.authentication.enabled` to `true` to enable upstream authentication. + +For OAuth 2.0 Client Credentials, you will need to add the `oauth` section within `upstream.authentication`. + +This has the following parameters: +- `enabled` set this to `true` to enable upstream OAuth authentication +- `allowedAuthorizeTypes` should include the value `clientCredentials` +- `clientCredentials` should be configured with: + - `tokenUrl` is the URL of the `/token` endpoint on the *auth server* + - `clientId` is the client ID to be provided to the *auth server* + - `clientSecret` is the client secret to be provided to the *auth server* + - `scopes` is an optional array of authorization scopes to be requested + - `extraMetadata` is an optional array of additional fields to be extracted from the *auth server* response + - `header.enabled` must be set to `true` if your upstream expects the credentials to be in a custom header, otherwise it can be omitted to use `Authorization` header + - `header.name` is the custom header to be used if `header.enabled` is set to `true` + +Note that if you use the [Tyk API Designer](/api-management/upstream-authentication/basic-auth#configuring-upstream-basic-auth-using-the-api-designer) in Tyk Dashboard it will always configure the `header` parameter - even if you are using the default `Authorization` value. + +For example: + +```json {hl_lines=["43-62"],linenos=true, linenostart=1} expandable +{ + "info": { + "title": "example-upstream-client-credentials", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "servers": [ + { + "url": "http://localhost:8181/example-upstream-client-credentials/" + } + ], + "security": [], + "paths": { + "/anything": { + "get": { + "operationId": "anythingget", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "components": { + "securitySchemes": {} + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-upstream-client-credentials", + "state": { + "active": true + } + }, + "server": { + "listenPath": { + "strip": true, + "value": "/example-upstream-client-credentials/" + } + }, + "upstream": { + "url": "https://httpbin.org/", + "authentication": { + "enabled": true, + "oauth": { + "enabled": true, + "allowedAuthorizeTypes": [ + "clientCredentials" + ], + "clientCredentials": { + "header": { + "enabled": true, + "name": "Authorization" + }, + "tokenUrl": "http:///token", + "clientId": "client123", + "clientSecret": "secret123", + "scopes": ["scope1"], + "extraMetadata": ["instance_url"] + } + } + } + } + } +} +``` + +In this example upstream authentication has been enabled (line 44). The authentication method to be used is indicated in lines 46 (OAuth) and 48 (client credentials). When a request is made to the API, Tyk will request an access token from the *authorization server* at `http://` providing client credentials and the scope `scope1`. + +Tyk will parse the response from the *authorization server* for the key `instance_url`, storing any value found in the *request context* were it can be accessed by other middleware as `$tyk_context.instance_url` (note the rules on accessing [request context variables from middleware](/api-management/traffic-transformation/request-context-variables)). + +On receipt of an access token from the *authorization server*, Tyk will proxy the original request to the upstream server (`https://httpbin.org/`) passing the access token in the `Authorization` header. + +If you replace the `upstream.url` and *authorization server* details with valid details, then the configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the Upstream OAuth 2.0 Client Credentials feature. + +#### Configuring Upstream OAuth 2.0 Client Credentials using the API Designer + +Upstream Authentication is configured from the **Settings** tab of the Tyk OAS API Designer, where there is a dedicated section within the **Upstream** section. + +Select **OAuth** from the choice in the **Authentication Method** drop-down, then you can provide the header name, authorization server token URL and select **Client Credentials** to reveal the configuration for the credentials to be passed to the auth server. + +Tyk OAS API Designer showing Upstream OAuth client credentials configuration options + +#### Configuring Upstream OAuth 2.0 Password Grant in the Tyk OAS API definition + +Upstream Authentication is configured per-API in the Tyk extension (`x-tyk-api-gateway`) within the Tyk OAS API definition by adding the `authentication` section within the `upstream` section. + +Set `upstream.authentication.enabled` to `true` to enable upstream authentication. + +For OAuth 2.0 Resource Owner Password Credentials (*Password Grant*), you will need to add the `oauth` section within `upstream.authentication`. + +This has the following parameters: +- `enabled` set this to `true` to enable upstream OAuth authentication +- `allowedAuthorizeTypes` should include the value `password` +- `password` should be configured with: + - `tokenUrl` is the URL of the `/token` endpoint on the *auth server* + - `clientId` is the client ID to be provided to the *auth server* + - `clientSecret` is the client secret to be provided to the *auth server* + - `username` is the Resource Owner username to be provided to the *auth server* + - `password` is the Resource Owner password to be provided to the *auth server* + - `scopes` is an optional array of authorization scopes to be requested + - `extraMetadata` is an optional array of additional fields to be extracted from the *auth server* response + - `header.enabled` must be set to `true` if your upstream expects the credentials to be in a custom header, otherwise it can be omitted to use `Authorization` header + - `header.name` is the custom header to be used if `header.enabled` is set to `true` + +Note that if you use the [Tyk API Designer](/api-management/upstream-authentication/basic-auth#configuring-upstream-basic-auth-using-the-api-designer) in Tyk Dashboard it will always configure the `header` parameter - even if you are using the default `Authorization` value. + +For example: + +```json {hl_lines=["43-64"],linenos=true, linenostart=1} expandable +{ + "info": { + "title": "example-upstream-password-grant", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "servers": [ + { + "url": "http://localhost:8181/example-upstream-password-grant/" + } + ], + "security": [], + "paths": { + "/anything": { + "get": { + "operationId": "anythingget", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "components": { + "securitySchemes": {} + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-upstream-password-grant", + "state": { + "active": true + } + }, + "server": { + "listenPath": { + "strip": true, + "value": "/example-upstream-password-grant/" + } + }, + "upstream": { + "url": "https://httpbin.org/", + "authentication": { + "enabled": true, + "oauth": { + "enabled": true, + "allowedAuthorizeTypes": [ + "password" + ], + "password": { + "header": { + "enabled": true, + "name": "Authorization" + }, + "tokenUrl": "http:///token", + "clientId": "client123", + "clientSecret": "secret123", + "username": "user123", + "password": "pass123", + "scopes": ["scope1"], + "extraMetadata": ["instance_url"] + } + } + } + } + } +} +``` + +In this example upstream authentication has been enabled (line 44). The authentication method to be used is indicated in lines 46 (OAuth) and 48 (password grant). When a request is made to the API, Tyk will request an access token from the *authorization server* at `http://` providing client credentials, resource owner credentials and the scope `scope1`. + +Tyk will parse the response from the *authorization server* for the key `instance_url`, storing any value found in the *request context* were it can be accessed by other middleware as `$tyk_context.instance_url` (note the rules on accessing [request context variables from middleware](/api-management/traffic-transformation/request-context-variables)). + +On receipt of an access token from the *authorization server*, Tyk will proxy the original request to the upstream server (`https://httpbin.org/`) passing the access token in the `Authorization` header. + +If you replace the `upstream.url` and *authorization server* details with valid details, then the configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the Upstream OAuth 2.0 Password Grant feature. + +#### Configuring Upstream OAuth 2.0 Password Grant using the API Designer + +Upstream Authentication is configured from the **Settings** tab of the Tyk OAS API Designer, where there is a dedicated section within the **Upstream** section. + +Select **OAuth** from the choice in the **Authentication Method** drop-down, then you can provide the header name, authorization server token URL and select **Resource Owner Password Credentials** to reveal the configuration for the credentials to be passed to the auth server. + +Tyk OAS API Designer showing Upstream OAuth password grant configuration options diff --git a/api-management/upstream-authentication/request-signing.mdx b/api-management/upstream-authentication/request-signing.mdx new file mode 100644 index 00000000..0da3931d --- /dev/null +++ b/api-management/upstream-authentication/request-signing.mdx @@ -0,0 +1,133 @@ +--- +title: "Upstream Authentication using Request Signing" +'og:description': "How to authenticate upstream service using request signing" +tags: ['security', 'upstream authentication', 'gateway to upstream', 'request signing'] +sidebarTitle: "Request Signing" +--- + +## Request signing + +Request Signing is an access token method that adds another level of security where the client generates a unique signature that identifies the request temporally to ensure that the request is from the requesting user, using a secret key that is never broadcast over the wire. + +Tyk can apply either the symmetric Hash-Based Message Authentication Code (HMAC) or asymmetric Rivest-Shamir-Adleman (RSA) algorithms when generating the signature for a request to be sent upstream. For HMAC, Tyk supports different options for the hash length. + +The following algorithms are supported: + +| Hashing algorithm | Tyk identifier used in API definition | +| :------------------- | :--------------------------------------- | +| HMAC SHA1 | `hmac-sha1` | +| HMAC SHA256 | `hmac-sha256` | +| HMAC SHA384 | `hmac-sha384` | +| HMAC SHA512 | `hmac-sha512` | +| RSA SHA256 | `rsa-sha256` | + +This feature is implemented using [Draft 10](https://tools.ietf.org/html/draft-cavage-http-signatures-10) RFC. The signatures generated according to this standard are temporal - that is, they include a time stamp. If there is no `Date` header in the request that is to be proxied to the upstream, Tyk will add one. + +### Configuring Request Signing in the API definition +Upstream Authentication is configured per-API in the Tyk Vendor Extension by adding the authentication section within the `upstream` section. + +For Request Signing, you must configure [upstream.authentication.upstreamRequestSigning](/api-management/gateway-config-tyk-oas#upstreamrequestsigning), providing the following settings: + +- the `signatureHeader` in which the signature should be sent (typically `Authorization`) +- the `algorithm` to be used to generate the signature (from the table above) +- the `secret` to be used in the encryption operation +- optional `headers` that Tyk should include in the string that is encrypted to generate the signature +- the `keyId` that the upstream will use to identify Tyk as the client (used for HMAC encryption) +- the `certificateId` that the upstream will use to identify Tyk as the client (used for RSA encryption) + +The Tyk Classic equivalent is [request_signing](/api-management/gateway-config-tyk-classic#upstream-authentication). + +### Configuring Request Signing with Tyk Operator + +When using Tyk Operator, the `certificateId` and `secret` are encapsulated in Kubernetes references: +- `certificateRef`: references a Secret containing the private and secret key. +- `secretRef`: references a Kubernetes Secret that holds the secret used in the encryption operation. + +For example: + +```yaml{linenos=true, linenostart=1, hl_lines=["66-73"]} + apiVersion: v1 + data: + secretKey: cGFzc3dvcmQxMjM= + kind: Secret + metadata: + name: upstream-secret + namespace: default + type: Opaque + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: booking + namespace: default + data: + test_oas.json: |- + { + "info": { + "title": "bin", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "components": {}, + "paths": {}, + "x-tyk-api-gateway": { + "info": { + "name": "bin", + "state": { + "active": true, + "internal": false + } + }, + "server": { + "listenPath": { + "strip": true, + "value": "/bin/" + } + }, + "upstream": { + "url": "http://httpbin.org/", + "authentication": { + "requestSigning": { + "enabled": true, + "signatureHeader": "Signature", + "algorithm": "hmac-sha256", + "keyId": "random-key-id", + "headers": [], + "secret": "" + } + } + } + } + } + --- + apiVersion: tyk.tyk.io/v1alpha1 + kind: TykOasApiDefinition + metadata: + name: booking + namespace: default + spec: + tykOAS: + configmapRef: + namespace: default + name: booking + keyName: test_oas.json + upstreamRequestSigning: + certificateRef: "" + secretRef: + namespace: default + name: upstream-secret + secretKey: secretKey + algorithm: "hmac-sha256" + keyId: "" + ``` +In this example, a Tyk OAS API was created using the `upstreamRequestSigning` field. It can be broken down as follows: +- `upstreamRequestSigning`: This defines the settings for Upstream Request Signing. in the example manifest, it configures Upstream Request Signing using the `booking` API. + - `certificateRef`: References a Secret containing the private and secret key for signing client API requests. This should be used if `secretRef` is not specified. + - `secretRef`: References a Kubernetes Secret that holds the secret key for signing client requests. + - `algorithm`: Specifies the algorithm used for signing. + - For `secretRef`, supported algorithms include: `hmac-sha1`, `hmac-sha256`, `hmac-sha384`, and `hmac-sha512`. + - For `certificateRef`, the required algorithm is `rsa-sha256`. + - `keyId`: A user-defined key assumed to be available on the upstream service. This is used in the `SignatureHeader` and should be included when using `certificateRef`. It is required when using the RSA algorithm. + +
+ diff --git a/api-management/user-management.mdx b/api-management/user-management.mdx new file mode 100644 index 00000000..85346f83 --- /dev/null +++ b/api-management/user-management.mdx @@ -0,0 +1,485 @@ +--- +title: "User management with Tyk Dashboard" +'og:description': "How to manage users, teams, permissions, rbac in Tyk Dashboard" +tags: ['Dashboard', 'User Management', 'RBAC', 'Role Based Access Control', 'User Groups', 'Teams', 'Permissions', 'API Ownership', 'SSO', 'Single Sing On', 'Multi Tenancy'] +sidebarTitle: "User Management" +--- + +import { ButtonLeft } from '/snippets/ButtonLeft.mdx'; + +## Introduction + +Tyk Dashboard provides you with the ability to manage Users, Teams and Permissions enabling organizations to maintain robust control over access and visibility. These capabilities empower teams to manage large-scale API portfolios, mitigate risks of unauthorized access, and reduce operational complexity. + +In this section, we delve into the following key topics: + +1. **[Managing Users](#manage-tyk-dashboard-users)**: + Streamlining user administration by creating, updating, and deactivating accounts within the Tyk Dashboard using both the UI and API. +2. **[Managing User Permissions](#user-permissions)**: + Configuring and enforcing role-based access control for users within the Tyk Dashboard, using both the API and UI. +3. **[Managing User Groups/Teams](#manage-tyk-dashboard-user-groups)**: + Organizing users into groups or teams to simplify role assignment, permissions management, and collaborative workflows within the Tyk Dashboard. +4. **[Managing Passwords and Policy](#manage-user-passwords)**: + Establishing and enforcing password policies, including complexity and expiration settings, to secure user accounts. +5. **[Configuring API Ownership](#api-ownership)**: + Applying role-based access control to APIs to govern visibility and manageability for specific teams or users. +6. **[Managing Users across Multiple Tyk Organizations](#manage-tyk-dashboard-users-in-multiple-organizations)**: + Administering user access and roles across multiple organizations, ensuring consistent and secure management in multi-tenant setups. +7. **[Single Sign-On](#single-sign-on-integration)**: + Integrating and configuring Single Sign-On (SSO) solutions to streamline authentication and enhance security across the Tyk Dashboard. + +
+ + +The availability of some features described in this section depends on your license. +
+For further information, please check our [price comparison](https://tyk.io/price-comparison/) or consult our sales and expert engineers: + + + + +## Understanding "User" in Tyk + +In the context of Tyk, a User refers to an individual responsible for managing, configuring, and maintaining the Tyk API Gateway and its related components. These users interact with the Tyk Dashboard and API to control various aspects such as API management, user permissions, security policies, and organizational settings. This term does not refer to end-users or consumers of the APIs managed through Tyk but specifically to administrators and developers operating the Tyk ecosystem. + +## Initial Admin User Creation + +When you start the Tyk Dashboard the first time, the bootstrap process creates an initial "user" for you with admin permissions, which allows them access to control and configure everything in the Dashboard (via the UI or Tyk Dashboard API). + +## Manage Tyk Dashboard Users + +Dashboard users have twofold access to the dashboard: they can access both the Dashboard API and the dashboard itself, it is possible to generate users that have read-only access to certain sections of the dashboard and the underlying API. + +Dashboard users are not the same as developer portal users (a.k.a. [developers](/tyk-developer-portal/tyk-portal-classic/portal-concepts#developers)). The credentials are stored independently and have different mechanics relating to registration, management and access. For example, it is not possible to log into the developer portal using a dashboard account. + +### Using Dashboard UI + +To create a dashboard user from the GUI: + +1. **Select "Users" from the "System Management" section** + + Users menu + +2. **Click "ADD USER"** + + Add user button location + +3. **Add the user's basic details** + + User form + + In this section: + + * **First Name**: The user's first name. + * **Last Name**: The user's last name. + * **Email**: The email address of the user, this will also be their login username. + * **Password**: The password to assign to the user, this will automatically be hashed and salted before storing in the database. **NOTE** you need to inform the user about the password you have created for them. + * **Active**: Must be true for the user to have access to the dashboard or the dashboard API. + +4. **Set the user permissions** + + Admin checkbox location + + You can be very specific with regards to which pages and segments of the Dashboard the user has access to. Some Dashboard pages require access to multiple parts of the API, and so you may get errors if certain related elements are disabled (e.g. APIs + Policies) + + Permissions are set and enforced when they are set on this page. They can either be **read** or **write**. If set to **deny** then the record is non-existent in the object (there is no explicit "deny"). This means that if you set **deny** on all options it looks as if they have not been written, but they will still be enforced so long as even one read or write option has been set. + +5. **Click "Save"** + + Save button location + + The user will automatically be created, as will their API Access token, which you will be able to retrieve by opening the user listing page again and selecting the user's username. + +### Using Dashboard API + +To authenticate requests to the Tyk Dashboard API, you will need to provide an API Key in the `Authorization` header. + +This is your **Tyk Dashboard API Access Credentials**, which can be found on the user detail page: + +API key and RPC key locations + +You can [create a user](/api-management/dashboard-configuration#add-user) with a call to the `POST /api/users` endpoint, for example: + +``` bash expandable +curl -H "Authorization: {YOUR-TYK-DASHBOARD-API-ACCESS-CREDENTIALS}" \ + -s \ + -H "Content-Type: application/json" \ + -X POST \ + -d '{ + "first_name": "Test", + "last_name": "User", + "email_address": "test@testing.com", + "active": true, + "user_permissions": { + "IsAdmin": "admin" + }, + "password": "thisisatest" + }' http://{your-dashboard-host}:{port}/api/users | python -mjson.tool +``` + +In this example, we have given the user Admin privileges. To see a detailed breakdown of permission objects, please see below. + +You will see the following response to confirm that the user has been created: + +```json +{ + "Message": "User created", + "Meta": null, + "Status": "OK" +} +``` + +The user is now active. + +## Manage User Passwords +You can change your password in these circumstances: + +* If you have forgotten your password +* If you wish to change your password + +### Forgotten Your Password? +If you have forgotten your password, you can request a password reset email from the **Dashboard Login** screen: + +password reset email + +Enter your login email address, and you will receive an email with a link that enables you to create a new password. + + + +This link will only be valid for 1000 seconds +
+You will need to configure your [outbound email settings](/configure/outbound-email-configuration) to enable this feature. + + + +### Change Your Password +If you wish to change your current password, from the **System Management > Users** screen, select **Edit** for your Username. + + + +You will not be able to change the password for other Dashboard users. + + + +From your user details, click **Reset Password**: + +reset password button +Enter your current and new password (and confirm it) in the dialog box that is displayed, and click **Reset Password**. +You will automatically be logged out of the Dashboard and will have to enter your username and new password to log back in. + +## Manage Tyk Dashboard User Groups + +Tyk has a flexible [user permissions](/api-management/user-management#user-permissions) system that provides Role Based Access Control (RBAC) for your Tyk Dashboard. + +When you have a large number of users and teams with different access requirements, instead of setting permissions per *user*, you can create a *user group* and configure the permissions for all users in the group. For example, if you only want certain *users* to access the Tyk Logs, you could create a "Logs Users" *user group*, then give those users the *Logs Read* permission and add them to your *Logs Users* group. + +Note that **a user can only belong to one group**. + +You must have either *admin* or *user groups* permission to be able to modify user groups. + +This also works for Single Sign-On (SSO), as you can specify the user group ID when setting up SSO. + + + +The availability of this feature depends on your license. +
+For further information, please check our [price comparison](https://tyk.io/price-comparison/) or consult our sales and expert engineers: + + + + + +### Using Dashboard UI + +1. **Select "User Groups" from the "System Management" section** + + User group menu + +2. **Click "ADD NEW USER GROUP"** + + Add user group location + +3. **Add User Group Name** + + Enter the name for your User Group, and an optional Description. + + Add name + +4. **Set User Group Permissions** + + Selet the User Group Permissions you want to apply. + + Add permissions + +5. **Click "Save" to create the Group** + + Click Save + +6. **Add Users to your Group** + + 1. From the **Users** menu, select **Edit** from the **Actions** drop-down list for a user to add to the group. + 2. Select your group from the **User group** drop-down list. + + select user group + + Click Update to save the User details + + update user + +### Using Dashboard API + +You can also manage User Groups via our [Dashboard API](/api-management/dashboard-configuration#user-groups-api). The following functions are available: + +* [List all User Groups](/api-management/dashboard-configuration#list-user-groups) +* [Get a User Group via the User Group ID](/api-management/dashboard-configuration#get-user-group) +* [Add a User Group](/api-management/dashboard-configuration#add-user-group) +* [Update a User Group](/api-management/dashboard-configuration#update-user-group) +* [Delete a User Group](/api-management/dashboard-configuration#delete-user-group) + +## Search Users + +You can search for a user (by email address) by entering the address in the search field. The user list will automatically refresh with that user being displayed. + +User Profile Search + +## Password Policy + +Tyk allows you to control password requirements for Dashboard users, developers (i.e. users registered to the developer portal) and basic auth keys. +Please note: This configuration is enforced by the Tyk-Dashboard and as such is not available in the Tyk Open Source Edition. Also, since it requires access to the Tyk Dashboard installation folder, it is *currently* not available for Tyk Cloud clients. + +There are other security options available from the Dashboard config file. See the [security section](/tyk-dashboard/configuration#security) for more details. + +You can find the configuration files in the `schemas` directory of your Tyk Dashboard installation folder, as follows: +- For Dashboard users you define policy in `schemas/password.json` +- For developers you define policy in `schemas/developer_password.json` +- For basic auth keys you define policy in `./schemas/basic_auth.json` + + +The following validators are available: + +* `minLength` - sets minimum password length +* `multiCase` - boolean, upper and lower case characters are required +* `minNumeric` - minimum number of numeric characters +* `minSpecial` - minimum number of special characters, like `@`, `$`, `%` etc. +* `disableSequential` - boolean, disable passwords which include at least 3 sequential characters. For example: `abc`, `123`, `111`, `xxx` etc. + +Below is an example of `password.json` file, with all options turned on: + +```{.copyWrapper} expandable +{ + "title": "User password schema", + "type": "string", + + "minLength": 6, + "multiCase": true, + "minNumeric": 2, + "minSpecial": 2, + "disableSequential": true +} +``` + +## User Permissions + +The Tyk Dashboard is multi-tenant capable and allows granular, role based user access. Users can be assigned specific permissions to ensure that they only have very specific access to the Dashboard pages, and to the underlying API. + +It is important to note that all user roles are defined and enforced **at the Dashboard API level**, and the UI is merely reactive. + +### Admin Users + +An *admin* user has read and write access to all properties. The initial user created during the dashboard's bootstrapping process is automatically assigned the *admin* role. + +There are two configuration parameters that restrict the admin user’s capabilities. For enhanced security, both of these values should be set to `true`: + +- [security.forbid_admin_view_access_token](/tyk-dashboard/configuration#securityforbid_admin_view_access_token): This parameter restricts admin users from viewing other users' Dashboard API Access Credentials, both in the API and the UI. + +- [security.forbid_admin_reset_access_token](/tyk-dashboard/configuration#securityforbid_admin_reset_access_token): This parameter prevents admin users from resetting the access tokens of other users. +### User permissions in the Tyk Dashboard API +The permissions object, which is provided to the Dashboard API has this structure: + +```json expandable +"user_permissions": { + "IsAdmin": "false", + "analytics": "read", + "apis": "write", + "hooks": "write", + "idm": "write", + "keys": "write", + "policy": "write", + "portal": "write", + "system": "write", + "users": "write", + "user_groups": "write", + "audit_logs": "read" + } +``` + +Note that the above list may not be complete as more features and flexibility are added to the Tyk Dashboard. + +The way the permissions object works is that: + - if it contains `"IsAdmin":"true"`, the user is an *admin* + - if it contains no properties, the user is assumed to be an *admin* + - if it contains even just one property, it acts as an allow-list: only the listed properties are allowed + - any non-listed properties are denied + - permissable values for each section (other than `IsAdmin`) are: `read` or `write`; to deny access to a property you must remove the property from the `user_permissions` object + +An *admin* user can be identified either by setting `IsAdmin` to `true` or by setting no properties in the `user_permissions` object. + +### User permissions in the Tyk Dashboard UI + +User permissions are configured in the user detail view: + +Admin account + +The configuration of each property will affect the dashboard navigation, with `denied` sections or screens hidden or disabled. Note that some side-effects can occur if pages that make use of multiple APIs to fetch configuration data cross over e.g. policies and API Definition listings. + +Selecting the **Account is Admin** checkbox from the Dashboard gives the user full access (it has the same [effect](/api-management/user-management#admin-users) as the `IsAdmin` property). + +### Custom User Permissions + +You can create your own custom permissions for use with the [Open Policy Agent (OPA)](/api-management/dashboard-configuration#extend-permissions-using-open-policy-agent-opa) using the [Additional Permissions](/api-management/dashboard-configuration#additional-permissions-api) endpoint in the Tyk Dashboard Admin API. This allows you to add and delete (CRUD) a list of additional (custom) permissions for your Dashboard users. Once created, a custom permission will be added to standard list of user permissions. + +You can also configure these custom permissions in the `security.additional_permissions` [map](/tyk-dashboard/configuration#securityadditional_permissions) in the Tyk Dashboard configuration file. + + +## API Ownership + +API Ownership is the concept of Role Based Access Control applied to the portfolio of APIs deployed on your Tyk Gateways and managed from your Tyk Dashboard. + +If you have multiple teams, where each team maintains its own APIs and you want to limit access of the dashboard to the team level. For each API, you can assign owners, where an owner can be either an individual user or user group. Only owners have access to these APIs, and objects created based on them like policies or analytics. + +### Multi-team setup using API Ownership + + + +The availability of this feature [depends on your license](/api-management/user-management#). + + + +### When to use API Ownership +#### Governing a multi-team API portfolio +API ownership is a key capability when you have multiple API development teams each working on their own suite of APIs. You can use API Ownership to simplify the experience of those developers when accessing Tyk by reducing the "clutter" of APIs that they are not working with, and also to avoid the risk of users accidentally or maliciously interfering with another team's APIs. + +#### Avoiding data leakage between users +The [user owned analytics](/api-management/user-management#owned-analytics) feature allows you to prevent users from seeing the traffic to (and existence of) APIs for which they are not responsible. This reduces the opportunity for data leakage within your business. + +### How API Ownership works +By default, APIs and associated objects (such as policies and Dashboard analytics) are visible to all Tyk Dashboard users. + +A Dashboard User (or User Group) can be assigned as the *owner* of an API, granting that user (or user group) exclusive visibility of and - given appropriate permissions - the ability to manage all the Tyk objects relating to that API, such as policies, key requests and Dashboard analytics. + +Once an API has been assigned an *owner*, all non-owning users will lose visibility of - and access to - that API in the Tyk Dashboard. + +Where there is a conflict, for example when a policy is associated with multiple APIs and the user owns only one of those APIs, the user will have access to the object (though not the other APIs themselves). + +#### API Ownership example +Imagine that you have two APIs: API1, API2. +You have three teams which have different roles and responsibilities as follows: +- **TeamA** which must have access to configure API1 +- **TeamB** which must have access to configure API2 +- **TeamAnalytics** which should only have access to view the analytics for both API1 and API2 + +To implement this structure, you would create three user groups and assign [permissions](/api-management/user-management#user-permissions) as indicated: +- **TeamA** requires `"apis": "write"` +- **TeamB** requires `"apis": "write"` +- **TeamAnalytics** requires `"analytics": "read"` + +Having configured the user groups, you can then assign API ownership as follows: +- API1 - **TeamA**, **TeamAnalytics** +- API2 - **TeamB**, **TeamAnalytics** + +Thus: +**TeamA** will have API `write` access only to API1 +**TeamB** will have API `write` access only to API2 +**TeamAnalytics** will have Analytics `read` access to both APIs + +#### Enabling API Ownership +API Ownership must be enabled in your Tyk Dashboard configuration, which you can do using either of the following approaches: + - set `enable_ownership` to `true` in your `tyk_analytics.conf` + - set the `TYK_DB_ENABLEOWNERSHIP` environment variable to `true` + +#### Owned Analytics +Access to Tyk Dashboard's [traffic analytics](/api-management/dashboard-configuration#traffic-analytics) is controlled via the `analytics` permission in the user or user group access control configuration. The default behavior of this control is to grant or restrict access to all traffic analytics and does not take into account API ownership. + +The additional `owned_analytics` permission was added in Tyk Dashboard v5.1 (and LTS patches v4.0.14 and v5.0.3) to provide more granular access to traffic analytics. By configuring this permission, the user (or user group) will gain visibility only of those analytics that can be filtered by API (due to the method Tyk Pump uses to aggregate the analytics records). + +Currently, only [API Usage](/api-management/dashboard-configuration#activity-by-api) and [Error Counts](/api-management/dashboard-configuration#activity-by-error) are available to users with the `owned_analytics` permission. + +Note that the `owned_analytics` permission depends upon the `analytics` permission being granted (set to `read`) - without this, the more granular control is ignored and the user will not have visibility of any Tyk Dashboard analytics. + +In the Dashboard UI, the control for `owned_analytics` is implemented as a drop-down option (`all` or `owned`) on the `analytics` permission. +Permissions with API Ownership + +### Managing API owners using the Dashboard UI +The Dashboard UI provides a simple method for managing *ownership* for your APIs, where you can select from the lists of users and user groups that have been created in the Dashboard. Users and user groups are managed in separate lists for ease of use. + +#### Using Tyk OAS APIs +When using Tyk OAS APIs, the option to assign owner(s) to an API is provided on the **Access** tab in the API Designer. You simply select the owner (or owners) that you wish to assign to the API from the drop-down boxes: +API ownership section for Tyk OAS APIs + +You can remove owners from the API by clicking on the `x` next to their name in the drop-down/selection box. + +#### Using Tyk Classic APIs +When using Tyk Classic APIs, the option to assign owner(s) to an API is provided on the **Core Settings** tab in the API Designer. You simply select the owner (or owners) that you wish to assign to the API from the drop-down boxes: +API ownership section for Tyk Classic APIs + +You can remove owners from the API by deselecting them from the drop-down. + +### Managing API owners using the Dashboard API +The [Tyk Dashboard API](/tyk-dashboard-api) provides endpoints to manage API ownership directly, if you are not using the API Designer. + +#### Using Tyk OAS APIs +When working with Tyk OAS APIs, you can manage owners for an API using these endpoints: + +| Method | Endpoint path | Action | +| :-------- | :------------------------- | :---------------------------------------------------------------------------------------- | +| `PUT` | `/api/apis/{apiID}/access` | Assign a list of owners to the specified API | +| `GET` | `/api/apis/{apiID}/access` | Retrieve the list of owners of the specified API | + +For each of these endpoints, the payload consists of two string lists: one for user IDs, the other for user group IDs. +```json expandable +{ + "userIds": [ + "string" + ], + "userGroupIds": [ + "string" + ] +} +``` + +#### Using Tyk Classic APIs +When working with Tyk Classic APIs, you manage owners for an API by modifying the `user_owners` and `user_group_owners` fields in the API definition and then updating the API in Tyk with that using these endpoints: + +| Method | Endpoint | Action | +| :-------- | :--------------------- | :----------------------------------------------------------------------------------------------------------------------- | +| `PUT` | `/api/apis/{apiID}` | Update the API definition for the specified API - CRUD API owners in the `user_owners` and `user_group_owners` fields | +| `GET` | `/api/apis/{apiID}` | Retrieve the API definition for the specified API - ownership details are included in `user_owners` and `user_group_owners` fields | + +## Manage Tyk Dashboard Users in Multiple Organizations + +If you have deployed multiple [Tyk Organizations](/api-management/dashboard-configuration#organizations), you may have users that need access to more than one Organization (known as a "multi-org user"). **This functionality requires a specific Tyk license.** + +To support multi-org users, you must first enable the feature in your Dashboard configuration by setting either of the following to `true`: + - `"enable_multi_org_users"` in `tyk_analytics.conf` + - `TYK_DB_ENABLEMULTIORGUSERS` environment variable + +You then must create users in both Organizations with identical credentials. + +During the login flow the user will see an additional page asking them to pick which available Organization they wish to log into. Once logged in, the user will have an additional drop-down in the top right navigation menu allowing them to switch between Organizations quickly. + + + +A user that does not belong to an Organization is sometimes referred to as an *unbounded user*. These users have visibility across all Organizations, but should be granted read-only access. + + + +## Single Sign-On integration +You can integrate your existing identity management server with the Tyk Dashboard, as explained in our detailed [Single Sign-On (SSO) guide](/api-management/external-service-integration#single-sign-on-sso). **This functionality is available with all Tyk licenses except Tyk Classic Cloud.** + +By default all users who login via SSO are granted admin permissions. You can change this behavior by setting either default permissions for *[users](/api-management/user-management#manage-tyk-dashboard-users)* or by creating a default *[user group](/api-management/user-management#manage-tyk-dashboard-user-groups)* to which all new users are assigned. With some IDPs you can automatically assign different SSO users to different *user groups* by dynamically mapping the IDP's user groups, for example with [Azure AD](/api-management/external-service-integration#user-group-mapping). + +If you want to maintain an individual set of permissions for your SSO users, you must first enable SSO user lookup in your Dashboard configuration by setting either of the following to `true`: + - `"sso_enable_user_lookup"` in `tyk_analytics.conf` + - `TYK_DB_SSOENABLEUSERLOOKUP` environment variable + +You must then create a *user* in the Dashboard with the required permissions and matching email address. During the SSO login flow, if a user with the same email address is found in the existing organization, their permissions are applied. diff --git a/apim.mdx b/apim.mdx new file mode 100644 index 00000000..2c29fb51 --- /dev/null +++ b/apim.mdx @@ -0,0 +1,53 @@ +--- +title: "Tyk API Management Deployment Options" +'og:description': "How to decide on which Tyk deployment option is best for you" +tags: ['Tyk API Management', 'Licensing', 'Open Source', 'Self-Managed', 'Tyk Cloud', 'API Gateway'] +sidebarTitle: "Deployment Options" +--- + +import SelfManagedLicensingInclude from '/snippets/self-managed-licensing-include.mdx'; +import { ButtonLeft } from '/snippets/ButtonLeft.mdx'; + +Tyk API Platform offers various deployment options, consisting of both [open source and proprietary](/tyk-stack) +components. + +Choosing the right one for your organization depends on your specific requirements and preferences. +
Don’t hesitate to contact us for assistance + +| | [Open Source](/tyk-open-source) | [Self-Managed](/tyk-self-managed/install) | [Cloud](https://account.cloud-ara.tyk.io/signup) +|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------|-------------------|--------- +| API Gateway Capabilities
  • Rate Limiting
  • Authentication
  • API Versioning
  • Granular Access Control
  • GraphQL
  • and [much more](/tyk-open-source)
| βœ… |βœ… |βœ… +| [Version Control](/api-management/automations/sync) Integration | - |βœ… |βœ… +| [API Analytics Exporter](/tyk-pump) | βœ… |βœ… |βœ… +| [Tyk Dashboard](/tyk-dashboard) | - |βœ… |βœ… +| [Single Sign On (SSO)](/api-management/external-service-integration#single-sign-on-sso) | - |βœ… |βœ… +| [RBAC and API Teams](/api-management/user-management#) | - |βœ… |βœ… +| [Universal Data Graph](/api-management/data-graph#overview) | - |βœ… |βœ… +| [Multi-Tenant](/api-management/dashboard-configuration#organizations) | - |βœ… |βœ… +| [Multi-Data Center](/api-management/mdcb#managing-geographically-distributed-gateways-to-minimize-latency-and-protect-data-sovereignty) | - |βœ… |βœ… +| [Developer Portal](/tyk-developer-portal) | - |βœ… |βœ… +| [Developer API Analytics](/api-management/dashboard-configuration#traffic-analytics) | - |βœ… |βœ… +| Hybrid Deployments | - |- |βœ… +| Fully-Managed SaaS | - |- |βœ… +| [HIPAA, SOC2, PCI](https://tyk.io/governance-and-auditing/) | βœ… |βœ… | - + + +## Licensing + +### Self-managed (On-Prem) + + + +### Cloud (Software as a Service / SaaS) + +Tyk cloud is a fully managed service that makes it easy for API teams to create, secure, publish and maintain APIs at any scale, anywhere in the world. Tyk Cloud includes everything you need to manage your global API ecosystem. + +Get your free account [here](https://tyk.io/sign-up/). + +### Open Source (OSS) + +The Tyk Gateway is the backbone of all our solutions and can be deployed for free, forever. It offers various [installation options](/apim/open-source/installation) to suit different needs. + +Visit the [OSS section](/tyk-open-source) for more information on it and other open source components. + +Explore the various open and closed source [Tyk components](/tyk-stack) that are part of the Tyk platform solutions. diff --git a/apim/open-source/installation.mdx b/apim/open-source/installation.mdx new file mode 100644 index 00000000..d7965a14 --- /dev/null +++ b/apim/open-source/installation.mdx @@ -0,0 +1,742 @@ +--- +title: "Installation Options for Tyk Gateway" +'og:description': "This page serves as a comprehensive guide to installing Tyk Gateway Open Source" +sidebarTitle: "Installation Options" +tags: ['installation', 'migration', 'open source'] +--- + +import { ResponsiveGrid } from '/snippets/ResponsiveGrid.mdx'; + +## Introduction + +The backbone of all our products is our open source Gateway. You can install our Open Source / Community Edition on the following platforms: + + + + + +**Read time: 10 mins** + +Install with Docker. + + + +**Read time: 10 mins** + +Install with K8s. + + + +**Read time: 10 mins** + +Install with Ansible. + + + +**Read time: 10 mins** + +Install on RHEL / CentOS. + + + +**Read time: 10 mins** + +Install on Debian / Ubuntu. + + + +**Read time: 10 mins** + +Visit our Gateway GitHub Repo. + + + + + +## Install Tyk Gateway with Docker + +We will show you two methods of installing our Community Edition Gateway on Docker. +The quickest way to get started is using docker-compose. Visit our [Dockerhub](https://hub.docker.com/u/tykio/) to view the official images. + +### Prerequisites + +The following are required for a Tyk OSS installation: + - Redis - Required for all Tyk installations. + Simple Redis installation instructions are included below. + - MongoDB - Required only if you chose to use the Tyk Pump with your Tyk OSS installation. Same goes with any [other pump data stores](/api-management/tyk-pump#external-data-stores) you choose to use. + +### Steps for Installation + +1. **Create a network** + +```bash +docker network create tyk +``` + +2. **Deploy Redis into the network, with the `6379` port open** + +```bash +docker run -itd --rm --name tyk-redis --network tyk -p 127.0.0.1:6379:6379 redis:4.0-alpine +``` + +3. **Next, let's download a JSON `tyk.conf` configuration file** + +```bash +wget https://raw.githubusercontent.com/TykTechnologies/tyk-gateway-docker/master/tyk.standalone.conf +``` + +4. **Run the Gateway, mounting the conf file into the container** + +```bash expandable +docker run \ + --name tyk_gateway \ + --network tyk \ + -p 8080:8080 \ + -v $(pwd)/tyk.standalone.conf:/opt/tyk-gateway/tyk.conf \ + -v $(pwd)/apps:/opt/tyk-gateway/apps \ + docker.tyk.io/tyk-gateway/tyk-gateway:latest +``` + + +### Test Installation + +Your Tyk Gateway is now configured and ready to use. Confirm this by making a network request to the 'hello' endpoint: + +```curl +curl localhost:8080/hello +``` + +Output should be similar to that shown below: +```json +{"status":"pass","version":"v3.2.1","description":"Tyk GW"} +``` + + +## Install Tyk Gateway with Kubernetes + +The main way to install the Open Source *Tyk Gateway* in a Kubernetes cluster is via Helm charts. +We are actively working to add flexibility and more user flows to our chart. Please reach out +to our teams on support or the community forum if you have questions, requests or suggestions for improvements. + +Get started with our [Quick Start guide](#quick-start-with-helm-chart) or go to [Tyk Open Source helm chart](/product-stack/tyk-charts/tyk-oss-chart) for detailed installation instructions and configuration options. + +### Quick Start with Helm Chart + +At the end of this quick start, Tyk Gateway should be accessible through the service `gateway-svc-tyk-oss-tyk-gateway` at port `8080`. +The following guides provide instructions to install Redis and Tyk Open Source with default configurations. It is intended for a quick start only. For production, you should install and configure Redis separately. + +#### Prerequisites + +1. [Kubernetes 1.19+](https://kubernetes.io/docs/setup/) +2. [Helm 3+](https://helm.sh/docs/intro/install/) + +#### Steps for Installation + +1. **Install Redis and Tyk** + +```bash expandable +NAMESPACE=tyk-oss +APISecret=foo +REDIS_BITNAMI_CHART_VERSION=19.0.2 + +helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ +helm repo update + +helm upgrade tyk-redis oci://registry-1.docker.io/bitnamicharts/redis -n $NAMESPACE --create-namespace --install --version $REDIS_BITNAMI_CHART_VERSION + +helm upgrade tyk-oss tyk-helm/tyk-oss -n $NAMESPACE --create-namespace \ + --install \ + --set global.secrets.APISecret="$APISecret" \ + --set global.redis.addrs="{tyk-redis-master.$NAMESPACE.svc.cluster.local:6379}" \ + --set global.redis.passSecret.name=tyk-redis \ + --set global.redis.passSecret.keyName=redis-password +``` + +2. **Done!** + +Now Tyk Gateway should be accessible through service `gateway-svc-tyk-oss-tyk-gateway` at port `8080`. + +You are now ready to [create an API](/api-management/gateway-config-managing-classic#create-an-api). + +For the complete installation guide and configuration options, please see [Tyk OSS Helm Chart](/product-stack/tyk-charts/tyk-oss-chart). + +### Configure Legacy Tyk Headless Helm Chart + + +`tyk-headless` chart is deprecated. Please use our Tyk Chart for Tyk Open Source at [tyk-oss](#quick-start-with-helm-chart) instead. + +We recommend all users migrate to the `tyk-oss` Chart. Please review the [Configuration](#quick-start-with-helm-chart) section of the new helm chart and cross-check with your existing configurations while planning for migration. + + + +This is the preferred (and easiest) way to install the Tyk OSS Gateway on Kubernetes. +It will install Tyk gateway in your Kubernetes cluster where you can add and manage APIs directly or via the *Tyk Operator*. + +#### Prerequisites + +The following are required for a Tyk OSS installation: +1. Redis - required for all the Tyk installations and must be installed in the cluster or reachable from inside K8s. + You can find instructions for a simple Redis installation below. +2. MongoDB/SQL - Required only if you choose to use the MongoDB/SQL Tyk pump with your Tyk OSS installation. The same goes for any + [other pump](/api-management/tyk-pump#external-data-stores) you choose to use. +3. Helm - Tyk Helm supports the Helm 3+ version. + +#### Steps for Installation + +As well as our official OSS Helm repo, you can also find it in [ArtifactHub](https://artifacthub.io/packages/helm/tyk-helm/tyk-headless). +[Open in ArtifactHub](https://artifacthub.io/packages/helm/tyk-helm/tyk-headless) + +If you are interested in contributing to our charts, suggesting changes, creating PRs, or any other way, +please use [GitHub Tyk-helm-chart repo](https://github.com/TykTechnologies/tyk-helm-chart/tree/master/tyk-headless) + +1. **Add Tyk official Helm repo** + +```bash +helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ +helm repo update +``` + +2. **Create a namespace for Tyk deployment** + +```bash +kubectl create namespace tyk +``` + +3. **Getting values.yaml** + +Before we proceed with installation of the chart you may need to set some custom values. +To see what options are configurable on a chart and save those options to a custom `values.yaml` file run: + +```bash +helm show values tyk-helm/tyk-headless > values.yaml +``` + +Some of the necessary configuration parameters will be explained in the next steps. + +4. **Installing Redis** + +* Recommended: via *Bitnami* chart - For Redis, you can use these rather excellent chart provided by Bitnami. +Copy the following commands to add it: + + ```bash expandable + helm repo add bitnami https://charts.bitnami.com/bitnami + helm install tyk-redis bitnami/redis -n tyk --version 19.0.2 + ``` + + + +Please make sure you are installing Redis versions that are supported by Tyk. Please refer to Tyk docs to get a list of [supported versions](/tyk-configuration-reference/redis-cluster-sentinel#supported-versions). + + + +Follow the notes from the installation output to get connection details and password. + +``` + Redis(TM) can be accessed on the following DNS names from within your cluster: + + tyk-redis-master.tyk.svc.cluster.local for read/write operations (port 6379) + tyk-redis-replicas.tyk.svc.cluster.local for read-only operations (port 6379) + + export REDIS_PASSWORD=$(kubectl get secret --namespace tyk tyk-redis -o jsonpath="{.data.redis-password}" | base64 --decode) +``` expandable + +The DNS name of your Redis as set by Bitnami is `tyk-redis-master.tyk.svc.cluster.local:6379` +You can update them in your local `values.yaml` file under `redis.addrs` and `redis.pass` +Alternatively, you can use `--set` flag to set it in the Tyk installation. For example `--set redis.pass=$REDIS_PASSWORD` + +**For evaluation only: Use *simple-redis* chart** + + + +Another option for Redis, to get started quickly, is to use our *simple-redis* chart. +Please note that these provided charts must never be used in production or for anything +but a quick start evaluation only. Use Bitnami Redis or Official Redis Helm chart in any other case. +We provide this chart, so you can quickly deploy *Tyk gateway*, but it is not meant for long-term storage of data. + + + +```bash +helm install redis tyk-helm/simple-redis -n tyk +``` + +5. **Installing Tyk Open Source Gateway** + +```bash +helm install tyk-ce tyk-helm/tyk-headless -f values.yaml -n tyk + ``` expandable + +Please note that by default, Gateway runs as `Deployment` with `ReplicaCount` as 1. You should not update this part because multiple instances of OSS gateways won't sync the API Definition. + +#### Installation Video + +See our short video on how to install the Tyk Open Source Gateway. +Please note that this video shows the use of the Github repository since it was recorded before the official repo was available, However, +it's very similar to the above commands. + + + +#### Pump Installation +By default pump installation is disabled. You can enable it by setting `pump.enabled` to `true` in `values.yaml` file. +Alternatively, you can use `--set pump.enabled=true` while doing Helm install. + +**Quick Pump configuration(Supported from tyk helm v0.10.0)** +*1. Mongo Pump* + +To configure the Mongo pump, make the following changes in `values.yaml` file: +1. Set `backend` to `mongo`. +2. Set connection string in `mongo.mongoURL`. + +*2. Postgres Pump* + +To configure the Postgres pump, make the following changes in `values.yaml` file: +1. Set `backend` to `postgres`. +2. Set connection string parameters in `postgres` section. + +#### Optional - Using TLS +You can turn on the TLS option under the gateway section in your local `values.yaml` file which will make your Gateway +listen on port 443 and load up a dummy certificate. +You can set your own default certificate by replacing the file in the `certs/` folder. + +#### Optional - Mounting Files +To mount files to any of the Tyk stack components, add the following to the mounts array in the section of that component. + +For example: + ```bash + - name: aws-mongo-ssl-cert + filename: rds-combined-ca-bundle.pem + mountPath: /etc/certs +``` + +#### Optional - Tyk Ingress +To set up an ingress for your Tyk Gateways see our [Tyk Operator GitHub repository](https://github.com/TykTechnologies/tyk-operator). + + +## Install Tyk Gateway with Ansible + +### Prerequisites + +1. [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) is required to run the following commands. +2. Ensure port `8080` is open: this is used in this guide for Gateway traffic (the API traffic to be proxied). + +### Steps for Installation +1. Clone the [tyk-ansible](https://github.com/TykTechnologies/tyk-ansible) repository + +```bash +$ git clone https://github.com/TykTechnologies/tyk-ansible +``` + +2. `cd` into the directory +```.bash +$ cd tyk-ansible +``` + +3. Run the init script to initialize the environment + +```bash +$ sh scripts/init.sh +``` + +4. Modify the `hosts.yml` file to update SSH variables to your server(s). For more information about the host file, visit the [Ansible inventory documentation] (https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html) + +5. Run ansible-playbook to install `tyk-ce` + +```bash +$ ansible-playbook playbook.yaml -t tyk-ce -t redis +``` + +You can choose to not install Redis by removing the `-t redis`. However, Redis is a requirement and needs to be installed for the gateway to run. + +### Supported Distributions +| Distribution | Version | Supported | +| --------- | :---------: | :---------: | +| Amazon Linux | 2 | βœ… | +| CentOS | 8 | βœ… | +| CentOS | 7 | βœ… | +| Debian | 10 | βœ… | +| Debian | 9 | βœ… | +| RHEL | 8 | βœ… | +| RHEL | 7 | βœ… | +| Ubuntu | 21 | βœ… | +| Ubuntu | 20 | βœ… | +| Ubuntu | 18 | βœ… | +| Ubuntu | 16 | βœ… | + +### Variables +- `vars/tyk.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| secrets.APISecret | `352d20ee67be67f6340b4c0605b044b7` | API secret | +| secrets.AdminSecret | `12345` | Admin secret | +| redis.host | | Redis server host if different than the host url | +| redis.port | `6379` | Redis server listening port | +| redis.pass | | Redis server password | +| redis.enableCluster | `false` | Enable if Redis is running in cluster mode | +| redis.storage.database | `0` | Redis server database | +| redis.tls | `false` | Enable if Redis connection is secured with SSL | +| gateway.service.host | | Gateway server host if different than the host url | +| gateway.service.port | `8080` | Gateway server listening port | +| gateway.service.proto | `http` | Gateway server protocol | +| gateway.service.tls | `false` | Set to `true` to enable SSL connections | +| gateway.sharding.enabled | `false` | Set to `true` to enable filtering (sharding) of APIs | +| gateway.sharding.tags | | The tags to use when filtering (sharding) Tyk Gateway nodes. Tags are processed as OR operations. If you include a non-filter tag (e.g. an identifier such as `node-id-1`, this will become available to your Dashboard analytics) | + +- `vars/redis.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| redis_bind_interface | `0.0.0.0` | Binding address of Redis | + +Read more about Redis configuration [here](https://github.com/geerlingguy/ansible-role-redis). + +## Install Tyk Gateway with Ubuntu + +The Tyk Gateway can be installed following different installation methods including *Ansible* and *Shell*. Please select by clicking the tab with the installation path most suitable for you. + +### Install Tyk Gateway On Ubuntu Through Shell + +#### + +| Distribution | Version | Supported | +| --------- | :---------: | :---------: | +| Debian | 11 | βœ… | +| Ubuntu | 20 | βœ… | +| Ubuntu | 18 | βœ… | +| Ubuntu | 16 | βœ… | + +#### Prerequisites + +1. Ensure port `8080` is open: this is used in this guide for Gateway traffic (the API traffic to be proxied). + +#### Steps for Installation + +1. **Install Redis** + +```console +$ sudo apt-get install -y redis-server +``` + +2. **First import the public key as required by Ubuntu APT** + +```console +$ sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927 +``` + +3. **Run Installation Scripts via our PackageCloud Repositories** + +From [https://packagecloud.io/tyk/tyk-gateway](https://packagecloud.io/tyk/tyk-gateway) you have the following options: + +* Via the correct package for your Ubuntu version. We have packages for the following: + * Xenial + * Trusty + * Precise + +* Via Quick Installation Instructions. You can use: + * [Manual Instructions](https://packagecloud.io/tyk/tyk-gateway/install#manual-deb) + * [Chef](https://packagecloud.io/tyk/tyk-gateway/install#chef) + * [Puppet](https://packagecloud.io/tyk/tyk-gateway/install#puppet) + * [CI and Build Tools](https://packagecloud.io/tyk/tyk-gateway/ci) + +4. **Configure The Gateway** + +You can set up the core settings for the Tyk Gateway with a single setup script, however for more involved deployments, you will want to provide your own configuration file. + + + +You need to replace `` for `--redishost=` with your own value to run this script. + + + + +```console +$ sudo /opt/tyk-gateway/install/setup.sh --listenport=8080 --redishost= --redisport=6379 --domain="" +``` + +What you've done here is tell the setup script that: + +* `--listenport=8080`: Listen on port `8080` for API traffic. +* `--redishost=`: The hostname for Redis. +* `--redisport=6379`: Use port `6379` for Redis. +* `--domain=""`: Do not filter domains for the Gateway, see the note on domains below for more about this. + +In this example, you don't want Tyk to listen on a single domain. It is recommended to leave the Tyk Gateway domain unbounded for flexibility and ease of deployment. + +5. **Starting Tyk** + +The Tyk Gateway can be started now that it is configured. Use this command to start the Tyk Gateway: +```console +$ sudo service tyk-gateway start +``` + +### Install Tyk Gateway On Ubuntu Through Ansible + +#### Supported Distributions + +| Distribution | Version | Supported | +| --------- | :---------: | :---------: | +| Debian | 11 | βœ… | +| Ubuntu | 20 | βœ… | +| Ubuntu | 18 | βœ… | +| Ubuntu | 16 | βœ… | + +#### Prerequisites + +Before you begin the installation process, make sure you have the following: +- [Git](https://git-scm.com/download/linux) - required for getting the installation files. +- [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) is required to run the following commands. +- Ensure port `8080` is open: this is used in this guide for Gateway traffic (the API traffic to be proxied). + +#### Steps for Installation + +1. **Clone the [tyk-ansible](https://github.com/TykTechnologies/tyk-ansible) repository** + +```bash +$ git clone https://github.com/TykTechnologies/tyk-ansible +``` + +2. **`cd` into the directory** +```bash +$ cd tyk-ansible +``` + +3. **Run initalisation script to initialise environment** + +```bash +$ sh scripts/init.sh +``` + +4. Modify `hosts.yml` file to update ssh variables to your server(s). You can learn more about the hosts file [here](https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html) + +5. **Run ansible-playbook to install `tyk-gateway-ce`** + +```bash +$ ansible-playbook playbook.yaml -t tyk-gateway-ce -t redis +``` + + +Installation flavors can be specified by using the -t {tag} at the end of the ansible-playbook command. In this case we are using: +-`tyk-gateway-ce`: Tyk Gateway with CE config +-`redis`: Redis database as Tyk Gateway dependency + + + +#### Variables + +- `vars/tyk.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| secrets.APISecret | `352d20ee67be67f6340b4c0605b044b7` | API secret | +| secrets.AdminSecret | `12345` | Admin secret | +| redis.host | | Redis server host if different than the hosts url | +| redis.port | `6379` | Redis server listening port | +| redis.pass | | Redis server password | +| redis.enableCluster | `false` | Enable if redis is running in cluster mode | +| redis.storage.database | `0` | Redis server database | +| redis.tls | `false` | Enable if redis connection is secured with SSL | +| gateway.service.host | | Gateway server host if different than the hosts url | +| gateway.service.port | `8080` | Gateway server listening port | +| gateway.service.proto | `http` | Gateway server protocol | +| gateway.service.tls | `false` | Set to `true` to enable SSL connections | +| gateway.sharding.enabled | `false` | Set to `true` to enable filtering (sharding) of APIs | +| gateway.sharding.tags | | The tags to use when filtering (sharding) Tyk Gateway nodes. Tags are processed as OR operations. If you include a non-filter tag (e.g. an identifier such as `node-id-1`, this will become available to your Dashboard analytics) | + +- `vars/redis.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| redis_bind_interface | `0.0.0.0` | Binding address of Redis | + +Read more about Redis configuration [here](https://github.com/geerlingguy/ansible-role-redis). + + +## Install Tyk Gateway on Red Hat (RHEL / CentOS) + +The Tyk Gateway can be installed following different installation methods including *Shell* and *Ansible*. Please select by clicking the tab with the installation path most suitable for you. + +### Install Tyk Gateway Through Shell + +#### Supported Distributions + +| Distribution | Version | Supported | +| --------- | :---------: | :---------: | +| CentOS | 8 | βœ… | +| CentOS | 7 | βœ… | +| RHEL | 8 | βœ… | +| RHEL | 7 | βœ… | + +#### Prerequisites + +Before you begin the installation process, make sure you have the following: + +* Ensure port `8080` is open for Gateway traffic (the API traffic to be proxied). +* The Tyk Gateway has a [dependency](/tyk-configuration-reference/redis-cluster-sentinel#supported-versions) on Redis. Follow the steps provided by Red Hat to make the installation of Redis, conducting a [search](https://access.redhat.com/search/?q=redis) for the correct version and distribution. + +#### Steps for Installation +1. **Create Tyk Gateway Repository Configuration** + +Create a file named `/etc/yum.repos.d/tyk_tyk-gateway.repo` that contains the repository configuration settings for YUM repositories `tyk_tyk-gateway` and `tyk_tyk-gateway-source` used to download packages from the specified URLs. This includes GPG key verification and SSL settings, on a Linux system. + +Make sure to replace `el` and `8` in the config below with your Linux distribution and version: +```bash expandable +[tyk_tyk-gateway] +name=tyk_tyk-gateway +baseurl=https://packagecloud.io/tyk/tyk-gateway/el/8/$basearch +repo_gpgcheck=1 +gpgcheck=0 +enabled=1 +gpgkey=https://packagecloud.io/tyk/tyk-gateway/gpgkey +sslverify=1 +sslcacert=/etc/pki/tls/certs/ca-bundle.crt +metadata_expire=300 + +[tyk_tyk-gateway-source] +name=tyk_tyk-gateway-source +baseurl=https://packagecloud.io/tyk/tyk-gateway/el/8/SRPMS +repo_gpgcheck=1 +gpgcheck=0 +enabled=1 +gpgkey=https://packagecloud.io/tyk/tyk-gateway/gpgkey +sslverify=1 +sslcacert=/etc/pki/tls/certs/ca-bundle.crt +metadata_expire=300 +``` + +Update your local yum cache by running: +```bash +sudo yum -q makecache -y --disablerepo='*' --enablerepo='tyk_tyk-gateway' +``` + +2. **Install Tyk Gateway** + +Install the Tyk Gateway using yum: +```bash +sudo yum install -y tyk-gateway +``` + + +You may be asked to accept the GPG key for our two repos and when the package installs, hit yes to continue. + + + +3. **Start Redis** + +If Redis is not running then start it using the following command: +```bash +sudo service redis start +``` +4. **Configuring The Gateway** + +You can set up the core settings for the Tyk Gateway with a single setup script, however for more complex deployments you will want to provide your own configuration file. + + + +Replace `` in `--redishost=` with your own value to run this script. + + + +```bash +sudo /opt/tyk-gateway/install/setup.sh --listenport=8080 --redishost= --redisport=6379 --domain="" +``` + +What you've done here is told the setup script that: + +* `--listenport=8080`: Listen on port `8080` for API traffic. +* `--redishost=`: The hostname for Redis. +* `--redisport=6379`: Use port `6379` for Redis. +* `--domain=""`: Do not filter domains for the Gateway, see the note on domains below for more about this. + +In this example, you don't want Tyk to listen on a single domain. It is recommended to leave the Tyk Gateway domain unbounded for flexibility and ease of deployment. + +5. **Start the Tyk Gateway** + +The Tyk Gateway can be started now that it is configured. Use this command to start the Tyk Gateway: +```bash +sudo service tyk-gateway start +``` + +### Install Tyk Gateway Through Ansible + +#### Supported Distributions + +| Distribution | Version | Supported | +| --------- | :---------: | :---------: | +| CentOS | 8 | βœ… | +| CentOS | 7 | βœ… | +| RHEL | 8 | βœ… | +| RHEL | 7 | βœ… | + +#### Prerequisites +Before you begin the installation process, make sure you have the following: + +1. [Git](https://git-scm.com/download/linux) - required for getting the installation files. +2. [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) - required for running the commands below. +3. Ensure port `8080` is open: this is used in this guide for Gateway traffic (the API traffic to be proxied). + +#### Steps for Installation + +1. **Clone the [tyk-ansible](https://github.com/TykTechnologies/tyk-ansible) repository** + +```bash +$ git clone https://github.com/TykTechnologies/tyk-ansible +``` + +2. **`cd` into the directory** +```bash +$ cd tyk-ansible +``` + +3. **Run the initalisation script to initialise your environment** + +```bash +$ sh scripts/init.sh +``` + +4. Modify the `hosts.yml` file to update ssh variables to your server(s). You can learn more about the hosts file [here](https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html) + +5. **Run ansible-playbook to install `tyk-gateway-ce`** + +```bash +$ ansible-playbook playbook.yaml -t tyk-gateway-ce -t redis +``` + + +Installation flavors can be specified by using the -t {tag} at the end of the ansible-playbook command. In this case we are using: + -`tyk-gateway-ce`: Tyk Gateway with CE config + -`redis`: Redis database as Tyk Gateway dependency + + + +#### Variables +- `vars/tyk.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| secrets.APISecret | `352d20ee67be67f6340b4c0605b044b7` | API secret | +| secrets.AdminSecret | `12345` | Admin secret | +| redis.host | | Redis server host if different than the hosts url | +| redis.port | `6379` | Redis server listening port | +| redis.pass | | Redis server password | +| redis.enableCluster | `false` | Enable if redis is running in cluster mode | +| redis.storage.database | `0` | Redis server database | +| redis.tls | `false` | Enable if redis connection is secured with SSL | +| gateway.service.host | | Gateway server host if different than the hosts url | +| gateway.service.port | `8080` | Gateway server listening port | +| gateway.service.proto | `http` | Gateway server protocol | +| gateway.service.tls | `false` | Set to `true` to enable SSL connections | +| gateway.sharding.enabled | `false` | Set to `true` to enable filtering (sharding) of APIs | +| gateway.sharding.tags | | The tags to use when filtering (sharding) Tyk Gateway nodes. Tags are processed as OR operations. If you include a non-filter tag (e.g. an identifier such as `node-id-1`, this will become available to your Dashboard analytics) | + +- `vars/redis.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| redis_bind_interface | `0.0.0.0` | Binding address of Redis | + +Read more about Redis configuration [here](https://github.com/geerlingguy/ansible-role-redis). + +## Install Tyk Gateway on Killercoda + +[Killercoda](https://killercoda.com/about) gives you instant access to a real Linux or Kubernetes command-line environment via your browser. +You can try this [Killercoda Tyk scenario](https://killercoda.com/tyk-tutorials/scenario/Tyk-install-OSS-docker-compose) to walk through the installation of our Open Source Gateway using Docker Compose (the exact same flow shown above). + diff --git a/basic-config-and-security/security/authentication-authorization/hmac-signatures.mdx b/basic-config-and-security/security/authentication-authorization/hmac-signatures.mdx new file mode 100644 index 00000000..487f75ac --- /dev/null +++ b/basic-config-and-security/security/authentication-authorization/hmac-signatures.mdx @@ -0,0 +1,147 @@ +--- +title: "Sign Requests with HMAC" +'og:description': "How to configure HMAC Signatures in Tyk" +tags: ['Authentication', 'HMAC'] +sidebarTitle: "HMAC Signatures" +--- + +## Introduction + +Hash-Based Message Authentication Code (HMAC) Signing is an access token method that adds another level of security by forcing the requesting client to also send along a signature that identifies the request temporally to ensure that the request is from the requesting user, using a secret key that is never broadcast over the wire. + +Tyk currently implements the latest draft of the [HMAC Request Signing standard](http://tools.ietf.org/html/draft-cavage-http-signatures-05). + +HMAC Signing is a good way to secure an API if message reliability is paramount, it goes without saying that all requests should go via TLS/SSL to ensure that MITM attacks can be minimized. There are many ways of managing HMAC, and because of the additional encryption processing overhead requests will be marginally slower than more standard access methods. + +An HMAC signature is essentially some additional data sent along with a request to identify the end-user using a hashed value, in our case we encode the 'date' header of a request, the algorithm would look like: + +``` +Base64Encode(HMAC-SHA1("date: Mon, 02 Jan 2006 15:04:05 MST", secret_key)) +``` + +The full request header for an HMAC request uses the standard `Authorization` header, and uses set, stripped comma-delimited fields to identify the user, from the draft proposal: + +``` +Authorization: Signature keyId="hmac-key-1",algorithm="hmac-sha1",signature="Base64Encode(HMAC-SHA1(signing string))" +``` + +Tyk supports the following HMAC algorithms: "hmac-sha1", "hmac-sha256", "hmac-sha384", "hmac-sha512”, and reads value from algorithm header. You can limit the allowed algorithms by setting the `hmac.allowedAlgorithms` (Tyk Classic: `hmac_allowed_algorithms`) field in your API definition, like this: `"hmac_allowed_algorithms": ["hmac-sha256", "hmac-sha512"]`. + +The date format for an encoded string is: + +``` +Mon, 02 Jan 2006 15:04:05 MST +``` + +This is the standard for most browsers, but it is worth noting that requests will fail if they do not use the above format. + +## How Tyk validates the signature of incoming requests + +When an HMAC-signed request comes into Tyk, the key is extracted from the `Authorization` header, and retrieved from Redis. If a key exists then Tyk will generate its own signature based on the request's "date" header, if this generated signature matches the signature in the `Authorization` header the request is passed. + +### Supported headers + +Tyk API Gateway supports full header signing through the use of the `headers` HMAC signature field. This includes the request method and path using the`(request-target)` value. For body signature verification, HTTP Digest headers should be included in the request and in the header field value. + + + +All headers should be in lowercase. + + + +#### Date header not allowed for legacy .Net + +Older versions of some programming frameworks do not allow the Date header to be set, which can causes problems with implementing HMAC, therefore, if Tyk detects a `x-aux-date` header, it will use this to replace the Date header. + +### Clock Skew + +Tyk also implements the recommended clock-skew from the specification to prevent against replay attacks, a minimum lag of 300ms is allowed on either side of the date stamp, any more or less and the request will be rejected. This means that requesting machines need to be synchronised with NTP if possible. + +You can edit the length of the clock skew in the API Definition by setting the `hmac.allowedClockSkew` (Tyk Classic: `hmac_allowed_clock_skew`) value. This value will default to 0, which deactivates clock skew checks. + +## Setting up HMAC using the Dashboard + +To enable the use of HMAC Signing in your API from the Dashboard: + +1. Scroll to the **Authentication** options +2. Select **HMAC (Signed Authentication Key)** from the drop-down list +3. Configure your **HMAC Request Signing** settings. +4. Select **Strip Authorization Data** to strip any authorization data from your API requests. +5. Select the location of the signature in the request. + +Configuring HMAC request signing + +## Configuring your API to use HMAC Request Signing + +HMAC request signing is configured within the Tyk Vendor Extension by adding the `hmac` object within the `server.authentication` section and enabling authentication. + +You must indicate where Tyk should look for the request signature (`header`, `query` or `cookie`) and which `algorithm` will be used to encrypt the secret to create the signature. You can also optionally configure a limit for the `allowedClockSkew` between the timestamp in the signature and the current time as measured by Tyk. + +```yaml expandable +x-tyk-api-gateway: + server: + authentication: + enabled: true + hmac: + enabled: true + header: + enabled: true + name: Authorization + allowedAlgorithms: + - hmac-sha256 + allowedClockSkew: -1 +``` + +Note that URL query parameter keys and cookie names are case sensitive, whereas header names are case insensitive. + +You can optionally [strip the auth token](/api-management/client-authentication#managing-authorization-data) from the request prior to proxying to the upstream using the `authentication.stripAuthorizationData` field (Tyk Classic: `strip_auth_data`). + +### Using Tyk Classic + +As noted in the Tyk Classic API [documentation](/api-management/gateway-config-tyk-classic#configuring-authentication-for-tyk-classic-apis), you can select HMAC Request Signing using the `enable_signature_checking` option. + +## Registering an HMAC user with Tyk + +When using HMAC request signing, you need to provide Tyk with sufficient information to verify the client's identity from the signature in the request. You do this by creating and registering HMAC user [session objects](/api-management/policies#what-is-a-session-object) with Tyk. When these are created, a matching HMAC secret is also generated, which must be used by the client when signing their requests. + +The way that this is implemented is through the creation of a key that grants access to the API (as you would for an API protected by [auth token](/api-management/authentication/bearer-token)) and indicating that the key is to be used for HMAC signed requests by setting `hmac_enabled` to `true`. Tyk will return the HMAC secret in the response confirming creation of the key. + +When calling the API, the client would never use the key itself as a token, instead they must sign requests using the provided secret. + +## Generating a signature + +This code snippet gives an example of how a client could construct and generate a Request Signature. + +```{.copyWrapper} expandable +... + +refDate := "Mon, 02 Jan 2006 15:04:05 MST" + +// Prepare the request headers: +tim := time.Now().Format(refDate) +req.Header.Add("Date", tim) +req.Header.Add("X-Test-1", "hello") +req.Header.Add("X-Test-2", "world") + +// Prepare the signature to include those headers: +signatureString := "(request-target): " + "get /your/path/goes/here" +signatureString += "date: " + tim + "\n" +signatureString += "x-test-1: " + "hello" + "\n" +signatureString += "x-test-2: " + "world" + +// SHA1 Encode the signature +HmacSecret := "secret-key" +key := []byte(HmacSecret) +h := hmac.New(sha1.New, key) +h.Write([]byte(signatureString)) + +// Base64 and URL Encode the string +sigString := base64.StdEncoding.EncodeToString(h.Sum(nil)) +encodedString := url.QueryEscape(sigString) + +// Add the header +req.Header.Add("Authorization", + fmt.Sprintf("Signature keyId="9876",algorithm="hmac-sha1",headers="(request-target) date x-test-1 x-test-2",signature="%s"", encodedString)) + +... +``` diff --git a/basic-config-and-security/security/authentication-authorization/json-web-tokens.mdx b/basic-config-and-security/security/authentication-authorization/json-web-tokens.mdx new file mode 100644 index 00000000..b53478f1 --- /dev/null +++ b/basic-config-and-security/security/authentication-authorization/json-web-tokens.mdx @@ -0,0 +1,641 @@ +--- +title: "JSON Web Tokens (JWT)" +'og:description': "How to configure JSON Web Tokens in Tyk" +tags: ['Authentication', 'JWT', 'JSON Web Tokens'] +sidebarTitle: "JSON Web Tokens" +--- + +## Introduction + +JSON Web Tokens (JWT) are a compact, URL-safe means of representing claims to be transferred between two parties. They are commonly used in API authentication and authorization. + +## Configuring your API to use JWT authentication + +The OpenAPI Specification treats JWT authentication as a variant of [bearer authentication](https://swagger.io/docs/specification/v3_0/authentication/bearer-authentication/) in the `components.securitySchemes` object using the `type: http`, `scheme: bearer` and `bearerFormat: jwt`: + +```yaml expandable +components: + securitySchemes: + myAuthScheme: + type: http + scheme: bearer + bearerFormat: jwt + +security: + - myAuthScheme: [] +``` + +With this configuration provided by the OpenAPI description, in the Tyk Vendor Extension we need to enable authentication, to select this security scheme and to indicate where Tyk should look for the credentials. Usually the credentials will be provided in the `Authorization` header, but Tyk is configurable, via the Tyk Vendor Extension, to support custom header keys and credential passing via query parameter or cooke. + +```yaml expandable +x-tyk-api-gateway: + server: + authentication: + enabled: true + securitySchemes: + myAuthScheme: + enabled: true + header: + enabled: true + name: Authorization +``` + +Note that URL query parameter keys and cookie names are case sensitive, whereas header names are case insensitive. + +You can optionally [strip the user credentials](/api-management/client-authentication#managing-authorization-data) from the request prior to proxying to the upstream using the `authentication.stripAuthorizationData` field (Tyk Classic: `strip_auth_data`). + +With the JWT method selected, you'll need to configure Tyk to handle the specific configuration of JSON Web Tokens that clients will be providing. All of the JWT specific configuration is performed within the [authentication.jwt](/api-management/gateway-config-tyk-oas#jwt) object in the Tyk Vendor Extension. + +### Multiple User Credential Locations + +The OpenAPI Specification's `securitySchemes` mechanism allows only one location for the user credentials, but in some scenarios an API might need to support multiple potential locations to support different clients. + +The Tyk Vendor Extension supports this by allowing configuration of alternative locations in the JWT entry in `server.authentication.securitySchemes`. Building on the previous example, we can add optional query and cookie locations as follows: + +```yaml expandable +x-tyk-api-gateway: + server: + authentication: + enabled: true + securitySchemes: + myAuthScheme: + enabled: true + header: + enabled: true + name: Authorization + query: + enabled: true + name: query-auth + cookie: + enabled: true + name: cookie-auth +``` + +### Using Tyk Classic APIs + +As noted in the Tyk Classic API [documentation](/api-management/gateway-config-tyk-classic#configuring-authentication-for-tyk-classic-apis), you can select JSON Web Token authentication using the `use_jwt` option. + +## Protecting an API with JWT + +To protect an API with JWT, we need to execute the following steps: +* Set Authentication Mode +* Set the JWT Signing Method +* Set the Identity Source and Policy Field Name +* Set a Default Policy +* Generate a JWT + +### Set Authentication Mode + +1. [Select JSON Web Tokens](/basic-config-and-security/security/authentication-authorization/json-web-tokens#configuring-your-api-to-use-jwt-authentication) as the Authentication mode +2. [Set the cryptographic signing method](/basic-config-and-security/security/authentication-authorization/json-web-tokens#set-up-jwt-signing-method) to `HMAC (shared)` and the public secret as `tyk123` +3. Set the Identity Source and Policy Field Name + + Target Details: JSON Web Token + +### Set a Default Policy + +If Tyk cannot find a `pol` claim, it will apply this Default Policy. Select a policy that gives access to this API we are protecting, or [create one](/api-management/gateway-config-managing-classic#secure-an-api) if none exists. + +Make sure to save the changes to the API Definition. + +### Generate a JWT + +Let's generate a JWT so we can test our new protected API. + +Head on over to [https://jwt.io/](https://jwt.io/). Sign the default JWT with our HMAC Shared Secret `tyk123` in the VERIFY SIGNATURE section. Your screen should look similar to this: + +Auth Configuration + +Copy the Encoded JWT and let's make a cURL against the Tyk API Definition: + +``` +$ curl http://localhost:8080/my-jwt-api/get \ +--header "Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.7u0ls1snw4tPEzd0JTFaf19oXoOvQYtowiHEAZnan74" +``` + +## Use the JWT + +The client includes the JWT in the Authorization header when making requests to the API. + +```bash +curl -X GET \ + https://api.example.com/protected-resource \ + -H 'Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...' +``` + +**Request:** + +| Parameter | Value | +| :--------------- | :----------------------------------------------------- | +| **Method** | `GET` | +| **URL** | The API endpoint for the protected resource. | +| **Authorization** | Bearer token, e.g., `Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...`. | + +## JWT and Auth0 with Tyk + +This will walk you through securing your APIs with JWTs via Auth0. We also have the following video that will walk you through the process. + + + +#### Prerequisites + +* A free account with Auth0 +* A Tyk Self-Managed or Cloud installation + +### Create an Application in Auth0 + +1. Log in to your Auth0 account. +2. Select APIs from the Applications menu. + + Auth0 Create API + +3. Click Create API and enter a name and identifier for your API. + + Auth0 API details + +4. From the Test tab, follow the instructions on how to get an access token. + + Auth0 Test with cURL + +5. From the cURL tab, copy the token request command. + + ```bash expandable + curl --request POST \ + --url https://dev-yjd8e8u5.us.auth0.com/oauth/token \ + --header 'content-type: application/json' \ + --data '{"client_id":{CLIENT_ID},"client_secret":{CLIENT_SECRET},"audience":{AUDIENCE},"grant_type":"client_credentials"}' + ``` + +6. Paste the command in a terminal window to generate your token. Save this token locally. + + ```yaml + { + "access_token": "xxxxxxxxxxx", + "token_type": "Bearer" + } + ``` + +7. After creating your API, a new Auth0 Application will be created. Go to the Applications section to view it. + + New Auth0 Application + +8. Copy the Domain from the Basic Information. You will use this when adding an API to Tyk. + + Auth0 Application Basic Information + + + +### Create your API in Tyk + +1. Log in to your Tyk Dashboard +2. Create a new HTTP API (the default http://httpbin.org upstream URL is fine) + +Tyk Create HTTP API + +1. From the Authentication section, select **JSON Web Token (JWT)** as your authentication mode. +2. Select RSA public Key as the JWT signing method. +3. Enter your Auth0 Application Domain from Step 8 above to complete the `jwks_uri` end point `https://<>/.well-known/jwks.json` +4. Copy your `jwks_uri` in to the **Public Key** field. + +Tyk API Authentication + +1. Add an **Identity Source** and **Policy Field Name**. The defaults of `sub` and `pol` are fine. +2. Save your API. +3. From the System Management section, select Policies +4. Click Add Policy +5. Select your Auth0 API + +Tyk Policy access rights + +1. You can keep the rest of the access rights at the defaults. +2. Click the **Configurations** tab and enter a **Policy Name** and a **Keys Expiry after** period. + +Tyk Policy Configuration + +1. Click **Create Policy**. +2. Edit your JWT Auth0 API and add the policy you created as the **Default Policy** from the Authentication section. + +Tyk API Default Policy Configuration + +1. From the top of the API copy the API URL +2. From a terminal window using the API URL and the Auth0 generated token. + +```.curl +curl -X GET {API URL} -H "Accept: application/json" -H "Authorization: Bearer {token}" +``` +18. If using the [httpbin upstream URL](https://httpbin.org/) as in the example Tyk API, you should see the HTML returned for the httpbin service in your terminal. +19. If there is an error with the request, you will see the following error message. + +```.bash +{ + "error": "Key not authorized:Unexpected signing method." +} +``` expandable + + +## JWT and Keycloak with Tyk + +This guide will walk you through securing your APIs with JWTs via Keycloak. + +#### Prerequisites + +* A Keycloak installation +* A Tyk Self-Managed or Cloud installation + +### Create an Application in Keycloak + +1. Access your Keycloak admin dashboard. +2. Navigate to the Administration console. + + Navigate to Keycloak Administration console + +3. Create a Keycloak realm from the top left-hand side dropdown. + + Create Keycloak Realm + +4. Create a Keycloak client. + + Create Client + +5. Enter the necessary client details. + + Add client details + +6. Enable client authentication and Service account roles under Authentication flow. + + Update client permissions + +7. Set the redirection URL rules. + + Add redirection URL rules + +8. Save. + + Example client + +9. Retrieve the client secret from the Credentials tab under the client you just created. + + Retrieve client secret + +10. Generate your JWT using `curl`. This is the token you will use to access your services through the Tyk Gateway. You can generate your JWT using either of the following methods. Make sure to replace the `KEYCLOAK` prefixed parameters with the appropriate values. + + **Password Grant Type:** + + ```bash + curl -L --insecure -s -X POST 'https://KEYCLOAK_URL/realms/KEYCLOAK_REALM/protocol/openid-connect/token' \ + -H "Content-Type: application/x-www-form-urlencoded" \ + --data-urlencode "client_id=KEYCLOAK_CLIENT_ID" \ + --data-urlencode "grant_type=password" \ + --data-urlencode "client_secret=KEYCLOAK_SECRET" \ + --data-urlencode "scope=openid" \ + --data-urlencode "username=KEYCLOAK_USERNAME" \ + --data-urlencode "password=KEYCLOAK_PASSWORD" + ``` + + **Client Credentials Grant Type:** + + ```bash + curl -L --insecure -s -X POST 'https://KEYCLOAK_URL/realms/KEYCLOAK_REALM/protocol/openid-connect/token' \ + -H "Content-Type: application/x-www-form-urlencoded" \ + --data-urlencode "client_id=KEYCLOAK_CLIENT_ID" \ + --data-urlencode "grant_type=client_credentials" \ + --data-urlencode "client_secret=KEYCLOAK_SECRET" + ``` + + A typical response will look something like this: + + ```yaml + { + "access_token": "...", + "expires_in": 300, + "refresh_expires_in": 1800, + "refresh_token": "...", + "token_type": "Bearer", + "id_token": "...", + "not-before-policy": 0, + "session_state": "...", + "scope": "openid profile email" + } + ``` + +### Running in k8s + +If you are looking to POC this functionality in Kubernetes, you can run a fully worked-out example using our tyk-k8s-demo library. You can read more [here](/deployment-and-operations/tyk-self-managed/tyk-demos-and-pocs/overview#kubernetes-demo). + + +### Create Your JWT API in Tyk + +1. Log in to your Tyk Dashboard. +2. Create a new HTTP API (the default `http://httpbin.org` upstream URL is fine). + + Create a new HTTP API + +3. Scroll to the Authentication mode section and select JWT from the list. +4. Select RSA public Key as JWT Signing method. +5. Add your JSON Web Key Sets (JWKS) URL in the Public Key box. This can be found through the well-known config endpoint or is typically `https://KEYCLOAK_URL/realms/KEYCLOAK_REALM/protocol/openid-connect/certs`. +6. Add an Identity Source and Policy Field Name. The defaults of `sub` and `pol` are fine. +7. Click on the update button to save the API. + + Create API + +8. Create a policy to manage access to your API. +9. Navigate to the Policies section on the left-hand side menu. +10. Click on Add Policy on the top right-hand side of your screen. +11. Select your API from the Add API Access Rights list. + + Select API for Security Policy + +12. Click on the Configurations tab and choose a policy name and TLL. + + Create API Security Policy + +13. Add the default policy to the API. + + Add default policy to API + +14. Test access to the API using curl. +15. Retrieve the API URL. + + Add default Policy to API + +16. Test with curl. Make sure to replace `TOKEN` with the JWT you received from the curl earlier. + + ```bash + curl 'friendly-slipper-gw.aws-use1.cloud-ara.tyk.io/keycloak.jwt/get' \ + -H "Authorization: Bearer TOKEN" + ``` + + + + +## Split Token + +OAuth2, OIDC, and their foundation, JWT, have been industry standards for many years and continue to evolve, particularly with the iterative improvements in the OAuth RFC, aligning with FHIR and Open Banking principles. The OAuth flow remains a dominant approach for secure API access. + +In the OAuth flow, two types of access tokens are commonly used: opaque and JWT (more precisely, JWS). However, the use of JWTs has sparked debates regarding security, as JWTs can leak information when base64 decoded. While some argue that JWTs should not contain sensitive information, others consider JWTs inherently insecure for authorization. + +### Introduction to Split Token Flow + +JWT Access Tokens can carry sensitive information, making them vulnerable if compromised. The Split Token Flow offers a solution by storing only the JWT signature on the client side while keeping the header and payload on the server side. This approach combines the flexibility of JWTs with the security of opaque tokens, ensuring that sensitive data is not exposed. + +### How Tyk Implements Split Token Flow + +Tyk API Gateway is well-positioned to broker the communication between the client and the authorization server. It can handle requests for new access tokens, split the JWT, and return only the signature to the client, storing the rest of the token internally. + +Here’s how you can implement the Split Token Flow using the client credentials flow: + +#### Request a JWT Access Token + +```bash +$ curl -X POST -H "Content-Type: application/x-www-form-urlencoded" \ +https://keycloak-host/auth/realms/tyk/protocol/openid-connect/token \ +-d "grant_type=client_credentials" \ +-d "client_id=efd952c8-df3a-4cf5-98e6-868133839433" \ +-d "client_secret=0ede3532-f042-4120-bece-225e55a4a2d6" -s | jq +``` expandable + +This request returns a JWT access token. + +#### Split the JWT + +The JWT consists of three parts: + +* Header: `eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9` +* Payload: `eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyLCJlbWFpbCI6ImhlbGxvQHdvcmxkLmNvbSJ9` +* Signature: `EwIaRgq4go4R2M2z7AADywZ2ToxG4gDMoG4SQ1X3GJ0` + +Using the Split Token Flow, only the signature is returned to the client, while the header and payload are stored server-side by Tyk. + +Split Token Example + +#### Create a Virtual Endpoint in Tyk + +Create a virtual endpoint or API in Tyk to handle the token request. This endpoint receives the auth request, exchanges credentials with the authorization server, and returns the split token. + +**Example script for the Virtual Endpoint:** + +```javascript +function login(request, session, config) { + var credentials = request.Body.split("&") + .map(function(item, index) { + return item.split("="); + }).reduce(function(p, c) { + p[c[0]] = c[1]; + return p; + }, {}); + + var newRequest = { + "Headers": {"Content-Type": "application/x-www-form-urlencoded"}, + "Method": "POST", + "FormData": { + grant_type: credentials.grant_type, + client_id: credentials.client_id, + client_secret: credentials.client_secret + }, + "Domain": "https://keycloak-host", + "resource": "/auth/realms/tyk/protocol/openid-connect/token", + }; + + var response = TykMakeHttpRequest(JSON.stringify(newRequest)); + var usableResponse = JSON.parse(response); + + if (usableResponse.Code !== 200) { + return TykJsResponse({ + Body: usableResponse.Body, + Code: usableResponse.Code + }, session.meta_data) + } + + var bodyObj = JSON.parse(usableResponse.Body); + var accessTokenComplete = bodyObj.access_token; + var signature = accessTokenComplete.split(".")[2]; + + log("completeAccessToken: " + accessTokenComplete); + + // Create key inside Tyk + createKeyInsideTyk(signature, bodyObj); + + // Override signature + bodyObj.access_token = signature; + delete bodyObj.refresh_expires_in; + delete bodyObj.refresh_token; + delete bodyObj.foo; + + var responseObject = { + Body: JSON.stringify(bodyObj), + Code: usableResponse.Code + } + return TykJsResponse(responseObject, session.meta_data); +} +``` + +This script handles the login process, splits the JWT, and stores the necessary information in Tyk. + +Once the setup is complete, you can test the Split Token Flow by making API calls using the opaque token returned by the virtual endpoint. Tyk will validate the token and reconstruct the full JWT for upstream services. + +```bash +$ curl localhost:8080/basic-protected-api/get -H "Authorization: MEw….GJ0" +``` expandable + +This request uses the opaque token, which Tyk validates and then injects the full JWT into the Authorization header for the API request. + +Split Token Key Metadata + +Split Token API Injection + + + +## Configure your JWT Setup +Learn how to configure and manage JWT authentication in your Tyk API Gateway. + + +### Set Up JWT Signing Method +Select the cryptographic method to verify JWT signatures from the following options: + +- RSA public key +- HMAC shared secret +- ECDSA +- [Public JWKS URL](/basic-config-and-security/security/authentication-authorization/json-web-tokens#enable-dynamic-public-key-rotation-using-jwks) + + + + + : Leave the field blank to configure at the key level. + + + +To generate an RSA keypair, use the following commands: +```bash +openssl genrsa -out key.rsa +openssl rsa -in key.rsa -pubout > key.rsa.pub +``` expandable + +### RSA Supported Algorithms + +Both RSA & PSA classes of RSA algorithms are supported by Tyk, including: +- RS256 +- RS384 +- RS512 +- PS256 +- PS384 +- PS512 + +Read more about the differences between RSA & PSA classes of RSA algorithms [here](https://www.encryptionconsulting.com/overview-of-rsassa-pss/). + +To use either - simply select the "RSA" signing method in the Dashboard, and Tyk will use the appropriate algorithm based on the key you provide. + +### Set Up Individual JWT Secrets +Enable Tyk to validate an inbound token using stored keys: + +1. Set up your token with the following fields: + ```{.json} + "jwt_data": { + "secret": "Secret" + } + ``` +2. Ensure the `kid` header field is included in the JWT for validation. + - If the `kid` header is missing, Tyk will check the `sub` field. This is not recommended but supported. + +The advantage of using RSA is that only the hashed ID and public key of the end user are stored, ensuring high security. + + +### Configure Identity Source and Policy Field Name +Define the identity and policy applied to the JWT: + +- **Identity Source**: Select which identity claim to use (e.g., `sub`) for rate-limiting and quota counting. +- **Policy Field Name**: Add a policy ID claim to the JWT that applies a specific security policy to the session. + + +### Enable Dynamic Public Key Rotation Using JWKs +Instead of a static public key, configure a public JSON Web Key Sets (JWKs) URL to dynamically verify JWT tokens: + +1. Use the JWKs URL to dynamically maintain and rotate active public keys. +2. Ensure JWTs contain the `kid` header, matching the `kid` in the JWK payload for verification. + + +JWKS Public Key Rotation + + +For example, cURLing the JWKs URL returns: + +```{.copyWrapper} +$ curl http://keycloak_host:8081/auth/realms/master/protocol/openid-connect/certs +{ + "keys": [ + { + "kid": "St1x2ip3-wzbrvdk4yVa3-inKWdOwbkD3Nj3gpFJwYM", + "kty": "RSA", + "alg": "RS256", + "use": "sig", + "n": "k-gUvKl9-sS1u8odZ5rZdVCGTe...m2bMmw", + "e": "AQAB", + "x5c": [ + "MIICmzCCAYMCBgFvyVrRq....K9XQYuuWSV5Tqvc7mzPd/7mUIlZQ=" + ], + "x5t": "6vqj9AeFBihIS6LjwZhwFLmgJXM", + "x5t#S256": "0iEMk3Dp0XWDITtA1hd0qsQwgES-BTxrz60Vk5MjGeQ" + } + ] +} +``` + +This is a JWKS complaint payload as it contains the "x5c" entry which contains the public key. Also, the issuer generates the ID Token or Access Token with a header that includes a "kid" that matches the one in the JWKS payload. + +Here's an example of a header belonging to an access token generated by the issuer above. +```{.json} +{ + "alg": "RS256", + "typ": "JWT", + "kid": "St1x2ip3-wzbrvdk4yVa3-inKWdOwbkD3Nj3gpFJwYM" +} +``` expandable + +The auth (bearer) tokens will be signed by the private key of the issuer, which in this example is our keycloak host. This token can be verified by Tyk using the public key available in the above payload under "x5C". + +All of this happens automatically. You just need to specify to Tyk what the JWKs url is, and then apply a "sub" and default policy in order for everything to work. See Step #3, 4, and 5 under option #1 for explanations and examples. + + +### Adjust JWT Clock Skew Configuration +Due to the nature of distributed systems it is expected that despite best efforts you can end up in a situation with clock skew between the issuing party (An OpenID/OAuth provider) and the validating party (Tyk). + +This means that in certain circumstances Tyk would reject requests to an API endpoint secured with JWT with the "Token is not valid yet" error . This occurs due to the clock on the Tyk server being behind the clock on the Identity Provider server even with all servers ntp sync'd from the same ntp server. + +You can configure maximum permissable skew on three claims (`IssueAt`, `ExpireAt` and `NotBefore`) in the API definition. + +### Map JWT Scopes to Policies +Assign JWT scopes to security policies to control access: + +1. Specify scope-to-policy mapping: + +```{.copyWrapper} + "jwt_scope_to_policy_mapping": { + { + "admin": "59672779fa4387000129507d", + "developer": "53222349fa4387004324324e" + }, + "jwt_scope_claim_name": "our_scope" +} +``` + - `"jwt_scope_to_policy_mapping"` provides mapping of scopes (read from claim) to actual policy ID. I.e. in this example we specify that scope "admin" will apply policy `"59672779fa4387000129507d"` to a key +- `"jwt_scope_claim_name"` identifies the JWT claim name which contains scopes. This API Spec field is optional with default value `"scope"`. This claim value could be any of the following: + - a string with space delimited list of values (by standard) + - a slice of strings + - a string with space delimited list of values inside a nested key. In this case, provide `"jwt_scope_claim_name"` in dot notation. For eg. `"scope1.scope2"`, `"scope2"` will be having the list of values nested inside `"scope1"` + - a slice of strings inside a nested key. In this case, provide `"jwt_scope_claim_name"` in dot notation. For eg. `"scope1.scope2"`, `"scope2"` will be having a slice of strings nested inside `"scope1"` + +2. Set the claim name that contains the scopes (default: `scope`): + ```{.json} + "jwt_scope_claim_name": "our_scope" + ``` + + + + + Several scopes in JWT claim will lead to have several policies applied to a key. In this case all policies should have `"per_api"` set to `true` and shouldn't have the same `API ID` in access rights. I.e. if claim with scopes contains value `"admin developer"` then two policies `"59672779fa4387000129507d"` and `"53222349fa4387004324324e"` will be applied to a key (with using our example config above). + + + +## Visualize JWT Flow in Tyk API Gateway +View the diagram below for an overview of JWT flow in Tyk: + +JSON Web Tokens Flow diff --git a/basic-config-and-security/security/authentication-authorization/multiple-auth.mdx b/basic-config-and-security/security/authentication-authorization/multiple-auth.mdx new file mode 100644 index 00000000..6f595e72 --- /dev/null +++ b/basic-config-and-security/security/authentication-authorization/multiple-auth.mdx @@ -0,0 +1,106 @@ +--- +title: "Combine Authentication Methods" +'og:description': "How to combine multiple authentication methods in Tyk to enhance security and flexibility." +sidebarTitle: "Combine Authentication" +tags: ['Authentication', 'Authorization', 'Tyk Authentication', 'Tyk Authorization', 'Multi Authentication', 'Chained Authentication'] +--- + +## Introduction + +Tyk allows you to chain multiple authentication methods together so that each authentication must be successful for access to be granted to the API. For example, you can use an Access Token in combination with Basic Auth or with a JSON Web Token. + +## Base Identity Provider + +When you configure Tyk to use multiple authentication methods, you must declare one to be the **base identity provider**. The [session object](/api-management/policies#what-is-a-session-object) (access key/token) provided in that authentication step will be used by Tyk as the common "request context" and hence the source of truth for authorization (access control, rate limits and quotas). + +You declare the base identity provider using the [server.authentication.baseIdentityProvider](/api-management/gateway-config-tyk-oas#authentication) field in the Tyk Vendor Extension (Tyk Classic: `base_identity_provided_by`). + + +## Enable Multi (Chained) Authentication win the API Designer + +You can configure chained authentication using the Dashboard UI by following these steps: + +1. Enable **Authentication** in the **Servers** section + +2. Select the **Multiple Authentication Mechanisms** option from the drop-down list. + + Select Multiple Auth + +3. Select the **Authentication methods** you want to implement and identify the **Base identity provider** + + Select Auth Methods + +4. You can now configure each of the individual authentication methods in the usual manner using the options in the API designer. + +{/* 18/3/25 removing this video as it's very old (Dashboard 1.9) and perhaps not as helpful as it could be + */} + + +## Configuring multiple auth methods in the API definition + +The OpenAPI description can define multiple `securitySchemes` and then lists those to be used to protect the API in the `security` section. The OpenAPI Specification allows multiple entries in the `security` section of the API description, each of which can contain one or multiple schemes. + +Tyk only takes into consideration the first object in the `security` list. If this contains multiple schemes, then Tyk will implement these sequentially. + +In the following example, the OpenAPI description includes multiple security schemes and then defines three objects in the `security` list: + +```yaml expandable +{ + ... + securitySchemes: { + "auth-A": {...}, + "auth-B": {...}, + "auth-C": {...}, + "auth-D": {...}, + }, + security: [ + { + "auth-A": [], + "auth-C": [] + }, + { + "auth-B": [] + }, + { + "auth-D": [] + } + ] +} +``` + +Tyk will consider only the first entry in the `security` list and so will implement the `auth-A` and `auth-C` schemes. + +In the Tyk Vendor Extension this would result in the following configuration: + +```yaml expandable +x-tyk-api-gateway: + server: + authentication: + enabled: true, + baseIdentityProvider: "auth-A" + securitySchemes: + auth-A: + enabled: true + auth-C: + enabled: true + ... +``` +Note the presence of the `baseIdentityProvider` field which is required. + +## Using Tyk Classic APIs + +To enable this mode, set the `base_identity_provided_by` field in your API Definitions to one of the supported chained enums below: + +* `AuthToken` +* `HMACKey` +* `BasicAuthUser` +* `JWTClaim` +* `OIDCUser` +* `OAuthKey` +* `UnsetAuth` + +The provider set here will then be the one that provides the session object that determines rate limits, ACL rules, and quotas. + +You must also configure the authentication methods to be used in the usual manner, as described in the relevant documentation. To ensure that auth token is implemented as part of the chained authentication, you must set `use_standard_auth` to `true`. + + diff --git a/basic-config-and-security/security/authentication-authorization/open-keyless.mdx b/basic-config-and-security/security/authentication-authorization/open-keyless.mdx new file mode 100644 index 00000000..55669fdd --- /dev/null +++ b/basic-config-and-security/security/authentication-authorization/open-keyless.mdx @@ -0,0 +1,14 @@ +--- +title: "Open (No Authentication)" +'og:description': "How to configure open or keyless authentication in Tyk." +sidebarTitle: "No Authentication" +tags: ['Authentication', 'Authorization', 'Tyk Authentication', 'Tyk Authorization', 'Open Authentication', 'Keyless Authentication'] +--- + +Open or keyless authentication allows access to APIs without any authentication. This method is suitable for public APIs where access control is not required. + +Tyk OAS APIs are inherently "open" unless authentication is configured, however the older Tyk Classic API applies [auth token](/api-management/authentication/bearer-token) protection by default. + +You can disable authentication for a Tyk Classic API by setting the `use_keyless` flag in the API definition. + + diff --git a/basic-config-and-security/security/mutual-tls/client-mtls.mdx b/basic-config-and-security/security/mutual-tls/client-mtls.mdx new file mode 100644 index 00000000..b67b4f4d --- /dev/null +++ b/basic-config-and-security/security/mutual-tls/client-mtls.mdx @@ -0,0 +1,313 @@ +--- +title: "Authentication using Mutual TLS" +'og:description': "How to configure Mutual TLS (mTLS) for client authentication in Tyk." +sidebarTitle: "Mutual TLS" +tags: ['Authentication', 'Authorization', 'Tyk Authentication', 'Tyk Authorization', 'Mutual TLS', 'mTLS', 'Client mTLS'] +--- + +## Introduction + +Mutual TLS (mTLS) is a robust security feature that ensures both the client and server authenticate each other using TLS certificates. This two-way authentication process provides enhanced security for API communications by cryptographically verifying the identity of both parties involved in the connection. + +In most cases when you try to access a secured HTTPS/TLS endpoint, you experience only the client-side check of the server certificate. The purpose of this check is to ensure that no fraud is involved and the data transfer between the client and server is encrypted. In fact, the TLS standard allows specifying the client certificate as well, so the server can accept connections only for clients with certificates registered with the server certificate authority, or provide additional security checks based on the information stored in the client certificate. This is what we call "Mutual TLS" - when both sides of the connection verify certificates. See the video below that gives you an introduction to mutual TLS and how it can be used to secure your APIs. + + + +## Why Use Mutual TLS? + +Mutual TLS is particularly valuable in environments where security is paramount, such as microservices architectures, financial services, healthcare, and any scenario requiring zero-trust security. It not only encrypts the data in transit but also ensures that the communicating parties are who they claim to be, mitigating the risks of unauthorized access and data breaches. + +* **Enhanced Security:** Provides two-way authentication, ensuring both the client and server are verified and trusted. +* **Data Integrity:** Protects the data exchanged between client and server by encrypting it, preventing tampering or interception. +* **Compliance:** Helps meet stringent security and compliance requirements, especially in regulated industries. + +## Client mTLS for Tyk Cloud + +Tyk Cloud users cannot currently use mTLS to secure the client to Gateway communication for Tyk-hosted gateways. + + +Tyk Hybrid users can, however, use mTLS with their self-hosted gateways. + + +## How Does Mutual TLS Work? + +Mutual TLS operates by requiring both the client and server to present and verify TLS certificates during the handshake process. Here’s how it works: + +**Client Authentication:** + +1. When a client attempts to connect to the server, the server requests the client’s TLS certificate. +2. The client provides its certificate, which the server verifies against a trusted Certificate Authority (CA). + +**Server Authentication:** + +1. Simultaneously, the server provides its own certificate to the client, which the client verifies against a trusted CA. + +This mutual verification ensures that both parties are legitimate, securing the connection from both ends. + +### Client authorization with mTLS +At the TLS level, authorization means only allowing access for clients who provide client certificates that are verified and trusted by the server. + +Tyk allows you to define a list of trusted certificates at the API level or Gateway (global) level. If you are updating API definition programmatically or via files, you need to set following the keys in your API +definition: +`use_mutual_tls_auth` to `true`, and `client_certificates` as an array of strings - certificate IDs. + +From the Tyk Dashboard, to do the same from the **API Designer Core settings** section you need to select **Mutual TLS** authentication mode from the **Authentication** section, and allow the certificates using the built-in widget, as below: + +mutual_tls_auth + +If all your APIs have a common set of certificates, you can define them in your Gateway configuration file via the `security.certificates.apis` key - string array of certificate IDs or paths. + +Select **Strip Authorization Data** to strip any authorization data from your API requests. + +Be aware that mutual TLS authorization has special treatment because it is not "authentication" and does not provide any identifying functionality, like keys, so you need to mix it with another authentication modes options like **Auth Key** or **Keyless**. On the dashboard, you need to choose **Use multiple auth mechanism** in the **Authentication mode** drop-down, where you should select **Mutual TLS** and another option which suits your use-case. + +### Fallback to HTTP Authorization +The TLS protocol has no access to the HTTP payload and works on the lower level; thus the only information we have at the TLS handshake level is the domain. In fact, even a domain is not included into a TLS handshake by default, but there is TLS extension called SNI (Server Name Indication) +which allows the client to send the domain name to the TLS handshake level. + +With this in mind, the only way to make API authorization work fully at the TLS level, each API protected by Mutual TLS should be deployed on its own domain. + +However, Tyk will gracefully fallback to a client certificate authorization at the HTTP level in cases when you want to have multiple mutual TLS protected APIs on the same domain, or you have clients that do not support the SNI extension. No additional configuration is needed. In case of such fallback, +instead of getting TLS error, a client will receive 403 HTTP error. + +### Authentication +Tyk can be configured to guess a user authentication key based on the provided client certificate. In other words, a user does not need to provide any key, except the certificate, and Tyk will be able to identify the user, apply policies, and do the monitoring - the same as with regular Keys. + +### Using with Authorization +Mutual TLS authentication does not require mutual TLS authorization to be turned on, and can be used separately. For example, you may allow some of the users to be authenticated by using a token in the header or similar, and some of the users via client certificates. + +If you want to use them both, just configure them separately. No additional knowledge is required. + +### Dynamic vs Static mTLS + +There are two ways to set up client mTLS in Tyk: static and dynamic. Each method is suited to different use cases, as outlined below: + +| Use Case | Static | Dynamic | +| ------------------------------------------------------------------ | :----: | :-----: | +| Let developers upload their own public certificates through the Developer Portal | ❌ | βœ… | +| Combine client mTLS with another authentication method | βœ… | βœ… | +| Allow certs at the API level (one or more APIs per cert) | βœ… | ❌ | +| Allow certs at an individual level (one or more APIs per cert) | ❌ | βœ… | + +## Dynamic mTLS + +Dynamic Client mTLS in Tyk allows you to authenticate users based solely on the provided client certificate, without the need for an additional authentication key. Tyk can identify the user, apply policies, and monitor usage just as with regular API keys. + +To set up Dynamic Client mTLS, we need to follow these steps: +* Protect the API: Configure the API in the API Designer by setting the authentication type to Auth Token and enabling Client Certificate. + +* Generate a Self-Signed Certificate: Use OpenSSL to generate a self-signed certificate and key if you don't have one. + +* Add a Key in the Dashboard: In the Tyk Dashboard, create a key for the API and upload only the public certificate. + +* Make an API Request: Use curl with your certificate and key to make an API request to the protected API, ensuring the request returns a 200 response. + +* Allow Developers to Upload Certificates: Create a policy and catalog entry for the API, allowing developers to request keys and upload their public certificates through the Developer Portal. Developers can then make API requests using their cert and private key. + + +### Protect the API + +In the API Designer, set the Authentication Type to Auth Token under Target Details > Authentication mode. Then select Enable Client Certificate. + +Enable Client Certificate + +### Generate a Self-Signed Key Pair + +If you don’t already have a certificate, generate a self-signed key pair using the following command: + +```bash +openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes +``` + +### Add a Key through the Dashboard + +In the Tyk Dashboard, add a key for the API you set up in step #1. When uploading the certificate, ensure you only upload the public certificate. + + + + +The certificate you upload for this key must only be the public certificate. + + + + +### Make an API Request Using the Certificate + +Now you can make a cURL request to the API using the certificate and private key: + +```bash +curl -k --cert cert.pem --key key.pem https://localhost:8080/mtls-api/my-endpoint +``` + +A successful request should return a 200 response. + +### Allow Developers to Upload Certificates + +Instead of manually creating keys, you can allow developers to upload their own certificates via the Developer Portal. + +1. **Create a Policy:** Create a policy for the API you set up earlier. +2. **Create a Catalog Entry:** Create a catalog entry for this policy. +3. **Request a Key through the Portal:** As a developer, request a key for the API through the Portal. This will present a screen where the developer can upload their public certificate. + +portal_cert_request + +Add your public cert (cert.pem from above) into here and hit "Request Key". + +4. **Make an API Request Using the Uploaded Certificate:** After adding the public certificate, developers can make API requests using their cert + private key: + + ```bash expandable + curl -k --cert cert.pem --key key.pem https://localhost:8080/mtls-api/my-endpoint + ``` + + A successful request should return a 200 response. + +## Static mTLS + +Static mTLS allows client certificates to be used at the API level. This method is straightforward and can be combined with another authentication method if needed. + +### Configure the API + +In the API authentication settings, choose mTLS as the authentication type and optionally select an additional authentication method. If you want to use only client certificates without another authentication method, select "keyless" as the other option. + +### Set the Base Identity + +The base identity can be anything, as the client certificate will be the primary authentication method. + + +### Setup Static mTLS in Tyk Operator using the Tyk Classic API Definition + +This setup requires mutual TLS (mTLS) for client authentication using specified client certificates. The example provided shows how to create an API definition with mTLS authentication for `httpbin-client-mtls`. + +1. **Generate Self-Signed Key Pair:** + +You can generate a self-signed key pair using the following OpenSSL command: + +```bash +openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes +``` + +2. **Create Kubernetes Secret:** + +Create a secret in Kubernetes to store the client certificate: + +```bash +kubectl create secret tls my-test-tls --cert cert.pem --key key.pem +``` + +3. **Create API Definition:** + +Below is the YAML configuration for an API that uses mTLS authentication. Note that the `client_certificate_refs` field references the Kubernetes secret created in the previous step. + +```yaml {hl_lines=["19-21"],linenos=false} +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-client-mtls +spec: + name: Httpbin Client MTLS + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_mutual_tls_auth: true + client_certificate_refs: + - my-test-tls +``` + +### Setup Static mTLS in Tyk Operator using Tyk OAS API Definition + +Client certificates, In Tyk OAS API Definition, are managed using the `TykOasApiDefinition` CRD. You can reference Kubernetes secrets that store client certificates in your API definitions. + +**Example of Referencing Client Certificates in Tyk OAS** + +In this example, the `clientCertificate` section allows you to enable client certificate management and specify a list of Kubernetes secrets (`tls-cert`) that store allowed client certificates. + +```yaml {hl_lines=["48-50"],linenos=false} +# Secret is not created in this manifest. +# Please store client certificate in k8s TLS secret `tls-cert`. + +apiVersion: v1 +data: + test_oas.json: |- + { + "info": { + "title": "Petstore", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "components": {}, + "paths": {}, + "x-tyk-api-gateway": { + "info": { + "name": "Petstore", + "state": { + "active": true + } + }, + "upstream": { + "url": "https://petstore.swagger.io/v2" + }, + "server": { + "listenPath": { + "value": "/petstore/", + "strip": true + } + } + } + } +kind: ConfigMap +metadata: + name: cm + namespace: default +--- +apiVersion: tyk.tyk.io/v1alpha1 +kind: TykOasApiDefinition +metadata: + name: petstore +spec: + tykOAS: + configmapRef: + name: cm + namespace: default + keyName: test_oas.json + clientCertificate: + enabled: true + allowlist: [tls-cert] +``` + + +## FAQ + +* **Why am I getting an error stating that certificates are not enabled for this API?** + + This issue can occur because client mTLS is an extension of Auth Token authentication mode. To enable this feature, ensure the API definition has `auth.use_certificate` set to `true`. + +* **Can I upload a full certificate chain when creating a key for dynamic client mTLS?** + + Yes, you can do this when manually creating a key as an Admin Dashboard user. However, through the Portal, you must upload only the public key (certificate). + +* **Can I use a root CA with client mTLS?** + + Yes, Tyk allows you to upload a root CA certificate for static mTLS authentication. This setup allows clients with certificates signed by the registered CA to be validated. + + **Key Points:** + + * The root CA certificate can be uploaded as a client certificate. + * Clients presenting certificates signed by this CA will be validated. + * Tyk traverses the certificate chain for validation. + + + + Root CA certificates are compatible only with Static mTLS and not with Dynamic mTLS. + + + + diff --git a/branches-config.json b/branches-config.json index 35b9d06a..28cb7b7a 100644 --- a/branches-config.json +++ b/branches-config.json @@ -1,18 +1,28 @@ { "versions": [ { - "branch": "release-5.8", + "branch": "release-5.9", "isLatest": true, - "folder": "5.8", - "label": "v5.8 (Latest)" + "sourceFolder": "5.9-source", + "targetFolder": "5.9", + "label": "v5.9 (latest)" }, { "branch": "main", "isLatest": false, "isMain": true, - "folder": "nightly", + "sourceFolder": "nightly-source", + "targetFolder": "nightly", "label": "Nightly" }, + { + "branch": "release-5.8", + "isLatest": false, + "isMain": false, + "sourceFolder": "5.8-source", + "targetFolder": "5.8", + "label": "v5.8" + }, { "isExternal": true, "externalUrl": "https://tyk.io/docs/5.7", diff --git a/configure/outbound-email-configuration.mdx b/configure/outbound-email-configuration.mdx new file mode 100644 index 00000000..cd571097 --- /dev/null +++ b/configure/outbound-email-configuration.mdx @@ -0,0 +1,155 @@ +--- +title: "Outbound Email Configuration" +order: 6 +noindex: True +sidebarTitle: "Outbound Email Configuration" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + + + +### Custom Email Templates + +The email templates for the Portal and system messages are located in the `portal/email_templates` directory. +The Tyk Dashboard will need to be restarted for changes to take effect. + +### Supported email drivers + +* SMTP +* Mandrill +* Sendgrid +* Mailgun +* AmazonSES + +To get email set up for your installation, add the following to your `tyk_analytics.conf` file: + +```{.copyWrapper} expandable +"email_backend": { + "enable_email_notifications": true, + "code": "{PROVIDER-NAME}", + "settings": { + // Client provider specific settings go here. You can find the specific field described below + }, + "default_from_email": "jeff@wheresmyrug.com", + "default_from_name": "Jeffrey (The Dude) Lebowski" +} +``` + +#### SMTP + +> Available from Tyk Dashboard version 1.7 + +```{.json} expandable +"code": "smtp", +"settings": { + "SMTPUsername": "email@example.com", + "SMTPPassword": "examplepassword", + "SMTPAddress": "smtp.example.com:587", + "TLSInsecureSkipVerify": "false" +}, +``` + +#### SMTP NoAuth + +> Available from Tyk Dashboard version 1.8 + +If `SMTPUsername` or `SMTPPassword` is omitted, Tyk assumes that authentication is not required for your SMTP server. When starting up and initialising the email driver, the Dashboard should output a log message as follows: + +``` +[May 6 13:46:41] INFO email: initializing SMTP email driver +[May 6 13:46:41] INFO email: SMTPUsername and/or SMTPPassword not set - smtp driver configured for no-auth +[May 6 13:46:41] INFO email: SMTP email driver initialized +``` + +#### Mandrill + +```{.json} +"code": "mandrill", +"settings": { + "ClientKey": "xxxxxxxxx" +}, +``` + +#### Sendgrid + +```{.json} +"code": "sendgrid", +"settings": { + "ClientKey": "xxxxxxxxx" +}, +``` + +#### Mailgun + +```{.json} expandable +"code": "mailgun", +"settings": { + "Domain": "KEY", + "PrivateKey": "KEY", + "PublicKey": "KEY" +}, +``` + +#### Amazon SES + +```{.json} expandable +"code": "amazonses", +"settings": { + "Endpoint": "Endpoint", + "AccessKeyId": "Access-key", + "SecretAccessKey": "KEY" +}, +``` +### Customize your Welcome Emails + +You can customize the welcome email that a developer recieves when they signup to your portal. You can use images and other HTML formatted content. The following video walks you through the process. + + + + +1. Select **Settings** from your **Dashboard** > **Portal Management** +2. You can change the from email address and the from email name for your welcome emails. +3. To use customized email content, select **Enable custom welcome email**. +4. You can then add the following custom content: + * Email Subject + * Email Body content + * Welcome email body copy + * Welcome email sign-off + +Welcome-Email + +5. Enter your plain text or HTML formatted content. If including an image, the `LINK TO IMAGE` in an image `` link must be a publicly hosted resource. +6. Click **Save** at the top of the Portal Settings screen. + + +### Customize your Key Approval Emails + +#### Editing the Email Body + +1. Select **Settings** from your **Dashboard** > **Portal Management** +2. From the "API Key approval email" section, select "Enable custom approval email", and edit the API Key email body. + +Email-Customization + +#### Add an image or logo to the Key Approval Email + +1. Select "Enable custom approval email" as above. +2. In the "API Key email body copy" field, enter `` + +Email-Image + + + +The `LINK TO IMAGE` must be a publicly hosted resource. + + + +In an Self-Managed installation you have full access to the HTML template, allowing you further customization. + +#### Portal Manager Email Settings + +Portal-Manager-Email + +1. Select **Settings** from your **Dashboard** > **Portal Management** +2. From the **Portal manager email address** section, enter the email address of the person responsible for approving your developer API subscription requests. See [Portal Key Requests](/tyk-developer-portal/tyk-portal-classic/portal-concepts#key-requests) for more details. diff --git a/dashboard-admin-api.mdx b/dashboard-admin-api.mdx new file mode 100644 index 00000000..083524ff --- /dev/null +++ b/dashboard-admin-api.mdx @@ -0,0 +1,6 @@ +--- +title: "Tyk Dashboard Admin API" +order: 4 +sidebarTitle: "Dashboard Admin" +--- + diff --git a/deployment-and-operations/tyk-open-source-api-gateway/quick-start.mdx b/deployment-and-operations/tyk-open-source-api-gateway/quick-start.mdx new file mode 100644 index 00000000..1809b74e --- /dev/null +++ b/deployment-and-operations/tyk-open-source-api-gateway/quick-start.mdx @@ -0,0 +1,50 @@ +--- +title: "Quick Start Tyk Gateway" +'og:description': "This page serves as a comprehensive guide to migrating workloads to Tyk Open Source" +sidebarTitle: "Quick Start" +tags: ['installation', 'migration', 'open source'] +--- + +## Introduction + +New to Tyk Gateway? On this page you'll get started with the basics - install Tyk and test it live in less than 2 minutes. + +We recommend [Tyk Gateway docker compose](https://github.com/TykTechnologies/tyk-gateway-docker) as the quickest way to get started. If you want to deploy it in a specific platform check our [installation options](/apim/open-source/installation) page. + +**Step 1 - Clone the docker-compose repository** +``` +git clone https://github.com/TykTechnologies/tyk-gateway-docker +``` + +**Step 2 - Change to the new directory** +``` +cd tyk-gateway-docker +``` + +**Step 3 - Deploy Tyk Gateway and Redis** +``` +docker-compose up +``` + +You can also run this in detached mode using the _-d_ flag: + +``` +docker-compose up -d +``` + +Congratulations, you’re done!!! + +## Test Installation + +Your Tyk Gateway is now configured and ready to use. Confirm this by checking against the β€˜hello’ endpoint: + +```curl +curl localhost:8080/hello +``` + +The output should be similar to that shown below: +```json +{"status": "pass", "version": "v5.1", "description": "Tyk GW"} +``` + + diff --git a/deployment-and-operations/tyk-self-managed/tyk-demos-and-pocs/overview.mdx b/deployment-and-operations/tyk-self-managed/tyk-demos-and-pocs/overview.mdx new file mode 100644 index 00000000..0c7eac54 --- /dev/null +++ b/deployment-and-operations/tyk-self-managed/tyk-demos-and-pocs/overview.mdx @@ -0,0 +1,740 @@ +--- +title: "Explore Demos and Proof of Concepts" +'og:description': "Explore Tyk demos and proof of concepts to quickly set up Tyk Self-Managed, including the Tyk Gateway, Dashboard, and analytics processing pipeline." +sidebarTitle: "Demos" +--- + +import { ButtonLeft } from '/snippets/ButtonLeft.mdx'; + +## Kubernetes Demo + +The [tyk-k8s-demo](https://github.com/TykTechnologies/tyk-k8s-demo) repository allows you to start up an entire Tyk Stack +with all its dependencies as well as other tools that can integrate with Tyk. +The repository will spin up everything in Kubernetes using `helm` and bash magic +to get you started. + +**Purpose** +Minimize the amount of effort needed to start up the Tyk infrastructure and +show examples of how Tyk can be set up in k8s using different deployment +architectures as well as different integrations. + +### Prerequisites + +#### Required Packages + +You will need the following tools to be able to run this project. +- [Kubectl](https://kubernetes.io/docs/tasks/tools/) - CLI tool for controlling Kubernetes clusters +- [Helm](https://helm.sh/docs/intro/install/) - Helps manage Kubernetes applications through Helm charts +- [jq](https://stedolan.github.io/jq/download/) - CLI for working with JSON output and manipulating it +- [git](https://git-scm.com/downloads) - CLI used to obtain the project from GitHub +- [Terraform](https://www.terraform.io/) (only when using `--cloud` flag) + +Tested on Linux/Unix based systems on AMD64 and ARM architectures + +#### License Requirements +- **Tyk OSS**: No license required as it is open-source. + +- **Licensed Products**: Sign up [here](https://tyk.io/sign-up) using the button below, and choose "Get in touch" to receive a guided evaluation of the Tyk Dashboard and your temporary license. + + + +**How to use the license key** + +Once you obtained the license key, create a `.env` file using the [example provided](https://github.com/TykTechnologies/tyk-k8s-demo/blob/main/.env.example) and update it with your licenses as follows: + +```bash +git clone https://github.com/TykTechnologies/tyk-k8s-demo.git +cd tyk-k8s-demo +cp .env.example .env +``` + +Depending on the deployments you would like to install set values of the `LICENSE`, `MDCB_LICENSE`, and `PORTAL_LICENSE` +inside the `.env` file. + +#### Minikube +If you are deploying this demo on [Minikube](https://minikube.sigs.k8s.io/docs/start), +you will need to enable the [ingress addon](https://kubernetes.io/docs/tasks/access-application-cluster/ingress-minikube/#enable-the-ingress-controller). +You can do so by running the following commands: + +```bash +minikube start +minikube addons enable ingress +``` + +### Quick Start +```bash +./up.sh --deployments portal,operator-httpbin tyk-stack +``` +This quick start command will start up the entire Tyk stack along with the +Tyk Enterprise Portal, Tyk Operator, and httpbin CRD example. + +### Possible deployments +- `tyk-stack`: A comprehensive Tyk Self Managed setup for a single region +- `tyk-cp`: Tyk control plane in a multi-region Tyk deployment +- `tyk-dp`: Data plane of hybrid gateways that connect to either Tyk Cloud or a Tyk Control Plane, facilitating scalable deployments +- `tyk-gateway`: Open Source Software (OSS) version of Tyk, self-managed and suitable for single-region deployments + + +### Dependencies Options + +**Redis Options** +- `redis`: Bitnami Redis deployment +- `redis-cluster`: Bitnami Redis Cluster deployment +- `redis-sentinel`: Bitnami Redis Sentinel deployment + +**Storage Options** +- `mongo`: [Bitnami Mongo](https://artifacthub.io/packages/helm/bitnami/mongodb) database deployment as a Tyk backend +- `postgres`: [Bitnami Postgres](https://artifacthub.io/packages/helm/bitnami/postgresql) database deployment as a Tyk backend + +**Supplementary Deployments** +Please see this [page](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/docs/FEATURES_MATRIX.md) for Tyk deployments compatibility charts. +- [cert-manager](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/cert-manager): deploys cert-manager. +- [datadog](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/datadog): deploys Datadog agent +and starts up Tyk Pump to push analytics data from the Tyk platform to Datadog. It will also create a Datadog dashboard +for you to view the analytics. +- [elasticsearch](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/elasticsearch): deploys +Elasticsearch and starts up Tyk pump to push analytics data from the Tyk platform to Elasticsearch. + - [elasticsearch-kibana](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/elasticsearch-kibana): deploys the Elasticsearch deployment as well as a Kibana deployment and creates a Kibana dashboard for you to view the analytics. +- [Jaeger](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/jaeger): deploys the Jaeger operator, a Jaeger instance, and the OpenTelemetry collector and configures the Tyk deployment to send telemetry data to Jaeger through the OpenTelemetry collector. +- [k6](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/k6): deploys a Grafana K6 Operator. + - [k6-slo-traffic](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/k6-slo-traffic): deploys a k6 CRD to generate a load of traffic to seed analytics data. +- [keycloak](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/keycloak): deploys the Keycloak Operator and a Keycloak instance. + - [keycloak-dcr](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/keycloak-dcr): starts up a Keycloak Dynamic Client Registration example. + - [keycloak-jwt](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/keycloak-jwt): starts up a Keycloak JWT Authentication example with Tyk. + - [keycloak-sso](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/keycloak-sso): starts up a Keycloak SSO example with the Tyk Dashboard. +- [newrelic](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/newrelic): deploys New Relic and starts up a Tyk Pump to push analytics data from the Tyk platform to New Relic. +- [opa](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/opa): enables Open Policy Agent to allow for Dashboard APIs governance. +- [opensearch](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/opensearch): deploys OpenSearch and starts up Tyk Pump to push analytics data from the Tyk platform to OpenSearch. +- [operator](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/operator): deploys the [Tyk Operator](https://github.com/TykTechnologies/tyk-operator) and its dependency [cert-manager](https://github.com/jetstack/cert-manager). + - [operator-federation](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/operator-federation): starts up Federation v1 API examples using the tyk-operator. + - [operator-graphql](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/operator-graphql): starts up GraphQL API examples using the tyk-operator. + - [operator-httpbin](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/operator-httpbin): starts up an API examples using the tyk-operator. + - [operator-jwt-hmac](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/operator-jwt-hmac): starts up API examples using the tyk-operator to demonstrate JWT HMAC auth. + - [operator-udg](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/operator-udg): starts up Universal Data Graph API examples using the tyk-operator. +- [portal](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/portal): deploys the [Tyk Enterprise Developer Portal](/portal/overview/intro) as well as its dependency PostgreSQL. +- [prometheus](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/prometheus): deploys Prometheus and starts up Tyk Pump to push analytics data from the Tyk platform to Prometheus. + - [prometheus-grafana](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/prometheus-grafana): deploys the Prometheus deployment as well as a Grafana deployment and creates a Grafana dashboard for you to view the analytics. +- [vault](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/deployments/vault): deploys Vault Operator and a Vault instance. + +If you are running a POC and would like an example of how to integrate a +specific tool, you are welcome to submit a [feature request](https://github.com/TykTechnologies/tyk-k8s-demo/issues/new/choose) + +**Example** +```bash +./up.sh \ + --storage postgres \ + --deployments prometheus-grafana,k6-slo-traffic \ + tyk-stack +``` + +The deployment process takes approximately 10 minutes, as the installation is sequential and some dependencies take +time to initialize. Once the installation is complete, the script will output a list of all the services that were +started, along with instructions on how to access them. Afterward, the k6 job will begin running in the background, +generating traffic for 15 minutes. To monitor live traffic, you can use the credentials provided by the script to +access Grafana or the Tyk Dashboard + +### Bash Script Usage +
+ +#### Start Tyk deployment +Create and start up the deployments + +```bash expandable +Usage: + ./up.sh [flags] [command] + +Available Commands: + tyk-stack + tyk-cp + tyk-dp + tyk-gateway + +Flags: + -v, --verbose bool set log level to debug + --dry-run bool set the execution mode to dry run. This will dump the kubectl and helm commands rather than execute them + -n, --namespace string namespace the tyk stack will be installed in, defaults to 'tyk' + -f, --flavor enum k8s environment flavor. This option can be set 'openshift' and defaults to 'vanilla' + -e, --expose enum set this option to 'port-forward' to expose the services as port-forwards or to 'load-balancer' to expose the services as load balancers or 'ingress' which exposes services as a k8s ingress object + -r, --redis enum the redis mode that tyk stack will use. This option can be set 'redis', 'redis-sentinel' and defaults to 'redis-cluster' + -s, --storage enum database the tyk stack will use. This option can be set 'mongo' (amd only) and defaults to 'postgres' + -d, --deployments string comma separated list of deployments to launch + -c, --cloud enum stand up k8s infrastructure in 'aws', 'gcp' or 'azure'. This will require Terraform and the CLIs associate with the cloud of choice + -l, --ssl bool enable ssl on deployments +``` + +#### Stop Tyk deployment +Shutdown deployment + +```bash expandable +Usage: + ./down.sh [flags] + +Flags: + -v, --verbose bool set log level to debug + -n, --namespace string namespace the tyk stack will be installed in, defaults to 'tyk' + -p, --ports bool disconnect port connections only + -c, --cloud enum tear down k8s cluster stood up +``` + +**Clusters** +You can get the repository to create demo clusters for you on AWS, GCP, or Azure. That can be set using the `--cloud` flag +and requires the respective cloud CLI to be installed and authorized on your system. You will also need to specify the +`CLUSTER_LOCATION`, `CLUSTER_MACHINE_TYPE`, `CLUSTER_NODE_COUNT`, and `GCP_PROJECT` (for GCP only) parameters in the .env file. + +You can find examples of .env files here: +- [AWS](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/clouds/aws/.env.example) +- [GCP](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/clouds/gcp/.env.example) +- [Azure](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/src/clouds/azure/.env.example) + +For more information about cloud CLIs: +- AWS: + - [aws](https://aws.amazon.com/cli/) +- GCP: + - [gcloud](https://cloud.google.com/sdk/gcloud) + - `GOOGLE_APPLICATION_CREDENTIALS` environment variable per [documentation from Google](https://cloud.google.com/docs/authentication/application-default-credentials) +- Azure: + - [az](https://learn.microsoft.com/en-us/cli/azure/install-azure-cli) + +### Customization +This repository can also act as a guide to help you get set up with Tyk. If you just want to know how to set up a specific +tool with Tyk, you can run the repository with the `--dry-run` and `--verbose` flags. This will output all the commands that +the repository will run to stand up any installation. This can help debug as well as figure out what +configuration options are required to set these tools up. + +Furthermore, you can also add any Tyk environment variables to your `.env` file and those variables will be mapped to +their respective Tyk deployments. + +Example: +```env +... +TYK_MDCB_SYNCWORKER_ENABLED=true +TYK_MDCB_SYNCWORKER_HASHKEYS=true +TYK_GW_SLAVEOPTIONS_SYNCHRONISERENABLED=true +``` + +### Variables +The script has defaults for minimal settings in [this env file](https://github.com/TykTechnologies/tyk-k8s-demo/tree/main/.env.example), +and it will give errors if something is missing. +You can also add or change any Tyk environment variables in the `.env` file, +and they will be mapped to the respective `extraEnvs` section in the helm charts. + +| Variable | Default | Comments | +|--------------------------------------|:---------------------:|-----------------------------------------------------------------------------------------------------------------| +| DASHBOARD_VERSION | `v5.5` | Dashboard version | +| GATEWAY_VERSION | `v5.5` | Gateway version | +| MDCB_VERSION | `v2.7` | MDCB version | +| PUMP_VERSION | `v1.11` | Pump version | +| PORTAL_VERSION | `v1.10` | Portal version | +| TYK_HELM_CHART_PATH | `tyk-helm` | Path to charts, can be a local directory or a helm repo | +| TYK_USERNAME | `default@example.com` | Default password for all the services deployed | +| TYK_PASSWORD | `topsecretpassword` | Default password for all the services deployed | +| LICENSE | | Dashboard license | +| MDCB_LICENSE | | MDCB license | +| PORTAL_LICENSE | | Portal license | +| TYK_WORKER_CONNECTIONSTRING | | MDCB URL for worker connection | +| TYK_WORKER_ORGID | | Org ID of dashboard user | +| TYK_WORKER_AUTHTOKEN | | Auth token of dashboard user | +| TYK_WORKER_USESSL | `true` | Set to `true` when the MDCB is serving on a TLS connection | +| TYK_WORKER_SHARDING_ENABLED | `false` | Set to `true` to enable API Sharding | +| TYK_WORKER_SHARDING_TAGS | | API Gateway segmentation tags | +| TYK_WORKER_GW_PORT | `8081` | Set the gateway service port to use | +| TYK_WORKER_OPERATOR_CONNECTIONSTRING | | Set the dashboard URL for the operator to be able to manage APIs and Policies | +| DATADOG_APIKEY | | Datadog API key | +| DATADOG_APPKEY | | Datadog Application key. This is used to create a dashboard and create a pipeline for the Tyk logs | +| DATADOG_SITE | `datadoghq.com` | Datadog site. Change to `datadoghq.eu` if using the European site | +| GCP_PROJECT | | The GCP project for terraform authentication on GCP | +| CLUSTER_LOCATION | | Cluster location that will be created on AKS, EKS, or GKE | +| CLUSTER_MACHINE_TYPE | | Machine type for the cluster that will be created on AKS, EKS, or GKE | +| CLUSTER_NODE_COUNT | | Number of nodes for the cluster that will be created on AKS, EKS, or GKE | +| INGRESS_CLASSNAME | `nginx` | The ingress classname to be used to associate the k8s ingress objects with the ingress controller/load balancer | + + +## Docker Demo + +### Purpose + +With *tyk-demo* repository, using docker-compose, you can set up quickly a **complete** Tyk stack, including +dependencies and integrations. + +Minimize the amount of effort needed to start up the Tyk infrastructure and show end-to-end complete examples of how to set up various capabilities in Tyk as well as different integrations. + +### Key Features + +- Full Tyk stack deployment +- Pre-configured demo APIs +- Analytics and monitoring tools +- Integration with common third-party services + +Watch the video *What Is Tyk Demo* for an overview and learn about the key features from our experts - + + + +### Prerequisites + +1. Docker compose +Make sure you have [docker compose](https://docs.docker.com/compose/install/) and that docker is running on your machine. + +2. License key +This Demo deploys and runs the full Tyk platform which is a licensed product. Please sign up using the button below, to obtain a license key. In the link, choose "Get in touch" to get a guided evaluation of the Tyk Dashboard and receive your temporary license. + + + +### Quick Start + +The following steps will enable you to quickly get started: + +1. **Clone the repository**: +```bash +git clone https://github.com/TykTechnologies/tyk-demo.git +``` + +2. **Navigate to the directory**: +```bash +cd tyk-demo +``` + +3. **Add license key to .env file**: +```bash +DASHBOARD_LICENCE= +``` + +4. **Run the setup script**: +```bash +./up.sh +``` + +5. **Access Tyk Dashboard**: [http://localhost:3000](http://localhost:3000) + +To complete the instruction above we have a tutorial video of tyk demo that covers: +- Downloading and starting tyk-demo +- Setting up your license +- Logging in to Tyk Dashboard + + + + +## Docker Compose Setup + +### Who is this page for? +This is the guide we recommend for a easy quick start. The instructions are the ones shared with you when you register to a [free trial](https://tyk.io/sign-up/). + +You can also use this guide for your PoC since it spins up a full Tyk Self Managed stack for you using our project *Docker Pro Demo*, however, if you are interested in learning Tyk, there's an option for [Tyk Demo](/deployment-and-operations/tyk-self-managed/tyk-demos-and-pocs/overview) which is a project that spins up full Tyk stack that includes a prepopulate API definitions of all kinds, with various middleware options and can also spin up supporting tools such as Prometheus, Keycloak (IDP) etc. + +### What's included? +The *Tyk Pro Docker Demo* is our [Self-Managed](/tyk-self-managed) solution, which includes our Gateway, Dashboard, and analytics processing pipeline. This demo will run Tyk Self-Managed on your machine, which contains 5 containers: Tyk Gateway, Tyk Dashboard, Tyk Pump, Redis and MongoDB. This demo is great for proof of concept and demo purposes, but if you want to test performance, you will need to move each component to a separate machine. + + + +This demo is NOT intended for production use or performance testing, since it uses docker compose and the configuration files are not specifically tuned for performance testing or high loads. Please visit the [Planning for Production](/planning-for-production) page to learn how to configure settings for optimal performance. + + + + +The Tyk Pro Docker demo does not provide access to the [Developer Portal](/portal/overview/intro). + + + +### Prerequisites + +* Our [Tyk Pro Docker demo on GitHub](https://github.com/TykTechnologies/tyk-pro-docker-demo) +* A Tyk Pro [trial license](https://tyk.io/sign-up/) + +### Steps for Installation + +1. **Clone the GitHub repo** + +Clone the Docker demo repo from GitHub to a location on your machine. + +2. **Edit your hosts file** + +You need to add the following to your hosts file: + +```bash +127.0.0.1 www.tyk-portal-test.com +127.0.0.1 www.tyk-test.com +``` + +3. **Add your developer license** + +From your installation folder: + +Create an `.env` file - `cp .env.example .env.` Then add your license string to `TYK_DB_LICENSEKEY`. + +4. **Initialise the Docker containers** + +**With MongoDB** + +Run the following command from your installation folder: + +```docker +docker-compose up +``` + +**With PostgreSQL** + +Run the following command from your installation folder: + +```docker +docker-compose -f ./docker-compose.yml -f ./docker-compose.postgres.yml up +``` + +This will will download and setup the five Docker containers. This may take some time and will run in non-daemonised mode so you can see all the output. + +5. **Bootstrap the Tyk installation** + +Go to [http://localhost:3000](http://localhost:3000) in your browser. You will be presented with the Bootstrap UI to create your first organization and admin user. + +Tyk Bootstrap sceen + + +6. **Create your organization and default user** + +You need to enter the following: + +* Your **Organization Name** +* Your **Organization Slug** +* Your User **Email Address** +* Your User **First and Last Name** +* A **Password** for your User +* **Re-enter** your user **Password** + + + + + For a password, we recommend a combination of alphanumeric characters, with both upper and lower case + letters. + + + + +Click **Bootstrap** to save the details. + +7. **log in to the Tyk Dashboard** + +You can now log in to the Tyk Dashboard from `127.0.0.1:3000`, using the username and password created in the Dashboard +Setup screen. + +### Removing the demo installation + +To delete all containers as well as remove all volumes from your host: + +**With MongoDB** + +```bash +docker-compose down -v +``` +**With PostgreSQL:** + +```bash +docker-compose -f ./docker-compose.yml -f ./docker-compose.postgres.yml down -v +``` + +## Using Windows + +### Tyk Pro on Windows using Docker Desktop + +The Tyk Pro Docker demo is our full [On-Premises](https://tyk.io/api-gateway/on-premise/) Pro solution, which includes our Gateway, Dashboard, and analytics processing pipeline. This demo will run Tyk Self-Managed Pro on your machine, which contains 5 containers: Tyk Gateway, Tyk Dashboard, Tyk Pump, Redis and MongoDB. This demo is great for proof of concept and demo purposes, but if you want to test performance, you will need to move each component to a separate machine. + + + +This demo is NOT designed for production use or performance testing. + + + + + +You use this at your own risk. Tyk is not supported on the Windows platform. However you can test it as a proof of concept using our Pro Demo Docker installation. + + + +**Prerequisites** + +- MS Windows 10 Pro +- [Docker Desktop for Windows](https://docs.docker.com/docker-for-windows/install/) running with a signed in [Docker ID](https://docs.docker.com/docker-id/) +- Git for Windows +- PowerShell running as administrator +- Postman for [Windows](https://www.getpostman.com/downloads/) +- Our Pro Demo Docker [GitHub repo](https://github.com/TykTechnologies/tyk-pro-docker-demo) +- A free Tyk Self-Managed [Developer license](https://tyk.io/sign-up/) + +**Step 1 - Clone the Repo** + +Clone the repo above to a location on your machine. + +**Step 2 - Edit your hosts file** + +You need to add the following to your Windows hosts file: + +```bash +127.0.0.1 www.tyk-portal-test.com +127.0.0.1 www.tyk-test.com +``` + +**Step 3 - Add your Developer License** + +You should have received your free developer license via email. Copy the license key in the following location from your `\confs\tyk_analytics.conf` file: + +```yaml +{ + ... + "license_key": "" + ... +} +``` + +**Step 4 - Run the Docker Compose File** + +From PowerShell, run the following command from your installation folder: + +```bash +docker-compose up +``` + +This will will download and setup the five Docker containers. This may take some time and will display all output. + +**Step 5 - Test the Tyk Dashboard URL** + +Go to: + +```bash +127.0.0.1:3000 +``` + +You should get to the Tyk Dashboard Setup screen: + +Tyk Dashboard Bootstrap Screen + +**Step 6 - Create your Organization and Default User** + +You need to enter the following: + +- Your **Organization Name** +- Your **Organization Slug** +- Your User **Email Address** +- Your User **First and Last Name** +- A **Password** for your User +- **Re-enter** your user **Password** + + + + + For a password, we recommend a combination of alphanumeric characters, with both upper and lower case + letters. + + + +Click **Bootstrap** to save the details. + +You can now log in to the Tyk Dashboard from `127.0.0.1:3000`, using the username and password created in the Dashboard Setup screen. + +**Step 7 - Set up a Portal Catalog** + +This creates a portal catalog for your developer portal. For the `Authorization` Header, the Value you need to enter is the `access_key` value from the create user request. In the body add the `org_id` value created in **Step One**. + +- **Request**: POST +- **URL**: `127.0.0.1:3000/api/portal/catalogue` +- **Header**: Key `Authorzation` Value `SECRET_VALUE` +- **Body** (raw set to application/json): + +*Sample Request* + +```json +{ "org_id": "5d07b4b0661ea80001b3d40d" } +``` + +*Sample Response* + +```json +{ + "Status": "OK", + "Message": "5d07b4b0661ea80001b3d40d", + "Meta": null +} +``` + +**Step 8 - Create your default Portal Pages** + +This creates the default home page for your developer portal. For the `Authorization` Header, the Value you need to enter is the `access_key` value from the create user request. + +- **Request**: POST +- **URL**: `127.0.0.1:3000/api/portal/catalogue` +- **Header**: Key `Authorzation` Value `SECRET_VALUE` +- **Body** (raw set to application/json): + +*Sample Request* + +```json expandable +{ + "fields": { + "JumboCTALink": "#cta", + "JumboCTALinkTitle": "Your awesome APIs, hosted with Tyk!", + "JumboCTATitle": "Tyk Developer Portal", + "PanelOneContent": "Panel 1 content.", + "PanelOneLink": "#panel1", + "PanelOneLinkTitle": "Panel 1 Button", + "PanelOneTitle": "Panel 1 Title", + "PanelThereeContent": "", + "PanelThreeContent": "Panel 3 content.", + "PanelThreeLink": "#panel3", + "PanelThreeLinkTitle": "Panel 3 Button", + "PanelThreeTitle": "Panel 3 Title", + "PanelTwoContent": "Panel 2 content.", + "PanelTwoLink": "#panel2", + "PanelTwoLinkTitle": "Panel 2 Button", + "PanelTwoTitle": "Panel 2 Title", + "SubHeading": "Sub Header" + }, + "is_homepage": true, + "slug": "home", + "template_name": "", + "title": "Tyk Developer Portal" +} +``` + +*Sample Response* + +```json +{ + "Status": "OK", + "Message": "5d07b4b0661ea80001b3d40d", + "Meta": null +} +``` + +**Step 9 - Setup the Portal URL** + +This creates the developer portal URL. For the `Authorization` Header, the Value you need to enter is the `secret` value from your `/confs/tyk_analytics.conf`. + +- **Request**: POST +- **URL**: `127.0.0.1:3000/api/portal/configuration` +- **Header**: Key `Authorzation` Value `SECRET_VALUE` +- **Body** (raw set to application/json): + +*Sample Request* + +```yaml +{SECRET_VALUE} +``` + +*Sample Response* + +```json +{ + "Status": "OK", + "Message": "5d07b4b0661ea80001b3d40d", + "Meta": null +} +``` + +### Tyk Pro on Windows using WSL + +The Tyk Pro Docker demo is our full [Self-Managed](#docker-demo) solution, which includes our Gateway, Dashboard, and analytics processing pipeline. This demo will run Tyk Self-Managed on your machine, which contains 5 containers: Tyk Gateway, Tyk Dashboard, Tyk Pump, Redis and MongoDB. This demo is great for proof of concept and demo purposes, but if you want to test performance, you will need to move each component to a separate machine. + + + +This demo is NOT designed for production use or performance testing. + + + + + +You use this at your own risk. Tyk is not supported on the Windows platform. However you can test it as a proof of concept using our Pro Demo Docker installation. + + + + +**Prerequisites** + +- MS Windows 10 Pro with [Windows Linux Subsystem](https://docs.microsoft.com/en-us/windows/wsl/install-win10) enabled +- [Docker Desktop for Windows](https://docs.docker.com/docker-for-windows/install/) running with a signed in [Docker ID](https://docs.docker.com/docker-id/) +- Git for Windows +- PowerShell running as administrator +- Postman for [Windows](https://www.getpostman.com/downloads/) +- Our Pro Demo Docker [GitHub repo](https://github.com/TykTechnologies/tyk-pro-docker-demo) +- A free Tyk Self-Managed [Developer license](https://tyk.io/sign-up/) +- Optional: Ubuntu on Windows + +**Step 1 - Clone the Repo** + +Clone the repo above to a location on your machine. + +**Step 2 - Edit your hosts file** + +You need to add the following to your Windows hosts file: + +```bash +127.0.0.1 www.tyk-portal-test.com +127.0.0.1 www.tyk-test.com +``` + +**Step 3 - Configure file permissions** +In order to mount the files, you need to allow Docker engine has access to your Drive. +You can do that by going to the Docker settings, Shared Drives view, and manage the access. +If after all you will get issue regarding path permissions, you will need to create a separate user specifically for the docker according to this instructions https://github.com/docker/for-win/issues/3385#issuecomment-571267988 + + +**Step 4 - Add your Developer License** + +You should have received your free developer license via email. Copy the license key in the following location from your `\confs\tyk_analytics.conf` file: + +``` +"license_key": "" +``` + +**Step 5 - Run the Docker Compose File** + +From PowerShell, run the following command from your installation folder: + +```console +docker-compose up +``` + +This will will download and setup the five Docker containers. This may take some time and will display all output. + +**NOTE** +If you are getting issues related to errors when mounting files, you may need to modify +`docker-compose.yml` file, and change configs paths from related to absolute, and from linux format to windows format, like this: +``` +volumes: + - C:\Tyk\confs\tyk_analytics.conf:/opt/tyk-dashboard/tyk_analytics.conf +``` + +**Step 6 - Got to the Dashboard URL** + +Go to: + +```bash +127.0.0.1:3000 +``` + +You should get to the Tyk Dashboard Setup screen: + +Tyk Dashboard Bootstrap Screen + +**Step 7 - Create your Organization and Default User** + +You need to enter the following: + +- Your **Organization Name** +- Your **Organization Slug** +- Your User **Email Address** +- Your User **First and Last Name** +- A **Password** for your User +- **Re-enter** your user **Password** + + + + + For a password, we recommend a combination of alphanumeric characters, with both upper and lower case + letters. + + + +Click **Bootstrap** to save the details. + +You can now log in to the Tyk Dashboard from `127.0.0.1:3000`, using the username and password created in the Dashboard Setup screen. + +**Configure your Developer Portal** + +To set up your [Developer Portal](/tyk-developer-portal) follow our Self-Managed [tutorial on publishing an API to the Portal Catalog](/getting-started/tutorials/publish-api). + diff --git a/developer-support/community.mdx b/developer-support/community.mdx new file mode 100644 index 00000000..e4db2ee0 --- /dev/null +++ b/developer-support/community.mdx @@ -0,0 +1,39 @@ +--- +title: "Community Support" +'og:description': "Developer support mechanisms, release notes and upgrading information for your Tyk installation" +tags: ['FAQ', 'Troubleshooting', 'Release Notes', 'Upgrading', 'Developer Support'] +sidebarTitle: "Community Support" +--- + +Tyk offers a range of developer support services to assist with troubleshooting and Frequently Asked Questions (FAQs). + +## Expert Advice + +Prospective customers seeking technical advice can [contact us](https://tyk.io/contact/) to arrange a consultation with +one of our expert engineers. + +## Community support + +Our [community](https://community.tyk.io/) is available to everyone, offering the following channels: + +| Channel | Purpose | +| :--------- | :--------- | +| [Support](https://community.tyk.io/c/support/) | Raise bugs and troubleshooting questions | +| [Products](https://community.tyk.io/c/product/) | Discuss specific features of the Tyk Product Stack | +| [Community](https://community.tyk.io/c/community/) | Access learning resources and community events | +| [Announcements](https://community.tyk.io/c/announcements/) | Stay updated on new release announcements | +| [Meta Feedback](https://community.tyk.io/c/meta) | Submit bug reports and forum feedback | + +You can also submit bug reports and feature requests through the +[Tyk OSS Gateway repository](https://github.com/TykTechnologies/tyk). + +## Licensed + +Tyk Self-Managed and Tyk Cloud license holders have access to 24/7 support through our +[helpdesk portal](https://support.tyk.io/hc/en-gb). + + +## Additional Resources + +- [Release Notes](/developer-support/release-notes/gateway) +- [Upgrading Tyk](/developer-support/upgrading) diff --git a/developer-support/contributing.mdx b/developer-support/contributing.mdx new file mode 100644 index 00000000..dca69e35 --- /dev/null +++ b/developer-support/contributing.mdx @@ -0,0 +1,38 @@ +--- +title: "Contributing to Tyk documentation" +'og:description': "How to contribute to Tyk documentation" +sidebarTitle: "Overview" +tags: ['Contributing', 'Documentation', 'Community'] +--- + +This section provides resources and guidance for contributors to Tyk's documentation. Whether you're fixing a typo, adding new content, or helping improve our documentation standards, you'll find the tools and guidelines you need here. + +## What this section covers + +This contribution section includes: + +- **[Inclusive naming](/developer-support/documentation-projects/inclusive-naming)** - Information about Tyk's inclusive language initiative and the work done to update documentation terminology +- **UI component examples** - Documentation and examples for using shortcodes in our documentation system: + - **[Pill label shortcode](/ui-examples/test-pill-label)** - For labeling content with edition indicators (Enterprise, Cloud, Lab Release, etc.) + - **[Feature cards shortcode](/ui-examples/feature-cards)** - For creating responsive feature highlight grids + - **[YouTube video embed shortcode](/ui-examples/youtube-video-embed)** - For embedding videos with proper styling and SEO + +## How to contribute to our docs + +We appreciate any form of engagement and contribution to our documentation. You can contribute in several ways: + +- [Report an error](https://github.com/TykTechnologies/tyk-docs/issues) in our GitHub repo +- [Suggest/request an improvement](https://github.com/TykTechnologies/tyk-docs/issues) in our GitHub repo +- Post a message in our [community forum](https://community.tyk.io/) +- Update the code yourself: + 1. To find the page that needs editing in the GitHub repository, copy a sentence from the HTML page on the [docs website](https://tyk.io/docs) and search for it using the GitHub search box in the top left corner of the repository + 2. [Submit a PR](https://github.com/TykTechnologies/tyk-docs/pulls) directly to our [docs repo](https://github.com/TykTechnologies/tyk-docs/). See our [Guidelines for Pull Requests](https://github.com/TykTechnologies/tyk-docs/blob/master/CONTRIBUTING.md#guidelines-for-pull-requests) for detailed instructions. + +## Getting started with contributions + +For detailed contribution guidelines, including pull request requirements, coding conventions, and technical guidance, please refer to our comprehensive [CONTRIBUTING.md](https://github.com/TykTechnologies/tyk-docs/blob/master/CONTRIBUTING.md) file in the repository. + +### Technical guidance + +Our docs are compiled using the [Hugo static site generator](https://gohugo.io/). For detailed technical guidance on creating and updating documentation pages, see the [technical guide](https://github.com/TykTechnologies/tyk-docs/blob/master/CONTRIBUTING-TECHNICAL-GUIDE.md) referenced in our [CONTRIBUTING.md](https://github.com/TykTechnologies/tyk-docs/blob/master/CONTRIBUTING.md) file. + diff --git a/developer-support/deprecation.mdx b/developer-support/deprecation.mdx new file mode 100644 index 00000000..9bce0f7c --- /dev/null +++ b/developer-support/deprecation.mdx @@ -0,0 +1,59 @@ +--- +title: "Tyk Deprecation and EOL Policy" +'og:description': "Official schedule and information regarding Tyk component deprecation and End-of-Life (EOL)." +sidebarTitle: "Deprecation and EOL Policy" +tags: ['Deprecation policy', 'EOL Policy', 'End of Life Policy'] +--- + +## Introduction + +This page provides important information about the deprecation and End-of-Life (EOL) schedules for various **Tyk components, features, and packages**. Our goal is to keep you informed so you can plan your upgrades effectively and maintain a secure and supported environment. + +**Scope:** This policy and the notices on this page apply specifically to software, features, and components developed and distributed by Tyk. While we actively manage and update our external dependencies (like databases, libraries, Go versions), their individual EOL schedules are governed by their respective maintainers and are outside the scope of this specific Tyk deprecation policy. + +Currently, this page primarily focuses on Tyk software **packages** (Debian/RPM distributions hosted on packagecloud.io). We will expand this page to include other components or features as their lifecycle status changes. Please consider this a living document and check back periodically for updates. + +## Understanding Deprecation and End-of-Life (EOL) + +It's crucial to understand the terms we use for the lifecycle of Tyk components and features: + +* **Deprecation:** This marks the phased retirement of a Tyk component or feature. + * **What it means:** The component/feature will still be available for a period but will **no longer receive active support, updates, or enhancements** from Tyk. + * **Action Required:** Customers are advised to plan and execute an upgrade or migration to the recommended alternative within the specified timeframe (the deprecation window). + * **Future State:** Continued use is discouraged as the component/feature will eventually reach its End-of-Life (EOL). + +* **End-of-Life (EOL):** This is the final stage following deprecation. + * **What it means:** The Tyk component or feature is **completely removed from the product or service**, or rendered inaccessible. + * **Action Required:** After the specified EOL date, the component/feature will no longer be available or functional. All dependencies on it within your systems must be eliminated *before* this date. + +* **Deprecation Window:** This refers to the period between the initial deprecation announcement (when support ceases) and the final End-of-Life (EOL) date when the component/feature is removed or becomes inaccessible. + +**Key Distinction:** Deprecation is the *start* of the retirement process (support ends), while EOL is the *end* (removal/inaccessibility). + +## Deprecated Packages Schedule & Repository Removal + +The table below outlines the lifecycle dates for specific versions of Tyk software **packages** hosted on [packagecloud.io/tyk/](https://packagecloud.io/tyk/). + +| Component | Deprecation Date (Support Ends) | EOL Date (Package Removal) | Reason | Recommended Action | +| :------------------------ | :------------------------------ | :------------------------- | :--------------------------- | :----------------------------- | +| Gateway & Dashboard < v3.0 | 2023 | March 15, 2025 | Version reached End-of-Life | Upgrade to a supported version | +| Pump < v1.6.0 | 2023 | March 15, 2025 | Version reached End-of-Life | Upgrade to a supported version | +| Tyk Identity Broker < v1.1.0 | 2023 | March 15, 2025 | Version reached End-of-Life | Upgrade to a supported version | +| MDCB < v1.8.2 | 2023 | March 15, 2025 | Version reached End-of-Life | Upgrade to a supported version | + +*Deprecation Date indicates when support and updates ceased. EOL Date indicates when the package artifact will be removed from the public repository.* + +## Plan Your Upgrade + +Running outdated software versions can expose your systems to security vulnerabilities and prevent you from benefiting from the latest features, performance improvements, and bug fixes. Components that have passed their Deprecation Date are unsupported. + +We strongly recommend upgrading any components that are deprecated or nearing their EOL date to a currently supported version. + +* **Upgrade Guides:** Please consult the official [Tyk Upgrade Documentation](/developer-support/upgrading) for detailed instructions. +* **Release Notes:** Review the [Tyk Release Notes](/developer-support/release-notes/overview) for information on new features and changes in recent versions. + +## Need Assistance? + +If you have questions regarding this schedule, require help planning your upgrade strategy, or need specific guidance related to your Tyk deployment: + +* **Contact Tyk Support:** Please reach out to our [Support Team](https://tyk.io/contact) for assistance. We're here to help ensure your upgrade process is as smooth as possible. \ No newline at end of file diff --git a/developer-support/documentation-projects/inclusive-naming.mdx b/developer-support/documentation-projects/inclusive-naming.mdx new file mode 100644 index 00000000..12309656 --- /dev/null +++ b/developer-support/documentation-projects/inclusive-naming.mdx @@ -0,0 +1,114 @@ +--- +title: "Inclusive Naming" +'og:description': "Explains the inclusive naming initiative concerning Tyk docs" +sidebarTitle: "Inclusive Naming" +tags: ['Inclusive Naming Initiative', 'Inclusivity', 'Inclusive', 'Inclusive Naming Project'] +--- + +This document is intended for Tyk users, contributors, and anyone interested in our commitment to inclusive language within Tyk's documentation and product interfaces. + +## Inclusive Naming project +We are excited to announce the launch of our *Inclusive Naming* project, in June 2024, dedicated to updating our documentation and aligning with the [Inclusive Naming Initiative (INI)](https://inclusivenaming.org). This initiative reflects our commitment to fostering an inclusive and respectful environment for our users and within our company. + +The [Inclusive Naming Initiative](https://inclusivenaming.org/) is a community-driven effort to promote and standardize the use of inclusive language in software and documentation. By adhering to their guidelines, we aim to eliminate terms that can be considered exclusionary, offensive, or insensitive and replace them with language that is respectful and welcoming to all. + +### Purpose + +Our commitment to diversity, equity, and inclusion is foundational to our values. By updating our documentation to comply with the *Inclusive Naming Initiative (INI)*, we are taking a significant step towards ensuring that our language reflects our dedication to inclusivity. This project will help us: + +- **Create a more welcoming environment**: By using inclusive language, we create a space where everyone feels valued and respected. +- **Enhance accessibility**: Clear and inclusive documentation improves accessibility for all users, regardless of their background or identity. + +### Tier 1 word list + +INI sorts terms into word lists, considering both the severity of the term and the level of scrutiny it has received. [INI Tier 1 words](https://inclusivenaming.org/word-lists/tier-1) are considered critical and are recommended to be replaced immediately. + +INI has identified that terms included in this list have one or all of the following attributes: + +- Strong social consensus within the software development community on replacements +- Are identified by the INI as high-severity terms in need of immediate replacement +- Terms where the impact of change or removal is low: for example, there is little entanglement in low-level systems or standardized language set by standards bodies +- Have passed through all the review stages in Tiers 2 and 3 + +### Phase #1: Review of Tyk Documentation for Tier 1 Words + +An initial review of the Tyk documentation was conducted in April 2024 to check which tier 1 words can be replaced, which can't, and why. + +#### Findings and planning +The main findings of the review are: + +1. **Explanatory content with INI tier 1 words**: The content on these pages can be easily rephrased and is now completed. +2. **Configuration parameters containing INI tier 1 words**: + - Some *INI tier 1 words* are embedded in the code as part of the name of parameters, fields, and keywords. + - These words are sourced from Tyk products, and third-party libraries, and tooling, e.g. Redis. + - Possible actions: + - Being part of the source code, we can't simply rephrase and use a different terminology. Such change requires following a deprecation process to ensure there are no breaking changes for our users. + - For now, we can minimize their usage and rephrase the explanatory content to use inclusive words. + + +### Phase #2: Removing Tier 1 words from Tyk documentation + +In June 2024, based on the review we executed the planned changes to the content in our [documentation](https://github.com/tykTechnologies/tyk-docs/). + +#### Status update +This is the update on the status of our documentation +1. **Regular explanatory content with INI tier 1 words**: Content in the documentation has been rephrased and the work is now completed. +2. **Configuration parameters containing INI tier 1 words**: These words are still in our docs, however, we minimized their usage and rephrased their explanatory content to use inclusive words. + - **Tyk products**: These words are still in our docs, however, Tyk aims to gradually replace them (in a backward-compatible way) and update the docs accordingly. + - **Third-party and dependencies**: There's nothing much we can do at the moment except wait for them to replace these parameters, however, we are committed to updating our docs once this gets done. + +#### List of configuration parameters containing INI tier 1 word +For your records, the following sections highlight the existing *INI tier 1 words* in our docs, per Tyk component: + +##### Tyk Gateway + +**Config parameters** + +- [allow_master_keys](/tyk-oss-gateway/configuration#allow_master_keys) +- [analytics_storage.master_name](/tyk-oss-gateway/configuration#analytics_storagemaster_name) +- [cache_storage.master_name](/tyk-oss-gateway/configuration#cache_storagemaster_name) +- [storage.master_name](/tyk-oss-gateway/configuration#storagemaster_name) +- [slave_options](/tyk-oss-gateway/configuration#slave_options) +- [blacklisted_ips](/api-management/gateway-config-tyk-classic#ip-access-control) +- [disable_ports_whitelist](/key-concepts/tcp-proxy#allowing-specific-ports) +- [enable_ip_blacklisting](/api-management/gateway-config-tyk-classic#ip-access-control) +- [ports_whitelist](/key-concepts/tcp-proxy#allowing-specific-ports) + +
Tyk Classic API Definition
+The [Tyk Gateway OpenAPI Document](https://github.com/TykTechnologies/tyk-docs/blob/master/tyk-docs/assets/others/gateway-swagger.yml) (Tyk Gateway swagger), includes references to the following Tyk Classic API Definition parameters: + +- [version_data.versions.\{version-name\}.extended_paths.black_list](/api-management/traffic-transformation/block-list#api-definition). There is also a parameter with equivalent functionality under the `paths` object (`version_data.versions.{version_name}.paths.black_list`). +- [version_data.versions.\{version-name\}.extended_paths.white_list](/api-management/traffic-transformation/allow-list#api-definition). There is also a parameter with equivalent functionality under the `paths` object (`version_data.versions.{version_name}.paths.while_list`). + +##### Tyk Dashboard + +**Config parameters** +- [enable_master_keys](/tyk-dashboard/configuration#enable_master_keys) +- [redis_master_name](/tyk-dashboard/configuration#redis_master_name) + +**Tyk Classic API Definition** + +The [Tyk Dashboard OpenAPI Document](https://github.com/TykTechnologies/tyk-docs/blob/master/tyk-docs/assets/others/dashboard-swagger.yml) (Tyk Dashboard swagger), contains references to [the parameters from the above Tyk Classic API Definition list](#gw-classic-api-definition). + +**Dashboard UI** + +The Tyk Classic APIs *Endpoint Designer* shows configuration of [blacklist](/api-management/traffic-transformation/block-list#api-designer) and [whitelist](/api-management/traffic-transformation/allow-list#api-definition) middleware plugins. + +##### Tyk MDCB + +The following MDCB configuration parameters contain tier 1 word occurrences: +- [analytics_storage.master_name](/tyk-multi-data-centre/mdcb-configuration-options#analytics_storagemaster_name) +- [storage.master_name](/tyk-multi-data-centre/mdcb-configuration-options#storagemaster_name) + +##### Tyk Pump + +The following Tyk Pump configuration parameters contain tier 1 word occurrences: +- [analytics_storage_config.master_name](/tyk-pump/tyk-pump-configuration/tyk-pump-environment-variables#analytics_storage_configmaster_name) + +##### Third-party dependencies + +Content contains *INI Tier 1 word* occurrences due to the following external dependencies: +- Links to Tyk Component GitHub repositories with a default branch set as `master`. +- Tyk Gateway and Tyk Pump content use Redis terminology for `master` in relation to key storage and analytics. +- Tyk Classic Developer Portal provides [documentation](/tyk-developer-portal/tyk-portal-classic/keycloak-dcr) that explains how to integrate Tyk with Keycloak using the [OpenID Connect Dynamic Client Registration](https://tools.ietf.org/html/rfc7591) protocol. The example in the guide uses the Keycloak default `master` realm. +- Tyk Bitnami Helm charts use a service with a DNS name of *tyk-redis-master.tyk.svc*. \ No newline at end of file diff --git a/developer-support/faq.mdx b/developer-support/faq.mdx new file mode 100644 index 00000000..f0ab3c93 --- /dev/null +++ b/developer-support/faq.mdx @@ -0,0 +1,206 @@ +--- +title: "Frequently Asked Questions" +tags: ['FAQ', 'frequently asked questions', 'answers', 'configuration files backup', 'backup tyk', 'tyk.conf', 'upgrade tyk', 'database backup', 'Analytics', 'Distributed Analytics', 'Redis', 'Redis Shards', 'analytics_config.enable_multiple_analytics_keys', 'do_not_track', 'RPS', 'Requests Per Second', 'CPU', 'high load', 'high traffic'] +order: 230 +sidebarTitle: "Frequently Asked Questions" +--- + +import { ButtonLeft } from '/snippets/ButtonLeft.mdx'; + +This section lists commonly asked questions or frequently encountered issues and explains how to resolve them. + +## Tyk Configuration + +1. **If there's a configuration error or an unwanted change, can I easily revert to a previous API or Policy configuration? What are the options or best practices for effective rollbacks?** + + You can configure [Tyk-Sync](/api-management/automations/sync) to synchronise APIs and Policies with any version control system, like GitHub or GitLab and use it to perform roll back. Keys are not synchronised with Tyk Sync. + +2. **How to Backup Configuration Files of Tyk Component's ?** + + Every Tyk component has a config file, that is used during start-up. They store critical settings and parameters that determine how your application behaves as well as settings for databases, passwords, and connections to other components. + To ensure a comprehensive backup strategy, regularly back up configuration files for all relevant components, using a version control system (such as Git). + +3. **What are the different Config file per Tyk component ?** + + These are the config files per component: + - Tyk Gateway - `tyk.conf` (For any user) + - Tyk Pump - `pump.conf` (For any user) + - Tyk Dashboard - `tyk_analytics.conf` (For Tyk Self-Managed clients) + - MDCB - `tyk_sink.conf` (for MDCB clients) + - Hybrid Tyk Gateway - `tyk.hybrid.conf` + + **Note:** These are the config files with Tyk's default names. You might have different names for your config files. + +4. **How to Backup Tyk's Certificates Directory ?** + + Path to the private keys and certificate are defined in the config files of any of Tyk components. Make sure to back up the certificate and keys used by these config files. + +5. **How to Backup Middleware Directory for Custom Plugins ?** + + If you use custom middleware plugins to extend your application's capabilities, regularly back up the middleware directory to preserve your customizations. + Details on middleware installation for [Tyk Self-Managed users](/api-management/plugins/javascript#installing-middleware-on-tyk-self-managed) (users of the licensed product). + Details on middleware installation for [Tyk Open Source users](/api-management/plugins/javascript#installing-middleware-on-tyk-oss) + + **Note:** Tyk's default name is `/middleware`. You might use a different name for it. + + +6. **How to Backup Tyk API Definitions Directory - For Tyk Open source users ?** + + When using *Tyk Gateway* as an open-source product, it could use a dedicated directory to load your Tyk API definitions and start serving them. + If you use such a directory, ensure these definitions are part of your backup plan. + + **Note:** Tyk's default name is `/apps`. You might use a different name for it based on this [config field](/tyk-stack/tyk-gateway/important-prerequisites#tyk-config) + +7. **How to Backup Tyk Policies Directory - For Tyk Open source users ?** + + When using *Tyk Gateway* as an open-source product, it could use a file to load policies to be used by API keys when consuming the APIs. + If you use a policies file, ensure it is also part of your backup plan. + + **Note:** Tyk's default name is `/policies`. You might use a different name for it based on this [config field](/tyk-stack/tyk-gateway/important-prerequisites#path-to-policies-file) + +8. **How to Backup Redis ?** + + **Users:** Redis is used in any type of deployment and as such used by any type of user (Open Source and paying users). + + Backup Redis is important as all of the keys used by *Tyk Gateway* are stored there. + Redis, being an in-memory data store, is ephemeral and doesn't have a built-in default backup policy, as such it requires specific considerations for backup. + To understand the best practices for Redis backup, please visit the official [Redis Backup Documentation](https://redis.io/docs/management/persistence/). + + +9. **How to Backup Databases Used by Tyk Dashboard - a licensed product ?** + + **Users:** For Tyk Self-managed and MDCB users who are using our licensed products. + + Tyk Dashboard allows you to choose between *MongoDB* and *PostgreSQL* as your database solution, depending on your preferences and requirements. Regardless of your choice, it's crucial to implement a robust backup strategy to ensure the safety and availability of your data. Please check the section that fits your use case: + + + **MongoDB Backup** + + MongoDB provides various methods for creating backups, and the choice of strategy often depends on your specific use case and environment. To learn more about MongoDB backup strategies, please refer to the [MongoDB Backup Documentation](https://www.mongodb.com/docs/manual/core/backups/). + - For *Amazon DocumentDB* user, check their [backup and restore documentation](https://docs.aws.amazon.com/documentdb/latest/developerguide/backup_restore.html) + - For *CosmosDB* users check their [online backup and on-demand data restore documentation](https://learn.microsoft.com/en-us/azure/cosmos-db/online-backup-and-restore) + + **Postgres Backup** + + PostgreSQL offers multiple backup options, including logical and physical backups. The right choice depends on your database size, recovery time objectives, and other factors. For detailed information on PostgreSQL backup strategies, please consult the [PostgreSQL Backup and Restore Documentation](https://www.postgresql.org/docs/current/backup.html). + + - For *Amazon RDS* users, check their [backup and restore documentation](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_CommonTasks.BackupRestore.html). To further enhance your PostgreSQL backup process, you can explore services like [AWS RDS Automated Backups](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_WorkingWithAutomatedBackups.html) if you're hosting your database on AWS. + - For *CosmosDB* users check their [online backup and on-demand data restore documentation](https://learn.microsoft.com/en-us/azure/cosmos-db/postgresql/concepts-backup) + +## Enterprise Developer Portal + +1. **What happens if the Portal goes down ?** + + In the event of the portal application being down, the other components of the Tyk Stack will remain unaffected. + This means your APIs will still be operational, and analytics will continue to be recorded. + Developers will also be able to use their credentials for both oAuth2.0 and API Keys APIs. + + However, since the portal application is down, developers won't be able to access their credentials or the analytical dashboard, request access to new API Products, or revoke or rotate their access credentials. + Additionally, admin users won't be able to use the portal, whether through its UI or APIs. + This means you won't be able to create, change, or remove any item managed by the portal, such as developers, organizations, content pages, API Products, plans, and more. + + Despite this, you still have some control over access credentials. + If you need to rotate or remove access credentials, you can do so directly in the Tyk Dashboard or in your identity provider. + +2. **What happens if the Dashboard goes down ?** + + If the Tyk Dashboard goes down, developers will still be able to access their access credentials, but they won't be able to rotate or remove their existing credentials, or request access to API Products. + Additionally, the API Analytics dashboard will be compromised. + + However, API traffic will remain unaffected, meaning that your APIs will continue to be operational, and analytics will continue to be recorded. + + In terms of admin functionality, the only limitation will be the inability to approve or reject access requests or revoke or rotate access credentials. + + +3. **Does the portal support SQL databases for storing the portal's CMS assets ?** + + + + + Tyk no longer supports SQLite as of Tyk 5.7.0. To avoid disruption, please transition to [PostgreSQL](/planning-for-production/database-settings#postgresql), [MongoDB](/planning-for-production/database-settings#mongodb), or one of the listed compatible alternatives. + + + + The Enterprise Developer Portal supports SQL databases (MariaDB, MySQL, and PostgreSQL) for storing the portal's CMS assets. + During the bootstrap process, the portal will create the appropriate tables in the main database. The only thing required to enable SQL storage for the portal's assets is to specify the `db` [storage type](/product-stack/tyk-enterprise-developer-portal/deploy/configuration#portal_storage) either via a config file or an environment variable. + +## Tyk Gateway + +1. **How to Check Your Gateway Version ?** + + Since Gateway version `5.0.8` or `5.2.3` you can inspect detailed build information including the release version by running `tyk version`. + + ```console + Release version: v5.3.0-dev + Built by: goreleaser + Build date: + Commit: + Go version: go1.20 + OS/Arch: linux/amd64 + ``` + + If you need this in a machine readable format, a `--json` flag is available. + + ```json + { + "Version": "v5.3.0-dev", + "BuiltBy": "goreleaser", + "BuildDate": "", + "Commit": "", + "Go": { + "Os": "linux", + "Arch": "amd64", + "Version": "go1.20" + } + } + ``` + + For older versions of Gateway, you can run `tyk --version` to print the release version for your tyk binary. + + The binary is installed in `/opt/tyk-gateway/tyk` by default. If your binary is not available in your `PATH` environment, invoke it from there. + + ``` + time="Oct 31 17:06:06" level=info msg="Tyk API Gateway v5.3.0-dev" prefix=main + ``` + +## Tyk Cloud + +1. **How can I track and audit changes made to Tyk Dashboard configurations ?** + + To track changes in Dashboard configurations, please submit a [support ticket](https://support.tyk.io/hc/en-gb/articles/8671452599708-Ticket-Submission-Guide). No configuration changes will occur without your explicit request. + +## Miscellaneous + +1. **Error initialising system couldn't unmarshal config** + + The error "Error initializing system: couldn't unmarshal config: invalid character" occurs in the Tyk Gateway logs due to improper syntax in the configuration files. To resolve this, carefully review and correct the syntax in all Tyk configuration files, restart the Gateway, and, if the issue persists, validate the JSON files using [JSONLint](https://jsonlint.com/). + +2. **All tyk container logs show up under the error status in Datadog logs** + + With Datadog you can view logs of all your Tyk components. + To allow Datadog Agent to scrape the logs of your Tyk deployment correctly you need to create a pipeline in Datadog, to process and underst and this data. + + To do that, we need to access the `/logs/pipelines` path on your datadog web application. + This will take us to the pipeline configuration page. + In here, we will create a new pipeline. + For the filter section, use `Service:tyk-*` this will capture logs for any of the Tyk related services. + + Create Datadog Logs Pipeline to process Tyk services' logs + + Next, we will need to add a processor to the pipeline. + + Create Datadog Logs Pipeline to process Tyk services' logs + + Select the Grok Parser processor type, give it a name and click on the `Parse My Logs` button and `Create` . + + Create pipeline processor to parse grok statements + + Lastly, add another processor to the pipeline. Select the Status Remapper processor type, give it a name and set the status attribute to `level` then `Create`. + + Create pipeline processor to remap the status of the log to level attribute value + + The Tyk logs statuses should now be shown under the right status. + + Contact us to learn more: + + diff --git a/developer-support/inclusive-naming-policy.mdx b/developer-support/inclusive-naming-policy.mdx new file mode 100644 index 00000000..0a248315 --- /dev/null +++ b/developer-support/inclusive-naming-policy.mdx @@ -0,0 +1,109 @@ +--- +title: "Inclusive Naming Policy" +sidebarTitle: "Inclusive Naming Policy" +tags: ['Inclusive Naming Initiative', 'Inclusivity', 'Inclusive'] +--- + +In June, 2024, we announced the launch of our *Inclusive Naming* project, dedicated to updating our documentation and aligning with the [Inclusive Naming Initiative (INI)](https://inclusivenaming.org). This initiative reflects our commitment to fostering an inclusive and respectful environment for our users and within our company. + +The [Inclusive Naming Initiative](https://inclusivenaming.org/) is a community-driven effort to promote and standardize the use of inclusive language in software and documentation. By adhering to their guidelines, we aim to eliminate terms that can be considered exclusionary, offensive, or insensitive and replace them with language that is respectful and welcoming to all. + +## Purpose + +Our commitment to diversity, equity, and inclusion is foundational to our values. By updating our documentation to comply with the *Inclusive Naming Initiative (INI)*, we are taking a significant step towards ensuring that our language reflects our dedication to inclusivity. This project will help us: + +- **Create a more welcoming environment**: By using inclusive language, we create a space where everyone feels valued and respected. +- **Enhance accessibility**: Clear and inclusive documentation improves accessibility for all users, regardless of their background or identity. + +## Tier 1 word list + +INI sorts terms into word lists, considering both the severity of the term and the level of scrutiny it has received. [INI Tier 1 words](https://inclusivenaming.org/word-lists/tier-1) are considered critical and are recommended to be replaced immediately. + +INI has identified that terms included in this list have one or all of the following attributes: + +- Strong social consensus within the software development community on replacements +- Are identified by the INI as high-severity terms in need of immediate replacement +- Terms where the impact of change or removal is low: for example, there is little entanglement in low-level systems or standardized language set by standards bodies +- Have passed through all the review stages in Tiers 2 and 3 + +## Phase #1: Review of Tyk Documentation for Tier 1 Words + +An initial review of the Tyk documentation was conducted in April 2024 to check which tier 1 words can be replaced, which can't, and why. + +### Findings and planning +The main findings of the review are: + +1. **Explanatory content with INI tier 1 words**: The content on these pages can be easily rephrased and is now completed. +2. **Configuration parameters containing INI tier 1 words**: + - Some *INI tier 1 words* are embedded in the code as part of the name of parameters, fields, and keywords. + - These words are sourced from Tyk products, and third-party libraries, and tooling, e.g. Redis. + - Possible actions: + - Being part of the source code, we can't simply rephrase and use a different terminology. Such change requires following a deprecation process to ensure there are no breaking changes for our users. + - For now, we can minimize their usage and rephrase the explanatory content to use inclusive words. + + +## Phase #2: Removing Tier 1 words from Tyk documentation + +In June 2024, based on the review we executed the planned changes to the content in our [documentation](https://github.com/tykTechnologies/tyk-docs/). + +### Status update +This is the update on the status of our documentation +1. **Regular explanatory content with INI tier 1 words**: Content in the documentation has been rephrased and the work is now completed. +2. **Configuration parameters containing INI tier 1 words**: These words are still in our docs, however, we minimized their usage and rephrased their explanatory content to use inclusive words. + - **Tyk products**: These words are still in our docs, however, Tyk aims to gradually replace them (in a backward-compatible way) and update the docs accordingly. + - **Third-party and dependencies**: There's nothing much we can do at the moment except wait for them to replace these parameters, however, we are committed to updating our docs once this gets done. + +For your records, the following sections highlight the existing *INI tier 1 words* in our docs, per Tyk component: + +#### Tyk Gateway + +**Config parameters** + +- [allow_master_keys](/tyk-oss-gateway/configuration#allow_master_keys) +- [analytics_storage.master_name](/tyk-oss-gateway/configuration#analytics_storagemaster_name) +- [cache_storage.master_name](/tyk-oss-gateway/configuration#cache_storagemaster_name) +- [storage.master_name](/tyk-oss-gateway/configuration#storagemaster_name) +- [slave_options](/tyk-oss-gateway/configuration#slave_options) +- [blacklisted_ips](/api-management/gateway-config-tyk-classic#ip-access-control) +- [disable_ports_whitelist](/key-concepts/tcp-proxy#allowing-specific-ports) +- [enable_ip_blacklisting](/api-management/gateway-config-tyk-classic#ip-access-control) +- [ports_whitelist](/key-concepts/tcp-proxy#allowing-specific-ports) + +
Tyk Classic API Definition
+The [Tyk Gateway OpenAPI Document](https://github.com/TykTechnologies/tyk-docs/blob/master/tyk-docs/assets/others/gateway-swagger.yml) (Tyk Gateway swagger), includes references to the following Tyk Classic API Definition parameters: + +- [version_data.versions.\{version-name\}.extended_paths.black_list](/api-management/traffic-transformation/block-list#api-definition). There is also a parameter with equivalent functionality under the `paths` object (`version_data.versions.{version_name}.paths.black_list`). +- [version_data.versions.\{version-name\}.extended_paths.white_list](/api-management/traffic-transformation/allow-list#api-definition). There is also a parameter with equivalent functionality under the `paths` object (`version_data.versions.{version_name}.paths.while_list`). + +#### Tyk Dashboard + +**Config parameters** +- [enable_master_keys](/tyk-dashboard/configuration#enable_master_keys) +- [redis_master_name](/tyk-dashboard/configuration#redis_master_name) + +**Tyk Classic API Definition** + +The [Tyk Dashboard OpenAPI Document](https://github.com/TykTechnologies/tyk-docs/blob/master/tyk-docs/assets/others/dashboard-swagger.yml) (Tyk Dashboard swagger), contains references to [the parameters from the above Tyk Classic API Definition list](#gw-classic-api-definition). + +**Dashboard UI** + +The Tyk Classic APIs *Endpoint Designer* shows configuration of [blacklist](/api-management/traffic-transformation/block-list#api-designer) and [whitelist](/api-management/traffic-transformation/allow-list#api-definition) middleware plugins. + +#### Tyk MDCB + +The following MDCB configuration parameters contain tier 1 word occurrences: +- [analytics_storage.master_name](/tyk-multi-data-centre/mdcb-configuration-options#analytics_storagemaster_name) +- [storage.master_name](/tyk-multi-data-centre/mdcb-configuration-options#storagemaster_name) + +#### Tyk Pump + +The following Tyk Pump configuration parameters contain tier 1 word occurrences: +- [analytics_storage_config.master_name](/tyk-pump/tyk-pump-configuration/tyk-pump-environment-variables#analytics_storage_configmaster_name) + +#### Third-party dependencies + +Content contains *INI Tier 1 word* occurrences due to the following external dependencies: +- Links to Tyk Component GitHub repositories with a default branch set as `master`. +- Tyk Gateway and Tyk Pump content use Redis terminology for `master` in relation to key storage and analytics. +- Tyk Classic Developer Portal provides [documentation](/tyk-developer-portal/tyk-portal-classic/keycloak-dcr) that explains how to integrate Tyk with Keycloak using the [OpenID Connect Dynamic Client Registration](https://tools.ietf.org/html/rfc7591) protocol. The example in the guide uses the Keycloak default `master` realm. +- Tyk Bitnami Helm charts use a service with a DNS name of *tyk-redis-master.tyk.svc*. diff --git a/developer-support/release-notes/archived.mdx b/developer-support/release-notes/archived.mdx new file mode 100644 index 00000000..f27937a2 --- /dev/null +++ b/developer-support/release-notes/archived.mdx @@ -0,0 +1,1233 @@ +--- +title: "Archived Releases" +'og:description': "Tyk Old releases" +sidebarTitle: "Archived Releases" +tags: ['Tyk', 'Archived', 'Release notes', 'v2.4', 'v2.5', 'v2.6', 'v2.7', 'v2.8', 'v2.9', '2.9'] +--- + +## 2.9.0 Release Notes + +### TCP Proxying + +Tyk now can be used as a reverse proxy for your TCP services. It means that you can put Tyk not only on top of your APIs, but on top of **any** network application, like databases, services using custom protocols and etc. + +The main benefit of using Tyk as your TCP proxy is that functionality you used to managed your APIs now can be used for your TCP services as well. Features like load balancing, service discovery, Mutual TLS (both authorization and communication with upstream), certificate pinning: all work exactly the same way as for your HTTP APIs. + +See our [TCP Proxy Docs](/key-concepts/tcp-proxy) for more details. + +### APIs as Products + +With this release we have removed all the barriers on how you can mix and match policies together, providing you with ultimate flexibility for configuring your access rules. + +Now a key can have multiple policies, each containing rules for different APIs. In this case each distinct policy will have its own rate limit and quota counters. For example if the first policy gives access to `API1` and second policy to `API2` and `API3`, if you create a key with both policies, your user will have access to all three APIs, where `API1` will have quotas and rate limits defined inside the first policy, and `API2`, `API3` will have shared quotas and rate limits defined inside the second policy. + +Additionally you can now mix policies defined for the same API but having different path and methods access rules. For example you can have one policy which allows only access to `/users` and a second policy giving user access to a `/companies` path. If you create a key with both policies, their access rules will be merged, and user will get access to both paths. See [Multiple APIs for single Key Requests](/tyk-developer-portal/tyk-portal-classic/portal-concepts#multiple-apis-for-a-single-key-request). + +#### Developer Portal Updates + +Developers now can have multiple API keys, and subscribe to multiple catalogs with a single key. Go to the Portal settings and set `Enable subscribing to multiple APIs with single key` option to enable this new flow. When enabled, developers will see the new API generation user interface, which allows users to request access to multiple Catalogs of the **same type** with a single key. + +From an implementation point of view, Developer objects now have a `Keys` attribute, which is the map where the key is a `key` and the value is an array of policy IDs. The `Subscriptions` field can be considered as deprecated, with retained backwards compatibility. We have added new set of Developer APIs to manage the keys, similar to the deprecated subscriptions APIs. + +Other changes: + +- Added two new Portal templates, which are used by a new key request flow `portal/templates/request_multi_key.html`, `portal/templates/request_multi_key_success.html` +- The Portal Catalog list page has been updated to show the Catalog authentication mode +- The API dashboard screen now show keys instead of subscriptions, and if subscribed to multiple policies, it will show the allowance rules for all catalogs. +- The Key request API has been updated to accept an `apply_policies` array instead of `for_plan` + +### JWT and OpenID scope support + +Now you can set granular permissions on per user basis, by injecting permissions to the "scope" claim of a JSON Web Token. To make it work you need to provide mapping between the scope and policy ID, and thanks to enchanced policy merging capabilities mentioned above, Tyk will read the scope value from the JWT and will generate dynamic access rules. Your JWT scopes can look like `"users:read companies:write"` or similar, it is up to your imagination. OpenID supports it as well, but at the moment only if your OIDC provider can generate ID tokens in JWT format (which is very common this days). + +See our [JWT Scope docs](/basic-config-and-security/security/authentication-authorization/json-web-tokens) for more details. + +### Go plugins + +[Go](https://golang.org/) is an open source programming language that makes it easy to build simple, reliable, and efficient software. The whole Tyk stack is written in Go language, and it is one of the reasons of behind our success. + +With this release you now can write native Go plugins for Tyk. Which means extreme flexibility and the best performance without any overhead. + +Your plugin can be as simple as: + +```{.go} expandable +package main +import ( + "net/http" +) +// AddFooBarHeader adds custom "Foo: Bar" header to the request +func AddFooBarHeader(rw http.ResponseWriter, r *http.Request) { + r.Header.Add("Foo", "Bar") +} +``` + +See our [Golang plugin documentation](/api-management/plugins/golang#) for more details. + +### Distributed tracing + +We have listened to you, and tracing is recently one of your most common requests. Distributed tracing takes your monitoring and profiling experience to the next level, since you can see the whole request flow, even if it has complex route though multiple services. And inside this flow, you can go deep down into the details like individual middleware execution performance. +At the moment we are offering [OpenTracing](https://opentracing.io/) support, with [Zipkin](https://zipkin.io/) and [Jaeger](https://www.jaegertracing.io/) as supported tracers. + +See our [Distributed Tracing documentation](/api-management/logs-metrics#opentelemetry) for more details. + +### HMAC request signing + +Now Tyk can sign a request with HMAC, before sending to the upsteam target. + +This feature is implemented using [Draft 10](https://tools.ietf.org/html/draft-cavage-http-signatures-10) RFC. + +`(request-target)` and all the headers of the request will be used for generating signature string. +If the request doesn't contain a `Date` header, middleware will add one as it is required according to above draft. + +A new config option `request_signing` can be added in an API Definition to enable/disable the request signing. It has following format: + +```{.json} expandable +"request_signing": { + "is_enabled": true, + "secret": "xxxx", + "key_id": "1", + "algorithm": "hmac-sha256" +} +``` + +The following algorithms are supported: + +1. `hmac-sha1` +2. `hmac-sha256` +3. `hmac-sha384` +4. `hmac-sha512` + +### Simplified Dashboard installation experience + +We worked a lot with our clients to build a way nicer on-boarding experience for Tyk. Instead of using the command line, you can just run the Dashboard, and complete a form which will configure your Dashboard. However, we did not forget about our experienced users too, and now provide a CLI enchanced tool for bootstrapping Tyk via a command line. + +See our updated [Getting Started](/tyk-self-managed/install) section and [new CLI documentation](/tyk-self-managed). + +### DNS Caching + +Added a global DNS cache in order to reduce the number of request to a Gateway's local DNS server and the appropriate gateway config section. This feature is turned off by default. + +``` +"dns_cache": { + "enabled": true, //Turned off by default + "ttl": 60, //Time in seconds before the record will be removed from cache + "multiple_ips_handle_strategy": "pick_first" //A strategy, which will be used when dns query will reply with more than 1 ip address per single host. +}, +``` + +### Python Plugin Improvements + +We have completed a massive rewrite of our Python scripting engine in order to simplify the installation and usage of Python scripts. From now on, you no longer need to use a separate Tyk binary for Python plugins. Everything is now bundled to the main binary. +This also means that you can combine JSVM, Python and Coprocess plugins inside the same installation. +In addition you can now use any Python 3.x version. Tyk will automatically detect a supported version and will load the required libraries. If you have multiple Python versions available, you can specify the exact version using `python_version`. + +### Importing Custom Keys using the Dashboard API + +Previously if you wanted migrate to Tyk and keep existing API keys, you had to use our low level Tyk Gateway API, which has lot of constraints, especially regarding complex setups with multiple organizations and data centers. + +We have introduced a new Dashboard API for importing custom keys, which is as simple as `POST /api/keys/{custom_key} {key-payload}`. This new API ensures that Keys from multiple orgs will not intersect, and it also works for multi-data center setups, and even Tyk SaaS. + +### Single sign on for the Tyk SaaS + +Before SSO was possible only for Tyk On-Premise, since it required access to low-level Dashboard Admin APIs. With 2.9 we have added new a new Dashboard SSO API, which you can use without having super admin access, and it works at the organization level. This means that all our Tyk SaaS users can use 3rd party IDPs to manage Dashboard users and Portal developers. + +> **NOTE**: This feature is available by request. Please contact our sales team for details. + +See our [Dashboard SSO documentation](/api-management/dashboard-configuration#single-sign-on-api) for more details. + +### Importing WSDL APIs + +WSDL now is a first class citizen at Tyk. You can take your WSDL definition and simply import to the Dashboard, creating a nice boilerplate for your service. See [Import APIs](/api-management/gateway-config-managing-classic#import-an-api) for more details. + +### Updated Versions + +- Tyk Gateway 2.9.0 +- Tyk Dashboard 1.9.0 +- Tyk Pump 0.8.0 +- Tyk MDCB 1.7.0 + +### Upgrading From Version 2.8 + +#### Tyk On-Premises + +For this release, you should upgrade your Tyk Pump first. + +#### Tyk MDCB + +For this release, you should upgrade your MDCB component first. + +## 2.8.0 Release Notes +### Debugger + +You can now safely test all API changes without publishing them, and visually see the whole request flow, including which plugins are running and even their individual logs. + +We have added a new `Debugging` tab in the API designer which provides a "Postman" like HTTP client interface to simulate queries for the current API definition being edited. + +You can even debug your virtual endpoints by dynamically modifying the code, sending the request via `Debugger` and watching the virtual endpoint plugin logs. + +See [Debugging Tab](/api-management/dashboard-configuration#debugging) for more information. + +--- + +### Developer portal oAuth support + +The Developer portal now fully supports exposing oAuth2 APIs: + +* Developers can register their oAuth clients and see analytics +* Administrators can see list of oAuth clients from a developer screen + +--- + +### Multi-organization users + +NOTE: Currently only available with >2 node Dashboard license. + +You can now create users with the same email address in different organizations. Users will then be able to select an organization +when logging in, and can easily switch between organizations via the navigation menu. To enable set +`"enable_multi_org_users": true`. + +--- + +### Developer management improvements + +* You can now manually create developer subscriptions from the developer screen. +* We've added a quick way to change a subscription policy and reset a quota +* All actions on the developer screen now only require developer permissions + +### Dashboard Audit Log improvements + +There is a [new section](/api-management/dashboard-configuration#dashboard-audit-logs) in the Tyk Dashboard config file where you can specify parameters for the audit log (containing audit records for all requests made to all endpoints under the `/api` route). + +--- + + +### Detailed changelog + +- Added API Debugger tab to the API Designer. +- Extended the Portal templating functionality. +- Similar to the Gateway, you now can specify a list of acceptable TLS ciphers using the + `http_server_options.cipher_suites` array option. +- Audit log improvements +- Exposing oAuth2 APIs to developer portal +- Allow for the retrieval of an API via it's external API +- Allow updating keys by hash +- Added support for `SMTP` noauth. + +## 2.7.0 Release Notes + +### Tyk Gateway v2.7.0 + +#### Performance improvements + + +> **TLDR** +> To get benefit or performance improvements ensure that you have `close_connections` set to `false` and set `max_idle_connections_per_host` according to our [production perfomance guide](/planning-for-production) + +We have thoroughly analyzed every part of our Gateway, and the results are astounding, up to 160% improvement, compared to our 2.6 release. + +Such a performance boost comes from various factors, such as optimizing our default configs, better HTTP connection re-use, optimization of the analytics processing pipeline, regexp caching, doing fewer queries to the database, and numerous small changes in each of the middleware we have. + +Our performance testing plan was focused on replicating our customer's setup, and try not to optimize for β€œbenchmarks”: so no supercomputers and no sub-millisecond inner DC latency. Instead, we were testing on average performance 2 CPU Linode machine, with 50ms latency between Tyk and upstream. For testing, we used the Tyk Gateway in Hybrid mode, with a default config, except for a single 2.7 change where `max_idle_connections_per_host ` is set to 500, as apposed to 100 in 2.6. Test runner was using [Locust](https://locust.io/) framework and [Boomer](https://github.com/myzhan/boomer) for load generation. + +For a keyless API we were able to achieve 3.7K RPS (requests per second) for 2.7, while 2.6 showed about 2.5K RPS, which is a 47% improvement. + +For protected APIs, when Tyk needs to track both rate limits and quotas, 2.7 shows around 3.1K RPS, while 2.6 shows around 1.2K RPS, which is 160% improvement! + +In 2.7 we optimized the connection pool between Tyk and upstream, and previously `max_idle_connections_per_host` option was capped to 100. In 2.7 you can set it to any value. `max_idle_connections_per_host` by itself controls an amount of keep-alive connections between clients and Tyk. If you set this value too low, then Tyk will not re-use connections and will have to open a lot of new connections to your upstream. If you set this value too big, you may encounter issues with slow clients occupying your connection and you may reach OS limits. You can calculate the correct value using a straightforward formula: if latency between Tyk and Upstream is around 50ms, then a single connection can handle 1s / 50s = 20 requests. So if you plan to handle 2000 requests per second using Tyk, the size of your connection pool should be at least 2000 / 20 = 100. For example, on low-latency environments (like 5ms), a connection pool of 100 connections will be enough for 20k RPS. + +To get the benefit of optimized connection pooling, ensure that `close_connections` is set to `false`, which enables keep-alive between Tyk and Upstream. + +#### Custom key hashing algorithms + +Key hashing is a security technique introduced inside Tyk a long time ago, which allows you to prevent storing your API tokens in database, and instead, only store their hashes. Only API consumers have access to their API tokens, and API owners have access to the hashes, which gives them access to usage and analytics in a secure manner. Time goes on, algorithms age, and to keep up with the latest security trends, we introduce a way to change algorithms used for key hashing. + +This new feature is in public beta, and turned off by default, keeping old behavior when Tyk uses `murmur32` algorithm. To set the custom algorithm, you need to set `hash_key_function` to one of the following options: +- `murmur32` +- `murmur64` +- `murmur128` +- `sha256` + +MurMur non-cryptographic hash functions is considered as industry fastest and conflict-prone algorithms up to date, which gives a nice balance between security and performance. With this change you now you may choose the different hash length, depending on your organization security policies. As well, we have introduced a new `sha256` **cryptographic** key hashing algorithm, for cases when you are willing to sacrifice performance with additional security. + +Performance wise, setting new key hashing algorithms can increase key hash length, as well as key length itself, so expect that your analytics data size to grow (but not that much, up to 10%). Additionally, if you set the `sha256` algorithm, it will significantly slowdown Tyk, because `cryptographic` functions are slow by design but very secure. + +Technically wise, it is implemented by new key generation algorithms, which now embed additional metadata to the key itself, and if you are curious about the actual implementation details, feel free to check the following [pull request](https://github.com/TykTechnologies/tyk/pull/1753). + +Changing hashing algorithm is entirely backward compatible. All your existing keys will continue working with the old `murmur32` hashing algorithm, and your new keys will use algorithm specified in Tyk config. Moreover, changing algorithms is also backward compatible, and Tyk will maintain keys multiple hashing algorithms without any issues. + + +### Tyk Dashboard v1.7.0 + +#### User Groups + +Instead of setting permissions per user, you can now [create a user group](/api-management/user-management#manage-tyk-dashboard-user-groups), and assign it to multiple users. It works for Single Sign-On too, just specify group ID during [SSO API](/api-management/dashboard-configuration#single-sign-on-api-1) flow. + +This feature is available to all our Cloud and Hybrid users. For Self-Managed installations, this feature is available for customers with an "Unlimited" license. + +To manage user groups, ensure that you have either admin or β€œuser groups” permission for your user, which can be enabled by your admin. + +From an API standpoint, user groups can be managed by [new Dashboard API](/api-management/dashboard-configuration#user-groups-api). The User object now has a new `group_id` field, and if it is specified, all permissions will be inherited from the specified group. [SSO API](/api-management/dashboard-configuration#single-sign-on-api-1) has been updated to include `group_id` field as well. + +#### Added SMTP support +Now you can configure the Dashboard to send transactional emails using your SMTP provider. See [Outbound Email Configuration](/configure/outbound-email-configuration) for details. + +#### Upgrading all new Components + +For details on upgrading all Tyk versions, see [Upgrading Tyk](/developer-support/upgrading). + +### Don't Have Tyk Yet? + +Get started now, for free, or contact us with any questions. + +* [Get Started](https://tyk.io/pricing/compare-api-management-platforms/#get-started) +* [Contact Us](https://tyk.io/about/contact/) + + +## 2.6.0 Release Notes + +### Tyk Gateway v2.6.0 + +#### Organization Level Rate Limiting + +Endpoints Create organization keys and +Add/update organization keys now allow you to set rate limits at an organization level. You will need to add the following fields in your create/add/update key request: + +* `"allowance"` +* `"rate"` + +These are the number of allowed requests for the specified `per` value, and need to be set to the same value. + +* `"per"` is the time period, in seconds. + +So, if you want to restrict an organization rate limit to 100 requests per second you will need to add the following to your request: +``` + "allowance": 100, + "rate": 100, + "per": 5 +``` + +> **NOTE:** if you don't want to have organization level rate limiting, set `"rate"` or `"per"` to zero, or don't add them to your request. + +See the Keys section of the [Tyk Gateway REST API](/tyk-gateway-api) Swagger doc for more details. + +#### Keys hashing improvements + +Now it is possible to do more operations with key by hash (when we set `"hash_keys":` to `true` in `tyk.conf`): + +- endpoints `POST /keys/create`, `POST /keys` and `POST /keys/{keyName}` also return field `"key_hash"` for future use +- endpoint `GET /keys` get all (or per API) key hashes. You can disable this endpoint by using the new `tyk.conf` setting `enable_hashed_keys_listing` (set to false by default) +- endpoint `GET /keys/{keyName}` was modified to be able to get a key by hash. You just need provide the key hash as a `keyName` +and call it with the new optional query parameter `hashed=true`. So the new format is `GET /keys/{keyName}?hashed=true"` +- also, we already have the same optional parameter for endpoint `DELETE /keys/{keyName}?hashed=true` + +#### JSON schema validation + +You can now use Tyk to verify user requests against a specified JSON schema and check that the data sent to your API by a consumer is in the right format. This means you can offload data validation from your application to us. + +If it's not in the right format, then the request will be rejected. And even better, the response will be a meaningful error rather than just a 'computer says no'. + +Schema validation is implemented as for the rest of our plugins, and its configuration should be added to `extended_paths` in the following format: +``` expandable +"validate_json": [{ + "method": "POST", + "path": "me", + "schema": {..schema..}, // JSON object + "error_response_code": 422 // 422 default however can override. +}] +``` + +The schema must be a draft v4 JSON Schema spec, see http://json-schema.org/specification-links.html#draft-4 for details. An example schema can look like this: +``` expandable +{ + "title": "Person", + "type": "object", + "properties": { + "firstName": { + "type": "string" + }, + "lastName": { + "type": "string" + }, + "age": { + "description": "Age in years", + "type": "integer", + "minimum": 0 + } + }, + "required": ["firstName", "lastName"] +} +``` + +#### New endpoint to get list of tokens generated for provided OAuth-client + +`GET /oauth/clients/{apiID}/{oauthClientId}/tokens` + +This endpoint allows you to retrieve a list of all current tokens and their expiry date issued for a provided API ID and OAuth-client ID in the following format. New endpoint will work only for newly created tokens: +``` expandable +[ + { + "code": "5a7d110be6355b0c071cc339327563cb45174ae387f52f87a80d2496", + "expires": 1518158407 + }, + { + "code": "5a7d110be6355b0c071cc33988884222b0cf436eba7979c6c51d6dbd", + "expires": 1518158594 + }, + { + "code": "5a7d110be6355b0c071cc33990bac8b5261041c5a7d585bff291fec4", + "expires": 1518158638 + }, + { + "code": "5a7d110be6355b0c071cc339a66afe75521f49388065a106ef45af54", + "expires": 1518159792 + } +] +``` + +You can control how long you want to store expired tokens in this list using `oauth_token_expired_retain_period ` which specifies the retain period for expired tokens stored in Redis. The value is in seconds, and the default value is `0`. Using the default value means expired tokens are never removed from Redis. + +#### Creating OAuth clients with access to multiple APIs + +When creating a client using `POST /oauth/clients/create`, the `api_id` is now optional - these changes make the endpoint more generic. If you provide the `api_id` it works the same as in previous releases. If you don't provide the `api_id` the request uses policy access rights and enumerates APIs from their setting in the newly created OAuth-client. + +At the moment this changes not reflected on Dashboard UI yet, as we going to do major OAuth improvements in 2.7 + +#### Certificate public key pinning + +Certificate pinning is a feature which allows you to allow public keys used to generate certificates, so you will be protected in case an upstream certificate is compromised. + +Using Tyk you can allow one or multiple public keys per domain. Wildcard domains are also supported. + +Public keys are stored inside the Tyk certificate storage, so you can use Certificate API to manage them. + +You can define them globally, from the Tyk Gateway configuration file using the `security.pinned_public_keys` option, or via an API definition `pinned_public_keys` field, using the following format: +``` +{ + "example.com": "", + "foo.com": "/path/to/pub.pem", + "*.wild.com": "," +} +``` + +For `key-id` you should set the ID returned after you upload the public key using the Certificate API. Additionally, you can just set path to public key, located on your server. You can specify multiple public keys by separating their IDs by a comma. + +Note that only public keys in PEM format are supported. + +If public keys are not provided by your upstream, you can extract them +by yourself using the following command: +> openssl s_client -connect the.host.name:443 | openssl x509 -pubkey -noout + +If you already have a certificate, and just need to get its public key, you can do it using the following command: +> openssl x509 -pubkey -noout -in cert.pem + +**Note:** Upstream certificates now also have wildcard domain support + +#### JQ transformations (experimental support) + +> This feature is experimental and can be used only if you compile Tyk yourself own using `jq` tag: `go build --tags 'jq'` + +If you work with JSON you are probably aware of the popular `jq` command line JSON processor. For more details, see here https://stedolan.github.io/jq/ + +Now you can use the full power of its queries and transformations to transform requests, responses, headers and even context variables. + +We have added two new plugins: + +* `transform_jq` - for request transforms. +* `transform_jq_response` - for response transforms + +Both have the same structure, similar to the rest of our plugins: +`{ "path": "", "method": "", "filter": "" }` + +#### Request Transforms +Inside a request transform you can use following variables: +* `.body` - your current request body +* `._tyk_context` - Tyk context variables. You can use it to access request headers as well. + +Your JQ request transform should return an object in the following format: +`{ "body": , "rewrite_headers": , "tyk_context": }`. + +`body` is required, while `rewrite_headers` and `tyk_context` are optional. + + +#### Response Transforms +Inside a response transform you can use following variables: +* `.body` - your current response body +* `._tyk_context` - Tyk context variables. You can use it to access request headers as well. +* `._tyk_response_headers` - Access to response headers + +Your JQ response transform should return an object in the following format: +`{ "body": , "rewrite_headers": }`. + +`body` is required, while `rewrite_headers` is optional. + +#### Example +``` expandable +"extended_paths": { + "transform_jq": [{ + "path": "/post", + "method": "POST", + "filter": "{\"body\": (.body + {\"TRANSFORMED-REQUEST-BY-JQ\": true, path: ._tyk_context.path, user_agent: ._tyk_context.headers_User_Agent}), \"rewrite_headers\": {\"X-added-rewrite-headers\": \"test\"}, \"tyk_context\": {\"m2m_origin\": \"CSE3219/C9886\", \"deviceid\": .body.DEVICEID}}" + }], + "transform_jq_response": [{ + "path": "/post", + "method": "POST", + "filter": "{\"body\": (.body + {\"TRANSFORMED-RESPONSE-BY-JQ\": true, \"HEADERS-OF-RESPONSE\": ._tyk_response_headers}), \"rewrite_headers\": {\"JQ-Response-header\": .body.origin}}" + }] +} +``` + + +### Tyk Dashboard v1.6.0 + +#### API categories + +You can apply multiple categories to an API definition, and then filter by these categories on the API list page. + +They might refer to the APIs general focus: 'weather', 'share prices'; geographic location 'APAC', 'EMEA'; or technical markers 'Dev', 'Test'. It's completely up to you. + +From an API perspective, categories are stored inside API definition `name` field like this: "Api name #category1 #category2", e.g. categories just appended to the end of the name. + +Added new API `/api/apis/categories` to return list of all categories and belonging APIs. + +#### Raw API Definition mode + +Now you can directly edit a raw API definition JSON object directly from the API Designer, by selecting either the **Raw API Definition** or the **API Designer** at the top of the API Designer screen. + +Raw or Designer + +This feature comes especially handy if you need copy paste parts of one API to another, or if you need to access fields not yet exposed to the Dashboard UI. + +#### Certificate public key pinning + +You can configure certificate pinning on the **Advanced** tab of the API Designer, using a similar method to how you specify upstream client certificates. + +Certificate Pinning + +#### JSON schema validation + +Reflecting the Tyk Gateway changes, on the Dashboard we have added a new **Validate JSON** plugin, which you can specify per URL, and can set both a schema, and custom error code, if needed. + +#### Improved key hashing support + +The Tyk Dashboard API reflects changes made in the v2.6.0 Gateway API, and now supports more operations with key by hash (when we have set `"hash_keys":` to ` true` in `tyk_analytics.conf`): + +- endpoint `POST /keys/` also returns a new field `key_hash` per each key in the list +- endpoint `GET /apis/{apiId}/keys/{keyId}` supports query string parameter `hashed=true` to get the key info via hash +- endpoint `GET /apis/{apiId}/keys` returns keys hashes +- endpoint `DELETE /apis/{apiId}/keys?hashed=true` can delete a key by its hash, but its functionality is disabled by default, unless you set `enable_delete_key_by_hash` boolean option inside the Dashboard configuration file. + + +#### Key requests management API now supports OAuth + +For this release we've improved our developer portal APIs to fully support an OAuth2.0 based workflow. Developers using your API will now be able to register OAuth clients and manage them. + +This change is not yet supported by our built-in portal, but if you are using custom developer portals, you can start using this new functionality right away. Full UI support for built-in portal will be shipped with our next 2.7 release. + +Developers can request access to an API protected with OAuth and get OAuth client credentials. + +The endpoint `POST /api/portal/requests` now has an optional `"oauth_info"` field which identifies the OAuth key request. + +Example of the OAuth key request: +``` expandable +{ + "by_user": "5a3b2e7798b28f03a4b7b3f0", + "date_created": "2018-01-15T04:49:20.992-04:00", + "for_plan": "5a52dfce1c3b4802c10053c8", + "version": "v2", + "oauth_info": { + "redirect_uri": "http://new1.com,http://new2.com" + } +} +``` + +Where: + +- `"by_user"` - contains the ID of portal developer who is requesting OAuth access +- `"for_plan"` - subscription ID +- `"version"` - is expected to have the value `"v2"` +- `"oauth_info"` - simple structure which contains a field with comma-separated list of redirect URI for OAuth flow + +A new field `"oauth_info"` will be present in replies for endpoints `GET /api/portal/requests/{id}` and `GET /api/portal/requests` + +When this kind of OAuth key request gets approved when using endpoint `PUT /api/portal/requests/approve/{id}` +a new OAuth-client is generated for a developer specified in the specified `"by_user"` field. + +Example of OAuth key request approval reply: +``` expandable +{ + "client_id": "203defa5162b42708c6bcafcfa28c9fb", + "secret": "YjUxZDJjNmYtMzgwMy00YzllLWI2YzctYTUxODQ4ODYwNWQw", + "policy_id": "5a52dfce1c3b4802c10053c8", + "redirect_uri": "http://new1.com,http://new2.com" +} +``` + +Where: + +- `"client_id"` and `"secret"` are OAuth-client credentials used to request the get token (they are to be kept in secret) +- `"policy_id"` - the subscription this OAuth-client provides access to +- `"redirect_uri"` - with comma-separated list of redirect URI for OAuth flow + +Also, if you set email notifications in your portal, an email with the OAuth-client credentials will be sent to the developer +who made that OAuth key request. + +There is also a change in the reply from the `GET /api/portal/developers` endpoint.The developer object will have new field - +`"oauth_clients"` which will contain a mapping of subscription IDs to the list of OAuth clients that the developer requested and +was approved, i.e.: +``` expandable +"oauth_clients": { + "5a52dfce1c3b4802c10053c8": [ + { + "client_id": "203defa5162b42708c6bcafcfa28c9fb", + "redirect_uri": "http://new1.com,http://new2.com", + "secret": "YjUxZDJjNmYtMzgwMy00YzllLWI2YzctYTUxODQ4ODYwNWQw" + } + ] +}, +``` + +#### New endpoints to get tokens per OAuth client + +These endpoints allow you to get a list of all current tokens issued for provided OAuth client ID: + +- `GET /apis/oauth/{apiId}/{oauthClientId}/tokens` +- `GET /apis/oauth/{oauthClientId}/tokens` when the API ID is unknown or OAuth-client provides access to several APIs + + +#### Renamed the response `_id` field to `id` in List Key Requests + +We have renamed the response `_id` field when retrieving a list of key requests to `id`. + +See [List Key Requests](/tyk-apis/tyk-dashboard-api/manage-key-requests#list-key-requests) for more details. + + +#### Developers can request a password reset email + +If a developer forgets their password, they can now request a password reset email from the Developer Portal Login screen. + +Request email reset + +See [Developer Profiles](/tyk-developer-portal/tyk-portal-classic/developer-profiles#reset-developer-password) for more details. + +#### SSO API custom email support + +Now you can set email address for users logging though the Dashboard SSO API, by adding an "Email" field to the JSON payload which you sent to `/admin/sso` endpoint. For example: +``` expandable +POST /admin/sso HTTP/1.1 +Host: localhost:3000 +admin-auth: 12345 + +{ + "ForSection": "dashboard", + "Email": "user@example.com", + "OrgID": "588b4f0bb275ff0001cc7471" +} +``` + +#### Set Catalog settings for each individual API + +Now you can override the global catalog settings and specify settings per catalog. +The Catalog object now has `config` field, with exactly same structure as Portal Config, except new `override` boolean field. +If set, Catalog settings will override global ones. + +At the moment the following options can be overriden: `Key request fields`, `Require key approval` and `Redirect on key request` (with `Redirect to` option as well). + +#### Blocklist IP Support + +Tyk allows you to block IP Addresses, which is located in the **Advanced Options** tab in the **Endpoint Designer**. + +Blocklist Support + +### Tyk Identity Broker v0.4.0 + +With this release TIB joins the Tyk product line as a first class citizen and is now distributed via packages and [Docker image](https://hub.docker.com/r/tykio/tyk-identity-broker/). + +#### Support for SSO API email field +If IDP provides a user email, it should be passed to the Dashboard SSO API, and you should see it in the Dashboard UI. + +#### Improved support for local IDPs +If you run a local IDP, like Ping, with an untrusted SSL certificate, you can now turn off SSL verification by setting `SSLInsecureSkipVerify` to `true` in the TIB configuration file. + +#### Added Redis TLS support +To enable set `BackEnd.UseSSL` and, optionally, `BackEnd.SSLInsecureSkipVerify`. + +### Tyk Pump v0.5.2 + +#### Redis TLS support +Added new `redis_use_ssl` and `redis_ssl_insecure_skip_verify` options. + + +### Redis TLS support + +Many Redis hosting providers now support TLS and we're pleased to confirm that we do too. + +Whether it's the open source API Gateway, or Dashboard, Pump, Sink and Tyk Identity Broker (TIB): you can now make secure connections to Redis from all Tyk products, as long as your provider allows it. + +### MDCB v1.5.3 + +#### Redis TLS support +Added new `redis_use_ssl` and `redis_ssl_insecure_skip_verify` options. + +### Upgrading all new Components + +For details on upgrading all Tyk versions, see [Upgrading Tyk](https://tyk.io/docs/upgrading-tyk/). + +### Don't Have Tyk Yet? + +Get started now, for free, or contact us with any questions. + +* [Get Started](https://tyk.io/pricing/compare-api-management-platforms/#get-started) +* [Contact Us](https://tyk.io/about/contact/) + +## 2.5.0 Release Notes + +This release touches all our products and brings you numerous features and fixes. Here are the packages and their versions we are releasing today: Tyk Gateway v2.5.0, Tyk Dashboard v1.5.0, Tyk Pump v0.6.0, MDCB v1.5.0, TIB v0.3. + + +### Major Highlights + +#### New Dashboard Look and Feel + +Our Dashboard has had a UI overhaul, with the following improvements: + +* A more modern, fun look and feel +* Consistent layouts and action buttons across each section +* Better feedback on errors and updates +* Various UX improvements + +#### SSO with OpenId Identity Providers + +With TIB v0.3 we have made it possible to integrate any OpenID supported Identity provider with Tyk so you can configure Single Sign On (SSO), if the provider supports those. + +#### Searching API and Policies List + +This long awaited feature has been added on the Dashboard UI. + +#### Default API Versioning + +You can now specify a default API version when using a versioning strategy. + +#### Tyk Pump with MDCB + +We've added MDCB support in this release of Tyk Pump + +### Tyk Gateway v2.5.0 + +#### New Relic Instrumentation Support + +We have added support for New Relic Instrumentation using: + +`"newrelic": {"app_name": "", "license_key": ""}` + +[Docs](/api-management/logs-metrics#statsd-instrumentation) + +#### Default API Versioning + +You can now specify a default API version, and it will be used if a version is not set via headers, or URL parameters. Use the new option: + +`spec.version_data.default_version` + +[Docs](/api-management/gateway-config-tyk-classic#tyk-classic-api-versioning) + +#### Disable URL Encoding + +You can disable URL encoding using a new boolean `http_server_options` setting: + +`skip_target_path_escaping` + +[Docs](/tyk-oss-gateway/configuration#http_server_options) + +#### Enable Key Logging + +By default all key ids in logs are hidden. You can now turn it on if you want to see them for debugging reasons using the `enable_key_logging` option. + +[Docs](/tyk-oss-gateway/configuration#enable_key_logging) + + +#### Specify TLS Cipher Suites + +We have added support for specifying allowed SSL ciphers using the following option: + +`http_server_options - ssl_ciphers` + +[Docs](/api-management/certificates) + +### Plugins Updates + +* Coprocess plugins now have access to `config_data` +* The JSVM `spec` object now has access to `APIID` and `OriginID` to reflect similar functionality of Coprocess plugins. +* Plugins now have access to Host HTTP Header. + +[JSVM Docs](/api-management/plugins/javascript#using-javascript-with-tyk) +[Plugin Data Structure Docs](/api-management/plugins/rich-plugins#rich-plugins-data-structures) + + +### Tyk Dashboard v1.5.0 + +#### A Fresh Look and Feel + +With this release we have refreshed the entire Dashboard UI with a new look-and-feel, bringing with it such improvements as: + +* A more modern, fun look and feel +* Consistent layouts and action buttons across each section +* Better feedback on errors and updates +* UX improvements + +#### Search on API and Policy List Pages + +We have added API and Policy search functionality, which should help those with long lists. + +* [API Docs](/api-management/dashboard-configuration#manage-apis-api-definition) +* [Policy Docs](/tyk-apis/tyk-dashboard-api/portal-policies) + +#### A New, Interactive Getting Started Walkthrough + +We have swapped out the old Getting started tutorial and added a new interactive one, which should make it easier for new users to get started with the Dashboard UI. + +#### Advanced URL Rewrites + +We have extended the URL Rewrite plugin functionality by enabling users to create more advanced rewrite rules based on Header matches, Query string variable/value matches, Path part matches, (i.e. components of the path itself), Session metadata values, and Payload matches. + +[Docs](/transform-traffic/url-rewriting#url-rewrite-middleware) + +#### Portal Session Lifetime + +You can now control the portal session lifetime using the `portal_session_lifetime` config variable. + +[Docs](https://tyk.io/docs/configure/tyk-dashboard-configuration-options/) + +#### Configure Port for WebSockets + +We have added `notifications_listen_port` option to configure the port used by WebSockets for real-time notifications. + +[Docs](/tyk-oss-gateway/configuration) + +#### Slug + +Once set, the API slug will no longer be overridden when the API title is changed. + +#### Custom Domain + +We have fixed the API URL if a custom domain is set. + + +### Tyk Pump v0.5.0 + +#### Splunk Support + +We added support for forwarding analytics data to Splunk. A sample configuration is: + +``` expandable +"pumps": { + "splunk": { + "name": "splunk", + "meta": { + "collector_token": "", + "collector_url": "https://.cloud.splunk.com: + 8088", + "ssl_insecure_skip_verify": true + } + } +}, +``` + +#### Analytics Collection Capping + +Detailed analytics collection capping is now enabled by default and configurable via the `collection_cap_enable` and `collection_cap_max_size_bytes` options. + + + +### MDCB v1.5.0 + +We've introduced long awaited support for using Tyk Pump in conjunction with MDCB to use any of services supported by Tyk Pump, like ElasticSearch, Splunk and etc. This works by setting `forward_analytics_to_pump` to true, which disables analytics processing by MDCB itself, and enables the forwarding of all data to Tyk Pump running inside your management environment. + + +### TIB v0.3 + +With this release, you now can use any OpenID Connect compatible provider with TIB. This means that you can use almost any Identity management solution, supporting OpenID, like Okta, Ping or Keycloak. + +Use `SocialProvider` with the following options: + +``` expandable +"UseProviders": [{ + "Name": "openid-connect", + "Key": "CLIENT-KEY", + "Secret": "CLIENT-SECRET", + "DiscoverURL": "https:///.well-known/openidconfiguration" +}] +``` + +### Packaging changes across all products + +New deb and rpm packages add the "tyk" user and group so that package files and directories would be owned by it and the process run with its effective uid and gid. In addition to this gateway PID now has to reside in its own sub-rundir due to this change, so that's created (and additionally managed by systemd where it's available), default pidfile location changed appropriately so that upgrade wouldn't require any config changes by the users. The gateway config file is now only readable and writable by the "tyk" user and group. This change is applied across all our products except Gateway: its changes scheduled to 2.6. + +The "default" init system files are not removed on upgrade/remove anymore so that it's now a way for users to run the respective process with custom environment variables. + +The bug with removal of init system files on upgrade in rpm-based systems is now fixed. + + +### Upgrading all new Components + +For details on upgrading all Tyk versions, see [Upgrading Tyk](https://tyk.io/docs/upgrading-tyk/). + +### Don't Have Tyk Yet? + +Get started now, for free, or contact us with any questions. + +* [Get Started](https://tyk.io/pricing/compare-api-management-platforms/#get-started) +* [Contact Us](https://tyk.io/about/contact/) + +## 2.4.0 Release Notes + +This release touch all our products and brings you numerous long awaited features and fixes. +Here are the packages and their versions we are releasing today: Tyk Gateway v2.4.0, Tyk Dashboard v1.4.0, Tyk Pump v0.4.2, MDCB v1.4.0, TIB v0.2. + +### Major highlights + +#### Mutual TLS + +A major feature of this release is the implementation of Mutual TLS. Now you can protect your APIs by allow listing certificates, idenitfy users based on them, and increase security between Tyk and upstream API. For details, see [Mutual TLS](/basic-config-and-security/security/mutual-tls/client-mtls#why-use-mutual-tls). + + +#### Extended use of Multiple Policies + +We have extended support for partitioned policies, and you can now mix them up when creating a key. Each policy should have own partition, and will not intersect, to avoid conflicts while merging their rules. + +Using this approach could be useful when you have lot of APIs and multiple subscription options. Before, you had to create a separate policy per API and subscription option. + +Using multiple partitioned policies you can create basic building blocks separately for accessing rules, rate limits and policies, and then mix them for the key, to creating unique combination that fit your needs. + +We have added a new `apply_policies` field to the Key definition, which is an string array of Policy IDs. +> **NOTE**: The old key apply_policy_id is supported, but is now deprecated. + +We have updated the Dashboard **Apply Policies** section of the **Add Key** section. + +apply-policy + +For this release multiple policies are only supported only via the Add Key section and via the API. Support for OIDC, oAuth, and Portal API Catalogs are planned for subsequent releases. + +[Docs](/api-management/policies#partitioned-policies) + +#### Global API Rate Limits + +We have added a new API definition field `global_rate_limit` which specifies a global API rate limit in the following format: `{"rate": 10, "per": 1}`, similar to policies or keys. + +The API rate limit is an aggregate value across all users, which works in parallel with user rate limits, but has higher priority. + +Extended Dashboard API designer Rate Limiting and Quotas section in Core settings: + +rate-limits + +[Docs](/api-management/policies#partitioned-policies) + +#### Specify custom analytics tags using HTTP headers + +We have added a new API definition field `tag_headers` which specifies a string array of HTTP headers which can be extracted and turned to tags. + +For example if you include `X-Request-ID` header to tag_headers, for each incoming request it will include a `x-request-id-` tag to request an analytic record. + +This functionality can be useful if you need to pass additional information from the request to the analytics, without enabling detailed logging, which records the full request and response objects. + +We have added a new **Tag headers** section to the Dashboard **API Designer Advanced** tab. + +tag_headers + +[Docs](/api-management/dashboard-configuration#activity-logs) + +#### Single-Sign-On (SSO) improvements + +More SSO functionality is something that a lot of our customers have been asking for. In this release we've significantly improved our support for SSO, and you can now: + +* Enable Tyk Identity Broker to apply LDAP filters to user search [Docs](/api-management/external-service-integration#advance-ldap-configuration) +* Set permissions for your users, logged via SSO, via `sso_permission_defaults` in Dashboard config file. [Docs](/api-management/external-service-integration) +* Setup a login page redirect, using `sso_custom_login_url` and `sso_custom_portal_login_url` Dashboard config options to enable users login using a custom SSO login page. [Docs](/api-management/external-service-integration) +* For those who love to build everything in-house, we have added new API for custom dashboard authentication integrations. [Docs](/api-management/external-service-integration#custom-proxy-identify-provider) + + + +### Tyk Gateway v2.4.0 + +#### Mutual TLS support +[Docs](/basic-config-and-security/security/mutual-tls/client-mtls#why-use-mutual-tls) + +#### Global API rate limits +[Docs](/api-management/rate-limit#rate-limiting-layers) + +#### Specify custom analytics tags using HTTP headers +[Docs](/api-management/dashboard-configuration#activity-logs) + +#### Attaching Multiple Policies to the Keys +[Docs](/api-management/policies#partitioned-policies) + +#### Default User Agent set to Tyk/$VERSION +If no user agent is specified in a request, it is now set as `Tyk/$VERSION`. + +#### Include `x-tyk-api-expires` date header for versioned APIs +If a request is made for an API which has an expiry date, the response will include the `x-tyk-api-expires` header with expiry date. + +[Docs](/api-management/gateway-config-tyk-classic#tyk-classic-api-versioning) + +#### Run Admin Control API on a separate port +Using `control_api_port` option in configuration file, you can run the admin control api on a separate port, and hide it behind firewall if needed. + +[Docs](/tyk-oss-gateway/configuration#control_api_port) + +#### Added a Configuration Linter + +We have added a new `tyk lint ` command which will validate your `tyk.conf` file and validate it for syntax correctness, misspelled attribute names or format of values. The Syntax can be: + +`tyk lint` or `tyk --conf=path lint` + +If `--conf` is not used, the first of the following paths to exist is used: + +`./tyk.conf` +`/etc/tyk/tyk.conf` + +[Docs](/tyk-oss-gateway/configuration) + +#### Set log_level from tyk.conf + +We have added a new `log_level` configuration variable to `tyk.conf` to control logging level. + +Possible values are: `debug`, `info`, `warn`, `error` + +[Docs](/tyk-oss-gateway/configuration#log_level) + +#### Added jsonMarshal to body transform templates + +We have added the `jsonMarshal` helper to the body transform templates. You can apply jsonMarshal on a string in order to perform JSON style character escaping, and on complex objects to serialise them to a JSON string. + +Example: `{{ .myField | jsonMarshal }}` + +[Docs](/api-management/traffic-transformation/request-body) + +#### Added a blocking reload endpoint + +Now you can add a `?block=true` argument to the `/tyk/reload` API endpoint, which will block a response, until the reload is performed. This can be useful in scripting environments like CI/CD workflows. + +[Docs](/tyk-gateway-api) + +#### `tyk_js_path` file now contains only user code + +Internal JS API not budled into tyk binary, and `js/tyk.js` file used only for custom user code. It is recommended to delete this file, if you are not using it, or remove Tyk internal code from it. New releases do not ship this file by default. + +#### Improved Swagger API import defaults + +When importing Swagger based APIs they now generate tracked URLs instead of allow listed ones. + +[More](https://github.com/TykTechnologies/tyk/issues/643) + +#### Respond with 503 if all hosts are down. +Previously, the internal load balancer was cycling though hosts even if they were known as down. + +#### Request with OPTIONS method should not be cached. +[More](https://github.com/TykTechnologies/tyk/issues/376) + +#### Health check API is officially deprecated. +This was very resource consuming and unstable feature. We recommend using load balancers of your choice for this. + +#### Fixed custom error templates for authentication errors. +[More](https://github.com/TykTechnologies/tyk/issues/438) + + +### Tyk Dashboard v1.4.0 + +#### Mutual TLS support +[Docs](/basic-config-and-security/security/mutual-tls/client-mtls#why-use-mutual-tls) + +#### Global API rate limits +[Docs](/api-management/rate-limit#rate-limiting-layers) + +#### Specify custom analytics tags using HTTP headers +[Docs](/api-management/dashboard-configuration#activity-logs) + +#### Attaching Multiple Policies to the Keys +[Docs](/api-management/policies#partitioned-policies) + +#### Set permissions for users logged via SSO (Tyk Identity Broker) +Added new option `sso_permission_defaults` in Dashboard config file. +Example: + +``` expandable +"sso_permission_defaults": { + "analytics": "read", + "apis": "write", + "hooks": "write", + "idm": "write", + "keys": "write", + "policy": "write", + "portal": "write", + "system": "write", + "users": "write" +}, +``` +[Docs](/api-management/external-service-integration) + +#### Set custom login pages for portal and dashboard +If you are using 3-rd party authentication like TIB, you maybe want to redirect from standard login pages to your own using following attributes in dashboard config: `sso_custom_login_url`, `sso_custom_portal_login_url`. + +[Docs](/api-management/external-service-integration) + +#### Added new set of APIs for custom dashboard authentication +Added new `/admin/sso` endpoint for custom integration. In fact, the same API is used by our own Tyk Identity Broker. + +[Docs](/api-management/external-service-integration#custom-proxy-identify-provider) + + +#### Service discovery form improved with most common pre-defined templates + +Now you can pre-fill the form with most popular templates like consul or etcd. + +#### RPC credentials renamed to Organization ID +Yay! + +#### Replaced text areas with a code editors + +All multi-line text fields now replaced with a code editors. + +#### Replace dropdowns with the live search component + +All the dropdown lists now support live search, and work with a large number of elements (especially handy for API or Policiy lists). + +#### Display user ID and email on when listing users + +The **Users list** now displays the **User ID** and **Email**. + +#### Added search for portal developers + +We have added search for the users listed in the developer portal. + +#### Key request email link to developer details + +The email address in a **Key Request** from the **Developer Portal** is now a link to the relevant developer profile. + +#### Country code in log browser links to geo report + +The country code in the log browser has been changed to a link to the geographic report. + +#### Added support for HEAD methods in the Dashboard API Designer. + +#### Redirect user to the login page if session is timed out. + +#### When creating a portal API catalog, you can now attach documentation without saving the catalog first. + +#### Fixed the` proxy.preserve_host_header` field when saved via the UI. +Previously, the field was available in the API definition, but got removed if the API was saved via the UI. + +#### Prevent an admin user revoking their own permissions. +This is a UI only fix, it is still allowable via the API (which is OK). + +#### Other UX Improvements + +* Key pieces of data made accessible to quickly copy+paste +* Improved help tips +* Get your API URL without having to save and go back +* Improved pagination +* Improved feedback messaging +* Improved charts +* Improved analytics search + +### Tyk Pump v0.4.2 + +#### Support added for Mongo SSL connections + +See https://tyk.io/docs/configure/tyk-pump-configuration/ for a sample pump.conf file. + +### MDCB v1.4.0 +Added support for Mutual TLS, mentioned by Gateway and Dashboard above. See [Docs](/basic-config-and-security/security/mutual-tls/client-mtls#why-use-mutual-tls) + +Also fixed bug when Mongo connections became growing though the roof if client with wrong credentials tries to connect. + + +### TIB v0.2 + +Tyk Identity Broker now fully support LDAP search with complex filters! [Docs](/api-management/external-service-integration#advance-ldap-configuration) + +### Upgrading all new Components + +> **NOTE**: This release is fully compatible with the previous version, except that if you want to use new features, like Mutual TLS, you need to upgrade all the related components. + +Cloud users will be automatically upgraded to the new release. + +Hybrid users should follow the upgrade instructions [here](/developer-support/upgrading#tyk-upgrade-guides-for-different-deployment-models). + +Self-Managed users can download the new release packages from their usual repositories. + +[3]: /img/release-notes/tag_headers.png +[4]: /img/release-notes/import_api_definition.png +[5]: /img/release-notes/live_search.png +[6]: /img/release-notes/user_list.png +[7]: /img/release-notes/dev_list.png +[8]: /img/release-notes/key_request_user.png + +## Upgrading to v2.3 from v2.2 + +Tyk v2.3 is backwards-compatible with v2.2 in terms of the configuration file and the original `tyk.conf` can be used with the new version. If you would like to keep your v2.2 settings, please remember to **backup your `tyk.conf` file before upgrading as it will be overwritten during the upgrade process.** + +*However*, there are behavioral differences in a v2.3 cluster when hooked up to a Dashboard that can cause some odd behavior if the upgrade is not conducted in the right order. + +Tyk v2.3 Gateways continuously talk to each other sharing load data, they also share information with the Dashboard regarding their current configuration. This chatter, if exposed to a v2.2 Gateway, can cause it go into a reload loop, which isn't ideal. Because of this, the recommended upgrade procedure for a Tyk v2.2 system is: + +1. Upgrade all the Tyk Gateways to v2.3 +2. Upgrade the Dashboard to v1.3 +3. Update the Tyk Pump to v0.4 + +If upgraded in this order, then the reload loop can be avoided on a production system. + +If the reload loop does occur it is not disastrous, Tyk will just keep proxying traffic even though it is constantly pulling new configurations. It's just not particularly efficient. + +> **Note for MDCB**: If you are using MDCB and want to upgrade your Gateways to v2.3, you will also need to upgrade your MDCB to v1.2.0.2. + +#### Retaining rate limiter functionality + +Tyk v2.3 introduces a new in-memory leaky-bucket *distributed* rate limiter, this is much more performant than the older rate limiter which hard-synchronised via Redis, and puts far less strain on a Redis instance or cluster than the old rate limiter. By default, Tyk v2.3 will switch to this rate limiter, however it is possible to retain the old behavior by enabling it explicitly in the `tyk.conf` file: + +``` + "enable_redis_rolling_limiter": true +``` + +This might be useful if you do not wish to switch over immediately and wish to test the new rate limiter first. + +#### Public and Private keys + +Tyk v2.3 introduces public/private key message authentication for messages that are sent from the management interface to the Gateways, and for code that is being deployed as a plugin to a Gateway via the bundle downloader. + +By default, Tyk's new config file has this feature *disabled*, however since it is new, an existing `tyk.conf` will assume a secure installation as the feature must be explicitly disabled. This means, prior to starting your new Gateways, either disable the security feature, or add a public/private key pair to your `tyk.conf` and `tyk_analytics.conf` files: + +##### Disable secure messages + +**If you are upgrading from v2.2 then you must either generate a public/private keypair or disable the option to validate inbound payloads against a key. You can do this by setting the following key in your `tyk.conf`:** + +``` + "allow_insecure_configs": true +``` + +##### Add a public key and private key pair + +First, generate the key pair: + +``` + # private key + openssl genrsa -out privkey.pem 2048 + + # public key + openssl rsa -in privkey.pem -pubout -out pubkey.pem +``` + +**`tyk.conf`:** + +``` + "public_key_path": "/path/to/public/key.pem" +``` + +**`tyk_analytics.conf`:** + +``` + "private_key_path": "/path/to/private/key.pem" +``` + +#### Conclusion + +The above are the key changes in v2.3 that could affect your setup and configuration during an upgrade. All other settings should be backwards compatible and not introduce breaking changes. + + diff --git a/developer-support/release-notes/cloud.mdx b/developer-support/release-notes/cloud.mdx new file mode 100644 index 00000000..2c264e61 --- /dev/null +++ b/developer-support/release-notes/cloud.mdx @@ -0,0 +1,563 @@ +--- +title: "Tyk Cloud Release Notes" +'og:description': "Release notes documenting updates, enhancements, and changes for Tyk Cloud" +tags: ['Tyk Cloud', 'Release notes', 'v1.23', '1.23.0', 'v1.24', '1.24.0', 'v1.25', '1.25.0', 'v1.26', '1.26.0', '1.27.0', '1.28.0', '1.28.1', 'changelog'] +sidebarTitle: "Tyk Cloud" +--- + +## 1.28.1 Release Notes + +### Release Date 03 June 2025 + +### Release Highlights + +This release improves Tyk Cloud’s security, scalability, and reliability. We’ve hardened the signup and login flows, improved sensitive data handling in API responses, and enhanced Redis scalability in Control Plane deployments to support larger workloads. + +For a full list of changes, see the detailed [changelog](#Changelog-v1.28.1) below. + +### Breaking Changes + +There are no breaking changes in this release. + +### Downloads +- [latest version of Mserv](https://github.com/TykTechnologies/mserv/releases/latest) + +### Deprecations + +There are no deprecations in this release. + +

Changelog

+#### Added + +
    +
  • + +The change brings more flexible Redis scaling capabilities in the Control Plane deployments. Control Planes are now able to handle more/larger API keys in storage without causing stability issues. Tyk Cloud will expand this storage using this capability when needed, but a permanent arrangement is subject to commercial terms. + +
    • +
+ +#### Fixed + +
    +
  • + + +Improved the signup logic to prevent unintended automation and ensure tighter control over account creation, enhancing platform security and reliability. + + + +Standardized response status codes in the login process to prevent discrepancies that could lead to information exposure through http status code. + + + +Resolved an issue where certain sensitive fields could be inadvertently included in API responses. The platform now enforces stricter data handling and output sanitization. + + +
    • +
+ +## 1.28.0 Release Notes + +### Release Date 26 May 2025 + +### Release Highlights + +This release focuses on further enhancing Tyk Cloud’s stability, security, and overall user experience. We've resolved several UI issues, improved input validation, standardized security defaults across versions, and strengthened core platform behavior. Additionally, we’ve introduced new 2025 pricing plans! + +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v1.28.0) below. + +### Breaking Changes + +There are no breaking changes in this release. + +### Downloads +- [latest version of Mserv](https://github.com/TykTechnologies/mserv/releases/latest) + +### Deprecations + +There are no deprecations in this release. + +

Changelog

+#### Added + +
    +
  • + +Tyk Cloud has introduced the 2025 base pricing plans, featuring new Core, Professional, and Enterprise tiers. These changes provide clearer value differentiation across plans while maintaining flexibility for both SaaS and Hybrid deployments. The "Account" section is now hidden for new customers, though existing customers remain unaffected. Usage and quota are still available in the "System Usage" and "Monitoring" sections. For more information, please check the [Pricing & Plans](https://tyk.io/pricing/) + +
    • +
+ +#### Fixed + +
    + +
  • + + +We’ve fixed an issue where pasting invalid characters, such as tabs and spaces, into the hybrid data plane name field would break the registration flow in Ara. The system now includes input validation on both the frontend and backend, ensuring only valid characters are accepted. This prevents configuration issues and improves the reliability of the hybrid data plane creation process. + + + +We’ve made backend improvements to the Tyk Cloud login functionality to enhance its security and resilience against automated attacks. These changes help ensure a more consistent and secure authentication experience. + + + +We’ve fixed an issue where secure configuration defaults for the Tyk Dashboard admin view were not properly applied in older versions (v5.3). These settings now default to true in v5.8+ as intended, while remaining false in v5.3 to preserve backward compatibility. This ensures consistent and expected security behavior across versions. + +**Note:** Changes in deployment configurations are applied during a redeployment, triggered by the user or system. In addition, defaults can be changed individually by creating a support ticket. + + +
    • + +
+ +## 1.27.0 Release Notes + +### Release Date 23 April 2025 + + +### Release Highlights + +This release focuses on improving Tyk Cloud's stability, security, and reliability. We’ve addressed several bugs, enhanced UI behavior, resolved performance bottlenecks, and implemented secure defaults to strengthen the platform’s security posture. + +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v1.27.0) below. + +### Breaking Changes + +There are no breaking changes in this release. + +### Downloads +- [latest version of Mserv](https://github.com/TykTechnologies/mserv/releases/latest) + +### Deprecations + +There are no deprecations in this release. + +

Changelog

+#### Added + +
    +
  • + + +Tyk Cloud now displays an informative error message when a Control Plane fails to deploy. Previously, users received no feedback, causing confusion. The new message lets users know the issue is being addressed and guides what to do next, improving transparency and user experience during deployment failures. + + + +Audit logging can now be enabled or disabled directly from the UI for Control Plane deployments if it is included in your plan. + + + +As announced in the previous release, Tyk Cloud now defaults to secure settings that restrict admin users from viewing or resetting other users' API tokens in the Dashboard. + + + +We’ve made backend improvements to enhance the performance and stability of hybrid node visibility features in Tyk Cloud. These changes help support larger datasets more efficiently, ensuring smoother operations and better scalability behind the scenes. + + +
    • + +
+ +#### Fixed + +
    +
  • + + +Fixed an issue where optional fields in the β€˜Edit New Relic Connection’ section of the Telemetry UI would disappear after saving. With this fix, all optional configuration fields now persist correctly, ensuring that user-defined telemetry settings are saved and visible for further updates. + + + +Fixed an issue where enabling telemetry export on an existing deployment did not apply the expected network policy changes. This fix ensures that configuration changes are consistently applied, so telemetry features work as intended without requiring manual intervention. + + + +We’ve resolved a UI issue where the team dropdown became unresponsive when switching between Control Plane and Hybrid Data Plane types during deployment creation. This fix ensures that users can seamlessly switch deployment types. + + + +We’ve removed unnecessary data from telemetry export configuration API responses to ensure cleaner and more secure payloads. + + +
    • + +
+ + +## 1.26.0 Release Notes + +### Release Date 17 of March 2025 + +### Release Highlights + +Tyk Cloud now provides greater compliance controls, allowing customers to manage audit logging and storage more effectively. With new audit log storage and the ability to enable or disable audit logging per Control Plane deployment, users can optimize costs while maintaining security and compliance. These improvements give organizations more flexibility in handling audit data based on their specific regulatory and operational needs. To enable this feature, please contact your account manager. + +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v1.26.0) below. + +#### Admin Behavior Update in Control Planes v5.8.0 +In the next Tyk Cloud release, starting with the Control Planes version 5.8.0, we will be restricting Tyk Dashboard admin users' ability to view and reset other users' API tokens. This change will automatically apply to all Control Plane deployments that are either newly created with or upgraded to version 5.8.0 or later. + +This security improvement helps protect sensitive access credentials and provides better isolation between user accounts. If your organization requires the previous behavior for specific operational needs, you can revert this change by submitting a request to Tyk Support. + +We remain committed to continuously improving the security of our platform while maintaining the flexibility needed for diverse environments. + +### Breaking Changes + +There are no breaking changes in this release. + +### Downloads +- [latest version of Mserv](https://github.com/TykTechnologies/mserv/releases/latest) + +### Deprecations + +There are no deprecations in this release. + +

Changelog

+#### Added + +
    +
  • + + +Tyk Cloud allows users to enable or disable audit logging for their [Control Plane](/api-management/mdcb#data-plane) (CP) deployments, providing greater flexibility in managing compliance and storage costs. To enable this feature, please contact your account manager. + +Tyk Cloud now also enforces audit log storage quotas based on contractual terms (a storage quota limit assigned to organizations based on their subscription), allowing customers to manage costs effectively. Similar to analytics storage, a size cap is applied to audit logs, dropping the oldest logs to fit the new ones within the quota. + +For more information, please check the [Tyk Dashboard documentation on the feature](https://tyk.io/docs/api-management/dashboard-configuration/#retrieving-audit-logs-via-api) + + + +Tyk Cloud now enables the onboarding wizard by default for trial users, providing a guided Quick Start experience in Tyk 5.8.0. + + + +Tyk Cloud now features enhanced stability for Control Plane deployments, even in cases of license server issues. + + +
    • + +
+ +#### Fixed + +
    +
  • + + +Tyk Cloud now moves MServ API definitions to a separate organization, preventing users from accidentally modifying or deleting them. Previously, these definitions were stored within the customer’s Tyk Dashboard deployment, posing the risk of breaking Go plugin functionality. With this update, Go plugins remain fully operational while deployments become more secure and error-proof. This change applies to new deployments only; existing deployments with plugins enabled will be gradually migrated in the future to avoid unexpected service disruption. + + + +Tyk Cloud now correctly populates first and last names when provisioning new users via SSO (Google SSO, KeyCloak). Previously, only the first name was set, causing validation errors when updating roles due to a missing last name. This fix ensures that SSO-provisioned users have complete profiles, preventing onboarding issues and improving role management for organizations + + + +Tyk Cloud now correctly enforces the 'Only Registered Users' flag for SSO (Google SSO, KeyCloak), preventing the creation of unregistered users via just-in-time provisioning. Previously, users without a corresponding local entry could still be created even when this setting was enabled. + + + +Tyk Cloud now correctly hides the custom domain field in the Enterprise Developer Portal (EDP) edit page when the custom domain entitlement is not enabled. Previously, the field was visible even for organizations without access to this feature, causing confusion. + + + +Tyk Cloud now ensures more reliable health checks for Control Plane deployments when SSO is configured. + + + +The monitoring chart in Tyk Cloud now correctly displays the subscription limit based on the organization’s entitlement. + + + +Changing the analytics storage quota in Tyk Cloud no longer removes critical database indices, which previously led to performance degradation. This fix ensures that indices are preserved, maintaining fast query performance and data integrity. Customers should now experience improved analytics performance, especially when adjusting storage quotas. + + +
    • + +
+ +--- + +## 1.25.0 Release Notes + +### Release Date 10 of February 2025 + +### Release Highlights +This Tyk Cloud update enhances Gateway version management, ensuring a more streamlined, secure, and user-friendly experience. With the new UI versioning updates, users have clearer visibility into supported versions, direct upgrade recommendations, and the ability to automate version upgrades for effortless maintenance. +These changes empower teams to stay on supported, secure, and high-performing versions. +For more details, check out the [documentation on Gateway versioning and auto-upgrades](/developer-support/release-types/long-term-support) + +### Breaking Changes + +There are no breaking changes in this release. + +### Downloads +- [latest version of Mserv](https://github.com/TykTechnologies/mserv/releases/latest) + +### Deprecations + +There are no deprecations in this release. + +

Changelog

+#### Added + +
    +
  • + +Tyk Cloud now supports custom domains for the API Manager Dashboard, allowing organizations to align their domain with their branding and provide a seamless experience for internal and external stakeholders. + +
    • + +
  • + +Tyk Cloud now provides improved visibility into supported LTS versions. Every patch version within the current LTS branch is labeled `Supported`, while the latest patch is marked as `Recommended`. This update eliminates confusion around deprecated versions and prevents unexpected `unsupported` status changes, allowing for smoother upgrade planning and system stability. + +
    • + +
+ +#### Changed + +
    +
  • + +We have redesigned the Overview page in Tyk Cloud to provide a more streamlined and actionable summary of an organization's key elements. +Users can now toggle between two views: +- **Task View**: Groups control planes and developer portals by high-level jobs to be done, such as managing APIs or publishing products. +- **Environment View**: Displays control planes and developer portals grouped by environment, making it easier to navigate large setups. +Additionally, we have moved organization statistics to a new System Usage page under Deployments. + +
    • +
+ +#### Fixed + +
    +
  • + +Team admins will no longer see the `Delete User` button when viewing the profiles of other team admins. Previously, this button was incorrectly displayed, leading to an error when clicked. This fix ensures the UI reflects the correct permissions and prevents confusion for administrators managing team members. + +
    • +
+ +--- + +## 1.24.0 Release Notes + +### 1.24.2 Release Notes + +#### Release Date 13 of January 2025 + +#### Release Highlights + +This Tyk Cloud update resolves an issue related to Telemetry export configurations. Previously, when deploying a Data Plane in a region different from the Control Plane, Telemetry export settings could encounter compatibility issues. With this patch, Telemetry export configuration now works seamlessly across regional deployments, ensuring consistent observability for distributed Tyk Cloud setups. + + +#### Breaking Changes +{/* Required. Use the following statement if there are no breaking changes, or explain if there are */} +There are no breaking changes in this release + +#### Downloads +- [latest version of Mserv](https://github.com/TykTechnologies/mserv/releases/latest) + +#### Deprecations +{/* Required. Use the following statement if there are no deprecations, or explain if there are */} +There are no deprecations in this release + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Fixed +{/* This section should be a bullet point list that describes the issues fixed in the release. For each fixed issue explain: +- What problem the issue caused +- How was the issue fixed +- Link to (new) documentation created as a result of a fix. For example, a new configuration parameter may have been introduced and documented for the fix +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} + +
    +
  • + +The Telemetry export configuration now functions as expected when a Data Plane is deployed in a region different from the Control Plane, ensuring compatibility across distributed setups. + +
    • +
+ +### 1.24.1 Release Notes + +#### Release Date 16 of December 2024 + +#### Release Highlights + +This Tyk Cloud hotfix addresses a critical issue affecting Control Plane Redis scheduling. In certain scenarios, re-deploying existing CDPs (Control Data Planes) that were created before the 1.24.0 release would fail due to an unintended modification of Redis deployment selectors. + +#### Breaking Changes +{/* Required. Use the following statement if there are no breaking changes, or explain if there are */} +There are no breaking changes in this release + +#### Downloads +- [latest version of Mserv](https://github.com/TykTechnologies/mserv/releases/latest) + +#### Deprecations +{/* Required. Use the following statement if there are no deprecations, or explain if there are */} +There are no deprecations in this release + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Fixed +{/* This section should be a bullet point list that describes the issues fixed in the release. For each fixed issue explain: +- What problem the issue caused +- How was the issue fixed +- Link to (new) documentation created as a result of a fix. For example, a new configuration parameter may have been introduced and documented for the fix +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} + +
    +
  • + +Corrected the Redis deployment selectors to prevent errors during both re-deployments of existing CDPs and deployments of new CDPs. + +
    • +
+ +### 1.24.0 Release Notes + +#### Release Date 16 of December 2024 + +#### Release Highlights + +This Tyk Cloud update introduces a groundbreaking feature for enhanced API observability and troubleshooting. With the new native Telemetry export, Tyk Cloud now allows organizations to seamlessly integrate their deployments with a variety of popular observability platforms, including built-in support for Datadog, Dynatrace, Elastic, and New Relic. For other systems, the custom provider option ensures compatibility with any platform that supports the OpenTelemetry Protocol (OTLP). + +This feature enables trace export capabilities, providing deep insights into API and plugin performance. It marks the first step in Tyk Cloud’s broader observability journey, empowering users to monitor and troubleshoot their APIs more effectively while leveraging their existing observability tools. + +For more details, check out the [documentation on setting up Telemetry export](/tyk-cloud#enabling-telemetry-in-tyk-cloud). + +### Breaking Changes + +There are no breaking changes in this release + +#### Downloads +- [latest version of Mserv](https://github.com/TykTechnologies/mserv/releases/latest) + +### Deprecations + +There are no deprecations in this release + +--- + +## 1.23 Release Notes + +### Release Date 14 of November 2024 + +### Release Highlights + +This Tyk Cloud update introduces features that improve both flexibility in plugin management and user onboarding. Now, [Mserv](/tyk-cloud#uploading-your-bundle), supports **multiple plugin bundles**, allowing greater customization and easier deployment of plugin configurations. Additionally, we added an **embedded product tour** to enhance the deployment experience, offering a guided walkthrough of Tyk Dashboard’s features, ideal for users familiarizing themselves with the platform during onboarding. + +For a comprehensive list of improvements and fixes, please check the detailed [changelog](#Changelog-v1.23.0) below. + +### Breaking Changes + +There are no breaking changes in this release + +### Downloads + +- [latest version of Mserv](https://github.com/TykTechnologies/mserv/releases/latest) + +### Deprecations + +There are no deprecations in this release + +

Changelog

+#### Added + +
    +
  • + + +A HubSpot contact form has been added in both Tyk Cloud and Dashboard to facilitate contacting Tyk for a Proof of Concept (PoC) when a trial expires. This new form makes it easier to connect with our team and explore further options once +the trial period ends. + + + +Tyk Cloud now supports multiple plugins in a bundle, allowing users to manage and deploy various binaries for the same plugin bundle. This enhancement provides greater flexibility in plugin configuration and deployment using `mservctl`. +within MServ. + + + +An embedded interactive product tour has been added within the deployment screen to guide users through the Tyk Dashboard while they wait for their free trial on-boarding to complete. This tour provides an overview of key features, helping users explore what they can do on Tyk Cloud during their trial. + + +
    • +
+ +#### Changed + +
    +
  • + +Users are now redirected to the "Activity by API" section in the Tyk Dashboard upon clicking on the Control Plane (CP) name within the Cloud Monitoring page. This update provides a more seamless +transition for users needing detailed activity insights directly from the monitoring interface. + +
    • +
+ +#### Fixed + +
    +
  • + + + + + + +The behavior for accessing the billing app through the 'Upgrade' and 'Account & Billing' buttons has been standardized. Previously, clicking the 'Upgrade' button opened the billing app in a new tab, while 'Account & Billing' opened it in the same tab. Now, both buttons open the billing app consistently in the same tab. + + + +Fixed an issue where accessing the /password-reset page directly redirected users to the login page. Now, users can navigate directly to the /password-reset page without being redirected, providing a consistent experience for password-reset requests regardless of how the page is accessed. + + + +We have resolved a display issue in the billing sidebar that occurred when no subscriptions were active. Now, the sidebar menu displays correctly regardless of subscription status, providing a consistent and clear UI for all users. + + + +This update addresses a critical bug in the Hybrid nodes visibility logic, which previously retained all connected node data for the Hybrid Data Planes indefinitely. The fix ensures that we only contains records from the last 7 days. This enhancement improves system performance at all stages within the Tyk Cloud UI for organizations with Hybrid Data Planes, especially those with multiple connected gateways. + + + +We have enhanced separation between free-trial and paid deployments to improve resilience and stability. + + +
    • +
+ +##### Security Fixes + +
    +
  • + +Dependencies across all Tyk Cloud components have been updated to address reported security issues. This update ensures compliance with security standards, aligning the project with best practices for secure dependency management. + +
    • +
+ +## Further Information + +### FAQ +Please visit our [Developer Support](/developer-support/community) page for further information relating to reporting bugs, upgrading Tyk, technical support and how to contribute. diff --git a/developer-support/release-notes/dashboard.mdx b/developer-support/release-notes/dashboard.mdx new file mode 100644 index 00000000..94cadbba --- /dev/null +++ b/developer-support/release-notes/dashboard.mdx @@ -0,0 +1,4304 @@ +--- +title: "Tyk Dashboard Release Notes" +'og:description': "Release notes documenting updates, enhancements, and changes for Tyk Dashboard." +tags: ['Tyk Dashboard', 'Release notes', 'changelog'] +sidebarTitle: "Dashboard" +--- + +{/* Required. oss or licensed. Choose one of the following: + **Licensed Protected Product** + Or + ****Open Source** ([Mozilla Public License](https://github.com/TykTechnologies/tyk/blob/master/LICENSE.md))** */} + +**This page contains all release notes for Dashboard displayed in a reverse chronological order** + +## Support Lifetime + +Our minor releases are supported until our next minor comes out. + +--- + + +## 5.8 Release Notes + +### 5.8.2 Release Notes + +#### Release Date 1st July 2025 + +#### Release Highlights + +This is a version bump to align with Gateway v5.8.2, no changes have been implemented in this release. + +#### Breaking Changes + +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components + +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.8.2 | MDCB v2.8.1 | MDCB v2.8.1 | +| | Operator v1.2.0 | Operator v0.17 | +| | Sync v2.1.0 | Sync v2.1.0 | +| | Helm Chart v3.0 | Helm all versions | +| | EDP v1.13 | EDP all versions | +| | Pump v1.12.0| Pump all versions | +| | TIB (if using standalone) v1.7.0 | TIB all versions | + +
3rd Party Dependencies & Tools
+| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.23 | 1.23 | [Go plugins](/api-management/plugins/golang) must be built using Go 1.23 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 13.x - 17.x | 13.x - 17.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas#tyk-vendor-extension-reference)| + +#### Deprecations + +There are no deprecations in this release. + +

Upgrade instructions

+If you are upgrading to 5.8.2, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads + +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.8.2) + - ```bash expandable + docker pull tykio/tyk-dashboard:v5.8.2 + ``` +- Helm charts + - [tyk-charts v3.0.0](/developer-support/release-notes/helm-chart#300-release-notes) + +

Changelog

+No changes in this release. + + +--- + +### 5.8.1 Release Notes + +#### Release Date 9 May 2025 + +#### Release Highlights + +This patch release contains various bug fixes. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.8.1) below. + +#### Breaking Changes + +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components + +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.8.1 | MDCB v2.8.1 | MDCB v2.8.1 | +| | Operator v1.2.0 | Operator v0.17 | +| | Sync v2.1.0 | Sync v2.1.0 | +| | Helm Chart v3.0 | Helm all versions | +| | EDP v1.13 | EDP all versions | +| | Pump v1.12.0| Pump all versions | +| | TIB (if using standalone) v1.7.0 | TIB all versions | + +
3rd Party Dependencies & Tools
+| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.23 | 1.23 | [Go plugins](/api-management/plugins/golang) must be built using Go 1.23 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 13.x - 17.x | 13.x - 17.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas#tyk-vendor-extension-reference)| + +#### Deprecations + +There are no deprecations in this release. + +

Upgrade instructions

+If you are upgrading to 5.8.1, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads + +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.8.1) + - ```bash + docker pull tykio/tyk-dashboard:v5.8.1 + ``` +- Helm charts + - [tyk-charts v3.0.0](/developer-support/release-notes/helm-chart#300-release-notes) + +

Changelog

+##### Fixed + +
    +
  • + + +Fixed an issue where the Dashboard might not allow the correct number of Gateways to connect. This was due to a conflict with license management in deployments with multiple Dashboards which has now been resolved. + + + +Fixed an issue where existing admin users could have their permissions overwritten by SSO group settings during login. Admin users now retain their original permissions when `sso_enable_user_lookup` is enabled. Group permissions are only applied to new or non-admin users. + + + +Fixed an issue that prevented creation of an API from a YAML format Tyk OAS API definition. You can now export your Tyk OAS API definition in YAML or JSON format and use the content of this file to create a new API. + + + +Fixed an issue when using API Designer to migrate an API from Tyk Classic to Tyk OAS. Previously the cancel button did not work in the pop-up when promoting a staged API. + + + +Fixed an issue that prevented import of multi-part OpenAPI descriptions in YAML format. + + + +We have implemented various fixes and improvements in the Dashboard UI to enhance usability. + + +
    • +
+ +##### Security Fixes + +
    +
  • + +Fixed the following high priority [CVEs](https://www.cve.org/) identified in the Tyk Dashboard, providing increased protection against security vulnerabilities: +- [CVE-2025-46569](https://nvd.nist.gov/vuln/detail/CVE-2025-46569) + +
    • +
+ +### 5.8.0 Release Notes + +#### Release Date 28 March 2025 + +#### Release Highlights + +With Tyk 5.8.0 we are delighted to unlock the power and flexibility of Tyk OAS for all users, with full feature parity with the legacy Tyk Classic style for REST APIs. We are thrilled to announce new updates and improvements in Tyk 5.8.0, delivering more control, flexibility, and performance. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.8.0) below. + +##### Full support for API configuration using Tyk OAS + +We have completed the journey with Tyk OAS that started back in Tyk 4.1 - and now anything that you can configure using the Tyk Classic API definition is also available in the Tyk OAS API definition. Tyk OAS is now the recommended API style for all REST services, with Tyk Classic recommended for use only for GraphQL and TCP services. + +With Tyk OAS we combine the industry standard OpenAPI description with the Tyk Vendor Extension, which encapsulates all of the Tyk Gateway settings that cannot be inferred from the OpenAPI Specification (OAS). You can keep your service description (OAS) as source of truth and update the OpenAPI description part of a Tyk OAS API independently from the Tyk Vendor Extension - no need to unpick distributed vendor extensions from your OAS. For more details, please see the [documentation](/api-management/gateway-config-tyk-oas). + +Now that we have achieved this milestone we are keen to support users in migrating their existing Tyk Classic API portfolio to Tyk OAS and offer methods to do this both within the Tyk Dashboard Classic API Designer and via the Tyk Dashboard API. For more details of the migration tool, please see the [documentation](/api-management/migrate-from-tyk-classic). + +##### Enhanced upstream authentication + +We are pleased to introduce advanced options for your Tyk OAS APIs when it comes to authenticating with the upstream service - a critical feature for integration with many partner services. With Tyk 5.8.0 you are now able to configure Tyk to act as an OAuth 2.0 client, retrieving an access token via the Client Credentials grant method. For legacy integrations Tyk can also support OAuth 2.0 Resource Owner Password Credentials grant and Basic Authentication methods. For more details please see the [documentation](/api-management/upstream-authentication). + +##### Enhanced user experience within the Tyk Dashboard API Designer + +To accompany the launch of fully featured Tyk OAS capabilities, we have made a raft of improvements in the Tyk Dashboard GUI. There's an all-new API test and debug facility in the API designer, allowing you to issue requests to your APIs and then examine the debug traces produced by the Gateway without leaving the Tyk Dashboard. Our new, enhanced code editor allows you to work in YAML or JSON. We've also given the UI a spring clean to improve the usability. + + +#### Breaking Changes + +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components + +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.8.0 | MDCB v2.8.0 | MDCB v2.8.0 | +| | Operator v1.2.0 | Operator v0.17 | +| | Sync v2.1.0 | Sync v2.1.0 | +| | Helm Chart v3.0 | Helm all versions | +| | EDP v1.13 | EDP all versions | +| | Pump v1.12.0| Pump all versions | +| | TIB (if using standalone) v1.7.0 | TIB all versions | + +
3rd Party Dependencies & Tools
+| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.23 | 1.23 | [Go plugins](/api-management/plugins/golang) must be built using Go 1.23 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 13.x - 17.x | 13.x - 17.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas#tyk-vendor-extension-reference)| + +#### Deprecations + +There are no deprecations in this release. + +

Upgrade instructions

+If you are upgrading to 5.8.0, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads + +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.8.0) + - ```bash + docker pull tykio/tyk-dashboard:v5.8.0 + ``` +- Helm charts + - [tyk-charts v3.0.0](/developer-support/release-notes/helm-chart#300-release-notes) + +

Changelog

+##### Added + +
    +
  • + + +Tyk Dashboard's API and UI now support both YAML and JSON for Tyk OAS CRUD operations, giving users greater flexibility in managing their APIs. + + + +We've enhanced Tyk's OpenAPI handling to allow the import of multi-part OpenAPI documents. This allows users to import OpenAPI descriptions that are split across multiple files, making it easier to manage complex API specifications. + + + +We’ve added built-in testing and debugging capabilities to the Tyk OAS API Designer, making validating and troubleshooting your APIs easier. With a floating debugging panel and an endpoint dropdown, you can test your endpoints within the Dashboard UI. + + + +Users can now configure separate RDS endpoints for read and write operations, optimizing database performance by handling reads on replicas and writes on the primary instance. + +New Configuration Fields: + +- **ReadConnectionString**: Defines the connection string for read operations. It is only used if `ConnectionString` is not set. +- **WriteConnectionString**: Defines the connection string for write operations. It is only used if `ConnectionString` is not set. + +For backward compatibility, if `ConnectionString` is set, it will take precedence over the new fields. + + + +When creating a GraphQL API in the API designer, you can now attach certificates to be used during introspection of mTLS-protected upstream services. This simplifies the creation of GraphQL APIs by reducing the number of steps, as previously you had to use a dummy upstream during the initial creation and then reconfigure the API to perform introspection later. + + + +Introduced a new [API migration facility](/api-management/migrate-from-tyk-classic) to the Tyk Dashboard API and API Designer enabling a seamless transition from Tyk Classic to Tyk OAS. Both methods offer direct and staged conversions of individual APIs. The Tyk Dashboard API also supports bulk API conversions. + +Note that there is a known issue where the Cancel button in the Promote Staged API confirmation modal does not function. If you select Promote Staged API but decide not to proceed, you will need to use your browser’s Back button to exit the modal. This issue will be resolved in the next patch. + + + +Tyk Dashboard now supports integration with upstream services secured using Basic Auth, OAuth 2.0 Client Credentials, and OAuth 2.0 Password Grant in Tyk OAS APIs, providing flexibility in securing upstream authentication flows. + + + +We have introduced a guided onboarding experience for Tyk Cloud users to help new users start effortlessly. Our step-by-step guide walks you through creating your first API, setting up policies and keys, testing endpoints, and exploring analytics - ensuring you can navigate the Dashboard and unlock its full potential from day one. + + + +Tyk Dashboard now provides enhanced visibility into user activity by allowing audit logs to be stored in a database and viewed directly in the Dashboard UI. Previously, these logs were only written to local files, limiting accessibility. With this update, admins can filter and review user actions more easily, improving security, and compliance tracking. + + +
    • + +
+ +##### Changed + +
    +
  • + + +The Dashboard now supports PostgreSQL 17, ensuring compatibility with the latest database version. + + + +Tyk Dashboard now runs on Golang 1.23, bringing security and performance improvements. Key changes include: + +- unbuffered Timer/Ticker channels +- removal of 3DES cipher suites +- updates to X509KeyPair handling. + +**You may need to adjust your setup for compatibility**. For more details, please see the official Go [release notes](https://go.dev/doc/go1.23). + + + +Upgraded the code editor component library and enhanced its styling for a better user experience. With this upgrade comes the facility to work in YAML or JSON and to switch seamlessly between formats. + + + +We have made minor adjustments to labels within the Dashboard UI to enhance clarity. + + + +The "Manage Account" link in the Tyk Dashboard, which previously directed users to an outdated cloud login page, has been removed. This improves the user experience by eliminating confusion around account management and ensuring a more cohesive navigation flow between the Dashboard and Tyk Cloud. + + + +We’ve optimized form validation in the API Designer to enhance user experience. Forms are now validated on blur instead of during every keystroke, preventing cursor jumps and improving typing responsiveness. + + + +Modified the default values of `allow_explicit_policy_id` and `enable_duplicate_slugs` to `true` in the example Dashboard configuration file, to eliminate config problems when deploying Tyk Sync and Tyk Operator. This has no impact on existing deployments. + + + +Removed unsupported TLS versions 1.0 and 1.1 from the Tyk Classic API Designer selector, improving clarity around supported TLS versions and enhancing security. + + +
    • +
+ +##### Fixed + +
    +
  • + +Updated OPA rules in the Dashboard to allow all users to reset their own access tokens and view their user data, improving self-service while maintaining security. +Note that users with custom OPA rules are strongly advised to update their configurations to include the `is_self_key_reset` and `is_me` helper rules. Additionally, they should modify their rules to exclude cases where `is_self_key_reset` or `is_me` apply, in order to enable this functionality. + +
    • +
  • + +Users can now always access their own Dashboard API credentials, regardless of permissions. Admins' ability to view or reset other users' credentials is now strictly controlled by security flags. + +
    • +
+ +--- + +## 5.7 Release Notes + +### 5.7.3 Release Notes + +#### Release Date 05 June 2025 + +#### Release Highlights + +This is a version bump to align with Gateway v5.7.3, no changes have been implemented in this release. + +#### Breaking Changes + +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components + +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.7.3 | MDCB v2.7.2 | MDCB v2.5.1 | +| | Operator v1.1.0 | Operator v0.17 | +| | Sync v2.0.2 | Sync v1.4.3 | +| | Helm Chart v2.2 | Helm all versions | +| | EDP v1.12 | EDP all versions | +| | Pump v1.11.1 | Pump all versions | +| | TIB (if using standalone) v1.6.1 | TIB all versions | + +
3rd Party Dependencies & Tools
+| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.22 | 1.22 | [Go plugins](/api-management/plugins/golang) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +#### Deprecations + +There are no deprecations in this release. + +

Upgrade instructions

+If you are upgrading to 5.7.3, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads + +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.7.3) + - ```bash + docker pull tykio/tyk-dashboard:v5.7.3 + ``` +- Helm charts + - [tyk-charts v2.2.0](/developer-support/release-notes/helm-chart#220-release-notes) + +

Changelog

+No changes in this release. + + +### 5.7.2 Release Notes + +#### Release Date 19 February 2025 + +#### Release Highlights + +This release focuses mainly on a security fix. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.7.2) below. + +#### Breaking Changes + +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components + +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.7.2 | MDCB v2.7.2 | MDCB v2.5.1 | +| | Operator v1.1.0 | Operator v0.17 | +| | Sync v2.0.2 | Sync v1.4.3 | +| | Helm Chart v2.2 | Helm all versions | +| | EDP v1.12 | EDP all versions | +| | Pump v1.11.1 | Pump all versions | +| | TIB (if using standalone) v1.6.1 | TIB all versions | + +
3rd Party Dependencies & Tools
+| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.22 | 1.22 | [Go plugins](/api-management/plugins/golang) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +#### Deprecations + +There are no deprecations in this release + +

Upgrade instructions

+If you are upgrading to 5.7.2, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.7.2) + - ```bash + docker pull tykio/tyk-dashboard:v5.7.2 + ``` +- Helm charts + - [tyk-charts v2.2.0](/developer-support/release-notes/helm-chart#220-release-notes) + +

Changelog

+No changes have been implemented in this release. + +##### Fixed + +
    +
  • + +Fixed an issue where attempting to add a certificate on the TLS/SSL Certificates page caused the page to go blank with an error. This has been resolved by ensuring the file input is handled correctly. + +
    • +
+ +##### Security Fixes + +
    +
  • + +- Fixed the following critical-priority CVE identified in the Dashboard UI, providing increased protection and improved security: + - [CVE-2025-21613](https://nvd.nist.gov/vuln/detail/CVE-2025-21613) + +
    • +
  • + +- Fixed the following CVE: + - [CVE-2025-21614](https://nvd.nist.gov/vuln/detail/CVE-2025-21614) + +
    • +
+ +--- + +### 5.7.1 Release Notes + +#### Release Date 31 December 2024 + +#### Release Highlights + +This release focuses mainly on bug fixes. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.7.1) below. + +#### Breaking Changes + +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components + +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.7.1 | MDCB v2.7.2 | MDCB v2.5.1 | +| | Operator v1.1.0 | Operator v0.17 | +| | Sync v2.0.1 | Sync v1.4.3 | +| | Helm Chart v2.2 | Helm all versions | +| | EDP v1.12 | EDP all versions | +| | Pump v1.11.1 | Pump all versions | +| | TIB (if using standalone) v1.6.1 | TIB all versions | + +
3rd Party Dependencies & Tools
+| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.22 | 1.22 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +#### Deprecations + +We have deprecated the obsolescent `http_server_options.prefer_server_ciphers` configuration option. This legacy control no longer has any effect on the underlying library and users are advised to remove this setting from their configurations. + +

Upgrade instructions

+If you are upgrading to 5.7.1, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.7.1) + - ```bash + docker pull tykio/tyk-dashboard:v5.7.1 + ``` +- Helm charts + - [tyk-charts v2.2.0](/developer-support/release-notes/helm-chart#220-release-notes) + +

Changelog

+##### Fixed + +
    +
  • + +Resolved a bug where clicking "Restore zooming to initial state" in the API Activity Dashboard would cause the graph to show blank instead of resetting to the initial zoom level. This fix ensures that users can now correctly restore the default zoom state in all charts on the Dashboard. + +
    • +
  • + +This option has been marked as deprecated due to its obsolescence in the underlying library functions used by Tyk. Users are advised to remove this configuration from their setups as it no longer has any effect. + +
    • +
+ +--- + +### 5.7.0 Release Notes + +#### Release Date 03 December 2024 + +#### Release Highlights + +We are thrilled to announce new updates and improvements in Tyk 5.7.0, bringing more control, flexibility, and performance. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.7.0) below. + +##### Tyk Streams can be configured through Tyk Dashboard + +With this release we are adding a possibility for users to configure their Stream & Events APIs using Tyk Dashboard. +The new API designer leads users step-by-step to create a new Stream configuration easily. Pre-filled stream configurations for different inputs and outputs make it easy to make sure that the Stream is configured correctly. + +##### Improved Audit Log Management + +Tyk 5.7.0 enhances Audit Log management with new features designed for efficiency and security. Users can now store Dashboard Audit Logs in a database for persistent retention and access them via the new /audit-logs API, which supports advanced filtering by attributes like action, IP, status, and user. Additionally, a dedicated Audit Log RBAC group ensures secure access to sensitive log data. These improvements simplify monitoring and compliance workflows, particularly in containerized environments. + +#### Breaking Changes +{/* Required. Use the following statement if there are no breaking changes, or explain if there are */} + +There are no breaking changes in this release. + +

Dependencies

+{/* Required. Use this section to announce the following types of dependencies compatible with the release: +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +3rd party dependencies and tools */} +##### Compatibility Matrix For Tyk Components +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.7.0 | MDCB v2.7.2 | MDCB v2.5.1 | +| | Operator v1.1.0 | Operator v0.17 | +| | Sync v2.0.1 | Sync v1.4.3 | +| | Helm Chart v2.2 | Helm all versions | +| | EDP v1.12 | EDP all versions | +| | Pump v1.11.1 | Pump all versions | +| | TIB (if using standalone) v1.6.1 | TIB all versions | + +
3rd Party Dependencies & Tools
+{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.22 | 1.22 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +#### Deprecations +{/* Required. Use the following statement if there are no deprecations, or explain if there are */} +In 5.7.0, we have deprecated the dedicated [External OAuth](/api-management/client-authentication#integrate-with-external-authorization-server-deprecated) (Tyk Classic: `external_oauth`, Tyk OAS: `server.authentication.securitySchemes.externalOAuth`) and [OpenID Connect](/api-management/client-authentication#integrate-with-openid-connect-deprecated) (Tyk Classic: `auth_configs.oidc`, Tyk OAS: `server.authentication.oidc`) authentication methods. We advise users to switch to [JWT Authentication](/basic-config-and-security/security/authentication-authorization/json-web-tokens). + +Additionally, SQLite has reached its End of Life in this release, enabling a fully static, CGO-free Tyk Dashboard optimised for RHEL8. Sqlite was previously recommended only to be used in basic proofs of concept. Now, for such scenarios and for production, we recommend migrating to PostgreSQL or MongoDB for better scalability and support. +{/* Optional section! +Used to share and notify users about our plan to deprecate features, configs etc. +Once you put an item in this section, we must keep this item listed in all the following releases till the deprecation happens. */} +{/* ###### Future deprecations */} +

Upgrade instructions

+If you are upgrading to 5.7.0, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.7.0) + - ```bash + docker pull tykio/tyk-dashboard:v5.7.0 + ``` +- Helm charts + - [tyk-charts v2.2.0](/developer-support/release-notes/helm-chart#220-release-notes) + +

Changelog

+##### Added + +
    +
  • + + +Introduced a confirmation prompt when deleting a stream, notifying users that this action will stop all data streaming and cannot be undone. This change ensures users are fully aware of the impact before proceeding with deletion. + + + +Added "Streams" as an API type in the API Overview table, making it easier for API developers to identify APIs categorised as Streams & Events. + + + +Added logic for the Streaming API creation process, allowing users to select config frameworks for inputs, processors, and outputs. An 'Advanced' option is also available, which leaves the code editor empty while generating and displaying the YAML Bento config based on the user's selections. + + + +Included new info messages and tooltips in the Policies & Keys section to guide users on securing Streaming & Events APIs. Updated messaging clarifies the combination of API types and revised copy in the Global Rate Limiting and Quota sections to better explain usage limits for keys and plans. + + + +Enabled URL view and copy functionality in the External Playgrounds tab, supporting scenarios with multiple organisations and URLs for playgrounds. + + + +Rolled out the `/streams` endpoint to the Tyk Dashboard API, dedicated to creating Stream and Events APIs in Tyk Streams. Documentation for the endpoint and its methods is available in the Tyk Docs. + + + +Separated Streaming API into its own type in the API Designer, introducing a new selection card for easier creation and configuration. Navigation enhancements, including a shortcut menu item, provide quicker access to the streaming configuration UI. + + + +Developed a step-by-step UI for Streaming API creation, enabling users to select a config framework for inputs, processors, and outputs. The dynamic wizard steps are integrated into the Tyk UI library to prefill configurations based on selections and prevent the combination of 'Custom' with other frameworks. + + + +Introduced a form on the Tyk Dashboard that allows users to easily contact Tyk support during their trial period. + + + +We have enhanced security for customers in highly regulated industries by introducing JSON Web Encryption (JWE) support for OIDC single sign-on (SSO). This ensures that tokens used in authentication flows are securely encrypted, providing an additional layer of protection. + +[Setup guide for JWE OIDC SSO](/api-management/external-service-integration#log-into-an-app-with-github-oauth) + + + +Users can now choose to store Dashboard Audit Logs directly in a database, enabling efficient and reliable log storage. This feature is particularly beneficial for organizations needing persistent audit log retention to meet compliance requirements or for forensic purposes. + + + +A new API endpoint, `/audit-logs`, has been introduced to provide programmatic access to audit logs stored in database. This allows users to retrieve, filter, and analyze logs more effectively. The API supports filtering logs by key attributes like action, IP address, URL accessed, date range, user, and page number. + +For detail usage of the `/audit-logs` endpoint, please see [Dashboard API documentation](/tyk-dashboard-api). + + + +To secure access to audit logs, we’ve added a new Audit Log RBAC group. This ensures that only authorized users can view or retrieve sensitive log information. Administrators can assign this permission as part of their security and compliance strategy. + + +
    • +
+ +##### Changed + +
    +
  • + + +Eliminated AJV validation in the Streams Config Editor to prevent false positives on valid YAML configurations. The frontend now solely checks the YAML structure, providing users with greater flexibility without enforcing strict Bento-specific schema rules + + + +Removed an unnecessary field from the API Designer page under the Streams section to enhance clarity. This update impacts the Event Handlers, Detailed Activity Logs, Caching, and Endpoints tabs. + + + +Tyk will now detect path-level parameters in the OpenAPI description and can be set to enable and configure the [Request Validation](/api-management/traffic-transformation/request-validation#request-validation-using-tyk-oas) middleware automatically for these. Previously this automatic detection only worked for method-level parameters in the OpenAPI description. + + + +Removed SQLite support to enhance portability and security, ensuring the released binary can now be built statically and no longer relies on system libraries. This change supports continued compatibility with RHEL8. + + + +The External OAuth and OpenID Connect authentication options have been deprecated in the Tyk Dashboard. Users are advised to utilize JWT Auth with external IDPs for a more complete integration, while existing functionality remains operational to avoid breaking changes. + + + +Updated NPM package dependencies of Dashboard, to address security vulnerabilities. + + +
    • +
+ +##### Fixed + +
    +
  • + + +Resolved an issue where the "Back to APIs Page" button was unresponsive on the Streams API page. The button now correctly redirects users to the main APIs page for all API types. + + + +Corrected an issue where the search box on the Tyk OAS and Streams API pages only accepted a single character. Users can now input complete search terms, allowing for more accurate searches. + + + +Fixed an issue with the *user group* dropdown in the Dashboard UI, ensuring that all available user groups are displayed when creating a new user. + + +
    • +
+ + + + +## 5.6 Release Notes + +### 5.6.1 Release Notes + +#### Release Date 18 October 2024 + +#### Release Highlights + +This is a version bump to align with Gateway v5.6.1, no changes have been implemented in this release. + +#### Breaking Changes + +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.6.1 | MDCB v2.7.1 | MDCB v2.5.1 | +| | Operator v1.0.0 | Operator v0.17 | +| | Sync v2.0 | Sync v1.4.3 | +| | Helm Chart v2.1 | Helm all versions | +| | EDP v1.11 | EDP all versions | +| | Pump v1.11 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +
3rd Party Dependencies & Tools
+{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.22 | 1.22 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +#### Deprecations + +There are no deprecations in this release. + +

Upgrade instructions

+If you are upgrading to 5.6.1, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.6.1) +- ```bash + docker pull tykio/tyk-dashboard:v5.6.1 + ``` +- Helm charts + - [Tyk Charts v2.0.0](/developer-support/release-notes/helm-chart#200-release-notes) + +

Changelog

+No changes in this release. + + +--- +### 5.6.0 Release Notes + +#### Release Date 10 October 2024 + +#### Release Highlights + +We are thrilled to announce new updates and improvements in Tyk 5.6.0, bringing more control, flexibility, and performance. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.6.0) below. + +##### Per endpoint Rate Limiting for clients + +Now you can configure rate limits at the [endpoint level per client](/api-management/rate-limit#key-level-rate-limiting), using new configuration options in the access key. Use Tyk's powerful [security policies](/api-management/policies#what-is-a-security-policy) to create templates to set appropriate rate limits for your different categories of user. + +##### Go upgrade to 1.22 + +We’ve upgraded the Tyk Dashboard to Golang 1.22, bringing improved performance, better security, and enhanced stability to the core system. + +##### Strengthened Role-Based Access Controls (RBAC) to combat privilege escalation risks + +We’ve tightened up the rules that govern a user's ability to create admin users and to reset other users' passwords when using Tyk's RBAC function. Now, only super-admins can create new admins, admin roles can't be assigned to user groups, and only admin users can reset another user's password (and only within their Tyk organization). + +#### Breaking Changes +{/* Required. Use the following statement if there are no breaking changes, or explain if there are */} + +There are no breaking changes in this release. + +

Dependencies

+{/* Required. Use this section to announce the following types of dependencies compatible with the release: +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +3rd party dependencies and tools */} +##### Compatibility Matrix For Tyk Components +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.6.0 | MDCB v2.7.1 | MDCB v2.5.1 | +| | Operator v1.0.0 | Operator v0.17 | +| | Sync v2.0 | Sync v1.4.3 | +| | Helm Chart v2.1 | Helm all versions | +| | EDP v1.11 | EDP all versions | +| | Pump v1.11 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +
3rd Party Dependencies & Tools
+{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.22 | 1.22 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +#### Deprecations +{/* Required. Use the following statement if there are no deprecations, or explain if there are */} + +We are deprecating support for SQLite, External OAuth Middleware, and OpenID Connect (OIDC) Middleware in Tyk Dashboard to simplify the platform and enhance overall performance. These changes will take effect from 5.7.0. + +#### Why the Change? + +#### SQLite + +While useful for testing, SQLite is not designed for production environments. By focusing on PostgreSQL and MongoDB, we can provide users with more scalable and reliable options. + +#### External OAuth Middleware + +This feature serves a similar purpose to our JWT Authentication and may lead to confusion. We recommend transitioning to JWT Authentication for a more streamlined experience. + +#### OpenID Connect (OIDC) Middleware + +The low adoption of this option, along with its functional overlap with other supported authentication methods, prompts us to deprecate OIDC middleware to reduce complexity within the platform. We recommend users transition to JWT Authentication. + + +We encourage users to switch to the recommended alternatives. For more detailed information, please refer to the [Documentation](/api-management/client-authentication#integrate-with-openid-connect-deprecated) + + +{/* Optional section! +Used to share and notify users about our plan to deprecate features, configs etc. +Once you put an item in this section, we must keep this item listed in all the following releases till the deprecation happens. */} +{/* ###### Future deprecations */} +

Upgrade instructions

+If you are upgrading to 5.6.0, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.6.0) +- ```bash + docker pull tykio/tyk-dashboard:v5.6.0 + ``` +- Helm charts + - [tyk-charts v2.1.0](/developer-support/release-notes/helm-chart#210-release-notes) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} +##### Added +{/* This section should be a bullet point list of new features. Explain: +- The purpose of the new feature +- How does the new feature benefit users? +- Link to documentation of the new feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + +Building on the [per-endpoint upstream rate limits](/api-management/rate-limit#api-level-rate-limiting) introduced in Tyk 5.5.0 we have now added [per-endpoint client rate limits](/api-management/rate-limit#key-level-rate-limiting). This new feature allows for more granular control over client consumption of API resources by associating the rate limit with the access key, enabling you to manage and optimize API usage more effectively. + +
    • +
+ +##### Changed + +{/* This should be a bullet-point list of updated features. Explain: + +- Why was the update necessary? +- How does the update benefit users? +- Link to documentation of the updated feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +The Tyk Dashboard has been upgraded from Golang 1.21 to Golang 1.22, bringing enhanced performance, strengthened security, and access to the latest features available in the new Golang release. + + + +We have updated the swagger.yml schema for Tyk Dashboard API to reflect the latest changes in product endpoints, payloads, and responses. This update includes new fields and endpoints, improved examples, documentation adjustments, and fixes for schema issues. These enhancements aim to improve usability and ensure that the documentation accurately represents the current code state. + + + +The "Playground" tab in the GraphQL API Designer has been renamed to "Playgrounds." This change consolidates access to both internal and external playgrounds within a single section, offering a more streamlined and intuitive experience for API design and testing. + + +
    • +
+ +##### Fixed +{/* This section should be a bullet point list that describes the issues fixed in the release. For each fixed issue explain: +- What problem the issue caused +- How was the issue fixed +- Link to (new) documentation created as a result of a fix. For example, a new configuration parameter may have been introduced and documented for the fix +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +- Resolved an issue where HTTP 429 status codes were not being displayed on the Activity Overview page. +- Fixed portal graphs by adding a default "day" grouping resolution to the query. +- Corrected issues with the Error Breakdown related to date parameters, ensuring accurate date handling and display. + + + +We have resolved an issue where the Keys page would display a blank screen if a key was associated with more than 10 policies. The UI has been fixed to display the page properly, regardless of the number of policies attached to a key. + + + +When working with Tyk Classic APIs, you cannot permit access to multiple versions of the same API from a single policy. We have fixed an issue in the Dashboard UI where users were able to attach multiple versions to a policy leading to an unusable policy. The UI now correctly prevents the addition of multiple versions of an API to a single policy. + + + +We have fixed an issue in the Dashboard UI when assigning multiple claim to policy mappings while configuring JWT auth for an API. The scope name was incorrectly recorded instead of the policy ID for the second and subsequent JWT scope mappings. The UI now correctly associates the defined claim with the appropriate policy, ensuring accurate JWT scope to policy mappings. + + + +We have fixed an issue in the Monitoring section of the Dashboard UI where the *Gateway logs* page was not displaying correctly. The page is now rendered properly, ensuring users with appropriate permissions can view and manage *Gateway logs* as expected. + + +
    • + +
+ +--- + +## 5.5 Release Notes + +### 5.5.2 Release Notes + +#### Release Date 03 October 2024 + +#### Release Highlights + +This release replaces Tyk Dashboard 5.5.1 which was accidentally released as a non-distroless image. + +#### Breaking Changes + +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components + +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.5.2 | MDCB v2.7 | MDCB v2.5.1 | +| | Operator v0.18 | Operator v0.17 | +| | Sync v1.5 | Sync v1.4.3 | +| | Helm Chart v2.0.0 | Helm all versions | +| | EDP v1.10 | EDP all versions | +| | Pump v1.11 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +
3rd Party Dependencies & Tools
+| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.21 | 1.21 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +#### Deprecations + +There are no deprecations in this release. + +

Upgrade instructions

+If you are upgrading to 5.5.2, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.5.2) +- ```bash + docker pull tykio/tyk-dashboard:v5.5.2 + ``` +- Helm charts + - [Tyk Charts v2.0.0](/developer-support/release-notes/helm-chart#200-release-notes) + +

Changelog

+No changes in this release. + +--- + +### 5.5.1 Release Notes + +#### Release Date 26 September 2024 + +#### Release Highlights + +This is a version bump to align with Gateway v5.5.1, no changes have been implemented in this release. + +#### Breaking Changes + +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components + +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.5.1 | MDCB v2.7 | MDCB v2.5.1 | +| | Operator v0.18 | Operator v0.17 | +| | Sync v1.5 | Sync v1.4.3 | +| | Helm Chart v2.0.0 | Helm all versions | +| | EDP v1.10 | EDP all versions | +| | Pump v1.11 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +
3rd Party Dependencies & Tools
+| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.21 | 1.21 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +#### Deprecations + +There are no deprecations in this release. + +

Upgrade instructions

+If you are upgrading to 5.5.1, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.5.1) +- ```bash + docker pull tykio/tyk-dashboard:v5.5.1 + ``` +- Helm charts + - [Tyk Charts v2.0.0](/developer-support/release-notes/helm-chart#200-release-notes) + +

Changelog

+No changes in this release. + +--- + +### 5.5.0 Release Notes + +#### Release Date 12 August 2024 + +#### Release Highlights + +We are excited to announce Tyk Dashboard 5.5, featuring a brand-new dashboard identity, advanced rate-limiting capabilities, and enhanced security options. For a comprehensive list of changes, please refer to the [changelog](#Changelog-v5.5.0) below. + +##### New Tyk brand identity + +Experience a refreshed and modern look with our updated brand identity. The new design enhances usability and provides a cleaner, more intuitive interface for managing your APIs. + +##### Per Endpoint Rate Limiting + +Now configure rate limits at the endpoint level for both [Tyk OAS](/api-management/rate-limit#tyk-oas-api-definition) and [Tyk Classic APIs](/api-management/rate-limit#tyk-classic-api-definition), providing granular protection for upstream services against overloading and abuse. + +#### Breaking Changes +{/* Required. Use the following statement if there are no breaking changes, or explain if there are */} + +There are no breaking changes in this release. + +

Dependencies

+{/* Required. Use this section to announce the following types of dependencies compatible with the release: +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +3rd party dependencies and tools */} +##### Compatibility Matrix For Tyk Components +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.5.0 | MDCB v2.7 | MDCB v2.5.1 | +| | Operator v0.18 | Operator v0.17 | +| | Sync v1.5 | Sync v1.4.3 | +| | Helm Chart v1.6 | Helm all versions | +| | EDP v1.10 | EDP all versions | +| | Pump v1.11 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +
3rd Party Dependencies & Tools
+{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.21 | 1.21 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +#### Deprecations +{/* Required. Use the following statement if there are no deprecations, or explain if there are */} +There are no deprecations in this release. +{/* Optional section! +Used to share and notify users about our plan to deprecate features, configs etc. +Once you put an item in this section, we must keep this item listed in all the following releases till the deprecation happens. */} +{/* ###### Future deprecations */} +

Upgrade instructions

+If you are upgrading to 5.5.0, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.5.0) +- ```bash + docker pull tykio/tyk-dashboard:v5.5.0 + ``` +- Helm charts + - [tyk-charts v1.6](/developer-support/release-notes/helm-chart#160-release-notes) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} +##### Added +{/* This section should be a bullet point list of new features. Explain: +- The purpose of the new feature +- How does the new feature benefit users? +- Link to documentation of the new feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +Rate limits can now be configured at the endpoint level in Tyk OAS and Tyk Classic API definitions. Configure these new more granular controls from the API Designer. + + + +When configuring API versioning settings for Tyk OAS APIs, you can now set a *Version Identifier Pattern* when using the URL path to indicate the API version (for example `/v1/my-api`). This will be used to avoid accidentally stripping part of the target URL (and failed upstream proxy) if the client doesn't provide any version identifier. If you're using Tyk Classic APIs you can set the `url_versioning_pattern` field in the API definition using the raw API editor. + + + +We've expanded the functionality of the schema editor for GQL APIs. Users can now easily import their schema from a file, export it, or quickly clean the entire editor if a mistake is made. + + +
    • +
+ +##### Changed + +
    +
  • + +Updated npm package dependencies of Dashboard, to address security vulnerabilities. + +
    • +
+ +##### Fixed +{/* This section should be a bullet point list that describes the issues fixed in the release. For each fixed issue explain: +- What problem the issue caused +- How was the issue fixed +- Link to (new) documentation created as a result of a fix. For example, a new configuration parameter may have been introduced and documented for the fix +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +Addressed a problem where Response Plugins were not being invoked for Tyk OAS APIs. + + + +Addressed an issue for SSO users where user permissions were not correctly applied. This led to the Save API button not being visible to all appropriate users in the API Designer. + + + +Resolved an issue where the Public GQL Playground displayed schema information despite introspection being turned off. Now, schema details are hidden unless valid authentication credentials are provided, ensuring a secure and consistent user experience. + + + +Addressed an issue where the Dashboard displayed a blank pane when accessing the Activity by Endpoint page after upgrading to Tyk 5.3.1. + + +
    • +
+ +##### Security Fixes +{/* This section should be a bullet point list that should be included when any security fixes have been made in the release, e.g. CVEs. For CVE fixes, consideration needs to be made as follows: +1. Dependency-tracked CVEs - External-tracked CVEs should be included on the release note. +2. Internal scanned CVEs - Refer to the relevant engineering and delivery policy. +For agreed CVE security fixes, provide a link to the corresponding entry on the NIST website. For example: +- Fixed the following CVEs: + - [CVE-2022-33082](https://nvd.nist.gov/vuln/detail/CVE-2022-33082) */} +
    +
  • + +Fixed the following high priority CVEs identified in the Tyk Dashboard, providing increased protection against security vulnerabilities: +- [CVE-2023-39325](https://nvd.nist.gov/vuln/detail/CVE-2023-39325) +- [CVE-2023-45283](https://nvd.nist.gov/vuln/detail/CVE-2023-45283) + +
    • +
+ +--- + +## 5.4 Release Notes +### 5.4.0 Release Notes +#### Release Date 2 July 2024 +#### Breaking Changes +{/* Required. Use the following statement if there are no breaking changes, or explain if there are */} +**Attention: Please read this section carefully** +There are no breaking changes in this release. + +

Dependencies

+{/* Required. Use this section to announce the following types of dependencies compatible with the release: +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +3rd party dependencies and tools */} +##### Compatibility Matrix For Tyk Components +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.4.0 | MDCB v2.6.0 | MDCB v2.5.1 | +| | Operator v0.18 | Operator v0.17 | +| | Sync v1.5.0 | Sync v1.4.3 | +| | Helm Chart v1.5.0 | Helm all versions | +| | EDP v1.10.0 | EDP all versions | +| | Pump v1.10.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +
3rd Party Dependencies & Tools
+{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.21 | 1.21 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +#### Deprecations +{/* Required. Use the following statement if there are no deprecations, or explain if there are */} +There are no deprecations in this release. +{/* Optional section! +Used to share and notify users about our plan to deprecate features, configs etc. +Once you put an item in this section, we must keep this item listed in all the following releases till the deprecation happens. */} +{/* ###### Future deprecations */} +

Upgrade instructions

+If you are upgrading to 5.4.0, please follow the detailed [upgrade instructions](#upgrading-tyk). +Add upgrade steps here if necessary. + +#### Release Highlights + +We're thrilled to introduce exciting enhancements in Tyk Dashboard 5.4, aimed at improving your experience with Tyk Dashboard. For a comprehensive list of changes, please refer to the change log below. + +#### Event handling for Tyk OAS APIs + +We’ve added support for you to register webhooks with your Tyk OAS APIs so that you can handle events triggered by the Gateway, including circuit breaker and quota expiry. You can also assign webhooks to be fired when using the new smoothing rate limiter to notify your systems of ongoing traffic spikes. For more details see the [documentation](/api-management/gateway-events#event-handling-with-webhooks). + +#### Enhanced Header Handling in GraphQL APIs + +Introduced a features object in API definitions for GQL APIs, including the `use_immutable_headers` attribute. This allows advanced header control, enabling users to add new headers, rewrite existing ones, and selectively remove specific headers. Existing APIs will have this attribute set to `false` by default, ensuring no change in behavior. For new APIs, this attribute is true by default, facilitating smoother migration and maintaining backward compatibility. + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.4.0) +- ```bash + docker pull tykio/tyk-dashboard:v5.4.0 + ``` +- Helm charts + - [tyk-charts v1.5](/developer-support/release-notes/helm-chart#150-release-notes) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} +##### Added +{/* This section should be a bullet point list of new features. Explain: +- The purpose of the new feature +- How does the new feature benefit users? +- Link to documentation of the new feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +Implemented a [rate limit smoothing mechanism](/api-management/rate-limit#rate-limit-smoothing) to gradually adjust the rate limit as the request rate increases and decreases between an intermediate threshold and the maximum rate limit. New `RateLimitSmoothingUp` and `RateLimitSmoothingDown` events will be triggered as this smoothing occurs, supporting auto-scaling of upstream capacity. The smoothing process gradually increases the rate, thereby unblocking clients that exceed the current request rate in a staggered manner. + + + +Revamped the API designer toolbar for GraphQL and Universal Data Graph, consolidating all relevant actions for each API type under a single menu dropdown for improved usability. + + + +Revamped the API designer toolbar for HTTP and TCP, consolidating all relevant actions for each API type under a single menu dropdown for improved usability. + + + +We’ve added some more features to the Tyk OAS API, moving closer to full parity with Tyk Classic. In this release we’ve added controls that allow you: to enable or prevent generation of traffic logs at the API-level; to enable or prevent the availability of session context to middleware and to pin public key certificates to an API. We’ve also added the facility to register webhooks that will be fired in response to Gateway events. + + + +We have added a new `/oas/dry-run` endpoint to the Tyk Dashboard API. This uses the Dashboard’s logic to create or update a Tyk OAS API definition using an OpenAPI document without instantiating the API on the Tyk platform. + + +
    • +
+ +##### Fixed +{/* This section should be a bullet point list that describes the issues fixed in the release. For each fixed issue explain: +- What problem the issue caused +- How was the issue fixed +- Link to (new) documentation created as a result of a fix. For example, a new configuration parameter may have been introduced and documented for the fix +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +Resolved a bug in the API Designer where certain properties, such as `use_immutable_headers`, were not correctly inherited from the new API template. This fix ensures all default settings from the template are properly applied when creating a new API. + + + +Fixed an issue where API Templates were not correctly assigned to Tyk Organizations, preventing potential accidental sharing of secret data between Organizations through the use of incorrect templates. + + + +Fixed an issue where common keyboard shortcuts (Cmd + X, A, C, V) were not functioning correctly when configuring the URL field for a UDG data source. + + + +Improved the data source import endpoint in the Dashboard API by removing the need for users to convert OpenAPI/AsyncAPI documents into strings before submission. Users can now provide the documents directly, enhancing the overall user experience. + + + +Modified default OPA rules to fix an issue where admins were unable to reset their own password. Tyk Dashboard clients using custom OPA rules should update their rule set accordingly. Contact your assigned Tyk representative for assistance. + + + +Addressed an issue in the api/usage endpoint where Dashboard analytics with PostgreSQL returned unfiltered results. The endpoint now correctly filters results, eliminating the need for duplicating parameters to handle multiple tags. + + + +We have made some improvements to the wording used in the Dashboard user interface and fixed some minor usability issues. + + +
    • +
+ +##### Security Fixes +{/* This section should be a bullet point list that should be included when any security fixes have been made in the release, e.g. CVEs. For CVE fixes, consideration needs to be made as follows: +1. Dependency-tracked CVEs - External-tracked CVEs should be included on the release note. +2. Internal scanned CVEs - Refer to the relevant engineering and delivery policy. +For agreed CVE security fixes, provide a link to the corresponding entry on the NIST website. For example: +- Fixed the following CVEs: + - [CVE-2022-33082](https://nvd.nist.gov/vuln/detail/CVE-2022-33082) */} +
    +
  • + +Fixed the following high priority CVEs identified in the Tyk Dashboard, providing increased protection against security vulnerabilities: +- [CVE-2023-39325](https://nvd.nist.gov/vuln/detail/CVE-2023-39325) +- [CVE-2023-45283](https://nvd.nist.gov/vuln/detail/CVE-2023-45283) + +
    • +
+ +--- +{/* */} + + +## 5.3 Release Notes + +### 5.3.11 Release Notes + +#### Release Date 7 May 2025 + +#### Release Highlights + +This patch release contains various bug fixes. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.3.11) below. + +#### Breaking Changes + +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components + +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.3.11 | MDCB v2.8.0 | MDCB v2.8.0 | +| | Operator v1.2.0 | Operator v0.17 | +| | Sync v2.1.0 | Sync v2.1.0 | +| | Helm Chart v3.0 | Helm all versions | +| | EDP v1.13 | EDP all versions | +| | Pump v1.12.0| Pump all versions | +| | TIB (if using standalone) v1.7.0 | TIB all versions | + +
3rd Party Dependencies & Tools
+| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.23 | 1.23 | [Go plugins](/api-management/plugins/golang) must be built using Go 1.23 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 13.x - 17.x | 13.x - 17.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas#tyk-vendor-extension-reference)| + +#### Deprecations + +There are no deprecations in this release. + +

Upgrade instructions

+If you are upgrading to 5.3.11, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads + +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.3.11) + - ```bash + docker pull tykio/tyk-dashboard:v5.3.11 + ``` +- Helm charts + - [tyk-charts v3.0.0](/developer-support/release-notes/helm-chart#300-release-notes) + +

Changelog

+##### Fixed + +
    +
  • + + +Fixed an issue where the Dashboard might not allow the correct number of Gateways to connect. This was due to a conflict with license management in deployments with multiple Dashboards which has now been resolved. + + + +Users can now always access their own Dashboard API credentials, regardless of permissions. Admins’ ability to view or reset other users’ credentials is now strictly controlled by security flags + + + +Updated OPA rules in the Dashboard to allow all users to reset their access tokens and view their user data, improving self-service while maintaining security. + +Users with custom OPA rules are strongly advised to add the `is_self_key_reset` and `is_me` helper rules to their configuration. They should then modify existing relevant rules to exclude cases where `is_self_key_reset` or `is_me` apply, to enable this functionality. + + + +We have implemented various fixes and improvements in the Dashboard GUI to enhance usability. + + +
    • +
+ +#### Security Fixes + +
    +
  • + +Fixed the following high priority CVEs identified in the Tyk Dashboard, providing increased protection against security vulnerabilities: +- [CVE-2025-46569](https://nvd.nist.gov/vuln/detail/CVE-2025-46569) + +
    • +
+ +### 5.3.10 Release Notes + +#### Release Date 19 February 2025 + +#### Release Highlights + +In this release, we upgraded the Golang version to `v1.23` and fixed a [CVE-2025-21613](https://nvd.nist.gov/vuln/detail/CVE-2025-21613). For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.3.10) below. + +#### Breaking Changes + +This release has no breaking changes. + +

Dependencies

+##### Compatibility Matrix For Tyk Components + +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.3.10 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v2.0.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + + +
3rd Party Dependencies & Tools
+| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.23 | 1.23 | [Go plugins](/api-management/plugins/golang) must be built using Go 1.23 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +Given the time difference between your upgrade and the release of this version, we recommend customers verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations +There are no deprecations in this release + +#### Upgrade Instructions +If you are upgrading to 5.3.10, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.3.10) + - ```bash + docker pull tykio/tyk-dashboard:v5.3.10 + ``` +- Helm charts + - [tyk-charts v2.0.0](/developer-support/release-notes/helm-chart#200-release-notes) + +

Changelog

+##### Fixed + +
    +
  • + +Tyk Dashboard now runs on Golang 1.23, bringing security and performance improvements. Key changes include unbuffered Timer/Ticker channels, removal of 3DES cipher suites, and updates to X509KeyPair handling. Users may need to adjust their setup for compatibility. + +
    • +
+ +##### Security Fixes + +
    +
  • + +Fixed the following critical priority CVE identified in the Dashboard UI, providing increased protection and improved security: + - [CVE-2025-21613](https://nvd.nist.gov/vuln/detail/CVE-2025-21613) + +
    • +
  • + +- Fixed the following CVE: + - [CVE-2025-21614](https://nvd.nist.gov/vuln/detail/CVE-2025-21614) + +
    • +
+ +--- + +### 5.3.9 Release Notes + +#### Release Date 31 December 2024 + +#### Release Highlights +This release contains bug fixes. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.3.9) below. + +#### Breaking Changes + +This release has no breaking changes. + +

Dependencies

+##### Compatibility Matrix For Tyk Components +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.3.9 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v2.0.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + + +
3rd Party Dependencies & Tools
+{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.22 | 1.22 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +Given the time difference between your upgrade and the release of this version, we recommend customers verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations +We have deprecated the obsolescent `http_server_options.prefer_server_ciphers` configuration option. This legacy control no longer has any effect on the underlying library and users are advised to remove this setting from their configurations. + +#### Upgrade Instructions +If you are upgrading to 5.3.9, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.3.9) + - ```bash + docker pull tykio/tyk-dashboard:v5.3.9 + ``` +- Helm charts + - [tyk-charts v2.0.0](/developer-support/release-notes/helm-chart#200-release-notes) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Fixed +{/* This section should be a bullet point list of new features. Explain: + +- The purpose of the new feature +- How does the new feature benefit users? +- Link to documentation of the new feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} + +
    +
  • + + +Resolved a bug where clicking "Restore zooming to initial state" in the API Activity Dashboard would cause the graph to show blank instead of resetting to the initial zoom level. This fix ensures that users can now correctly restore the default zoom state in all charts on the Dashboard. + + + +This option has been marked as deprecated due to its obsolescence in the underlying library functions used by Tyk. Users are advised to remove this configuration from their setups as it no longer has any effect. + + + +Resolved CVE-2020-8911 by updating the Tyk Dashboard's email driver to use AWS SDK v2, addressing a medium-severity security vulnerability identified in version 5.3.8. This update ensures enhanced security for the Dashboard while maintaining functionality. + + +
    • +
+ +--- + +### 5.3.8 Release Notes + +#### Release Date 07 November 2024 + +#### Release Highlights +This release focuses mainly on bug fixes. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.3.8) below. + +#### Breaking Changes + +This release has no breaking changes. + +

Dependencies

+##### Compatibility Matrix For Tyk Components +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.3.8 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v2.0.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + + +
3rd Party Dependencies & Tools
+{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.22 | 1.22 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +Given the time difference between your upgrade and the release of this version, we recommend customers verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations + +This is an advanced notice that the dedicated External OAuth, OpenID Connect (OIDC) authentication options, and SQLite support will be deprecated starting in version 5.7.0. We recommend that users of the [External OAuth](/api-management/client-authentication#integrate-with-external-authorization-server-deprecated) and [OpenID Connect](/api-management/client-authentication#integrate-with-openid-connect-deprecated) methods migrate to Tyk's dedicated [JWT Auth](/basic-config-and-security/security/authentication-authorization/json-web-tokens) method. Please review your API configurations, as the Gateway logs will provide notifications for any APIs utilizing these methods. + +#### Upgrade Instructions +If you are upgrading to 5.3.8, please follow the detailed [upgrade instructions](#upgrading-tyk). + + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.3.8) + - ```bash + docker pull tykio/tyk-dashboard:v5.3.8 + ``` +- Helm charts + - [tyk-charts v2.0.0](/developer-support/release-notes/helm-chart#200-release-notes) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} +##### Added + +
    +
  • + +The UI now displays a deprecation notice for the dedicated [External OAuth](/api-management/client-authentication#integrate-with-external-authorization-server-deprecated) and [OpenID Connect (OIDC)](/api-management/client-authentication#integrate-with-openid-connect-deprecated) authentication mechanisms. This provides advanced notification that these authentication options will be deprecated in version 5.7.0. Users are advised to migrate to the [JWT Auth](/basic-config-and-security/security/authentication-authorization/json-web-tokens) method, which supports integration with both OAuth and OIDC providers, in preparation for future upgrade. + +
    • +
+ +##### Fixed +{/* This section should be a bullet point list of new features. Explain: + +- The purpose of the new feature +- How does the new feature benefit users? +- Link to documentation of the new feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + +Fixed an issue with the user group dropdown in the Dashboard UI, ensuring that all available user groups are displayed when creating a new user. + +
    • +
  • + +Fixed an issue in the Tyk OAS API Designer where Rate Limiting settings were not saved when Upstream Certificates were enabled. This fix ensures that both Rate Limits and Upstream Certificates configurations can now be saved together + +
    • +
+ +--- +### 5.3.7 Release Notes + +#### Release Date 22 October 2024 + +#### Release Highlights + +This is a version bump to align with Gateway v5.3.7, no changes have been implemented in this release. + +#### Breaking Changes + +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.3.7 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v2.0.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + + +
3rd Party Dependencies & Tools
+{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.22 | 1.22 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +#### Deprecations + +There are no deprecations in this release. + +

Upgrade instructions

+If you are upgrading to 5.3.7, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.3.7) +- ```bash + docker pull tykio/tyk-dashboard:v5.3.7 + ``` +- Helm charts + - [Tyk Charts v2.0.0](/developer-support/release-notes/helm-chart#200-release-notes) + +

Changelog

+No changes in this release. + +--- + +### 5.3.6 Release Notes + +#### Release Date 04 October 2024 + +#### Release Highlights + +This release primarily focuses on bug fixes. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.3.6) below. + +#### Breaking Changes +**Attention**: Please read this section carefully. +Docker images are now based on [distroless](https://github.com/GoogleContainerTools/distroless). No shell is shipped in the image. + +If moving from a version of Tyk older than 5.3.0 please read the explanation provided with [5.3.0 release](#TykOAS-v5.3.0). + +#### Deprecations +There are no deprecations in this release. + +#### Upgrade Instructions +When upgrading to 5.3.6, please follow the [detailed upgrade instructions](#upgrading-tyk). + +

Dependencies

+{/* Required. Use this section to announce the following types of dependencies compatible with the release: +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +3rd party dependencies and tools */} + +With MongoDB 4.4 reaching [EOL](https://www.mongodb.com/legal/support-policy/lifecycles) in February 2024, we can no longer guarantee full compatibility with this version of the database. If you are [using MongoDB](/planning-for-production/database-settings#mongodb) we recommend that you upgrade to a version that we have tested with, as indicated [below](#3rdPartyTools-v5.3.6). + + +With PostgreSQL v11 reaching [EOL](https://www.postgresql.org/support/versioning/) in November 2023, we can no longer guarantee full compatibility with this version of the database. If you are [using PostgreSQL](/planning-for-production/database-settings#postgresql) we recommend that you upgrade to a version that we have tested with, as indicated [below](#3rdPartyTools-v5.3.6). + + +##### Compatibility Matrix For Tyk Components +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.3.6 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v2.0.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + + +
3rd Party Dependencies & Tools
+{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.22 | 1.22 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.3.6) + - ```bash + docker pull tykio/tyk-dashboard:v5.3.6 + ``` +- Helm charts + - [tyk-charts v2.0](/developer-support/release-notes/helm-chart#200-release-notes) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Changed +{/* This should be a bullet-point list of updated features. Explain: +- Why was the update necessary? +- How does the update benefit users? +- Link to documentation of the updated feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + +The Tyk Dashboard has been upgraded from Golang 1.21 to Golang 1.22, bringing enhanced performance, strengthened security, and access to the latest features available in the new Golang release. + +
    • +
  • + +In this release, we've enhanced the security of the Tyk Dashboard image by changing the build process to support [distroless](https://github.com/GoogleContainerTools/distroless) containers. This significant update addresses critical CVEs associated with Debian, ensuring a more secure and minimal runtime environment. Distroless containers reduce the attack surface by eliminating unnecessary packages, which bolsters the security of your deployments. + +
    • +
+ +##### Fixed +{/* This section should be a bullet point list of new features. Explain: + +- The purpose of the new feature +- How does the new feature benefit users? +- Link to documentation of the new feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +Resolved an issue where the Gateway secret was inadvertently included in the log generated by the Dashboard for a call to the `/api/keys` endpoint when in debug mode. This issue has been fixed to prevent sensitive information from appearing in system logs. + + + +We have resolved an issue where the Keys page would display a blank screen if a key was associated with more than 10 policies. The UI has been fixed to display the page properly, regardless of the number of policies attached to a key. + + + +When working with Tyk Classic APIs, you cannot permit access to multiple versions of the same API from a single policy. We have fixed an issue in the Dashboard UI where users were able to attach multiple versions to a policy leading to an unusable policy. The UI now correctly prevents the addition of multiple versions of an API to a single policy. + + + +We have fixed an issue in the Dashboard UI when assigning multiple claim to policy mappings while configuring JWT auth for an API. The scope name was incorrectly recorded instead of the policy ID for the second and subsequent JWT scope mappings. The UI now correctly associates the defined claim with the appropriate policy, ensuring accurate JWT scope to policy mappings. + + +
    • + +
+ +##### Security Fixes +{/* This section should be a bullet point list that should be included when any security fixes have been made in the release, e.g. CVEs. For CVE fixes, consideration needs to be made as follows: +1. Dependency-tracked CVEs - External-tracked CVEs should be included on the release note. +2. Internal scanned CVEs - Refer to the relevant engineering and delivery policy. +For agreed CVE security fixes, provide a link to the corresponding entry on the NIST website. For example: +- Fixed the following CVEs: + - [CVE-2022-33082](https://nvd.nist.gov/vuln/detail/CVE-2022-33082) */} +
    +
  • + +Fixed the following high-priority CVEs identified in the Tyk Dashboard, providing increased protection against security vulnerabilities: +- [CVE-2024-6104](https://nvd.nist.gov/vuln/detail/CVE-2024-6104) + +
    • + +
+ +--- + +### 5.3.5 Release Notes + + +#### Release Date 26 September 2024 + + +#### Release Highlights + +This is a version bump to align with Gateway v5.3.5, no changes have been implemented in this release. + +#### Breaking Changes + +**Attention**: Please read this section carefully. + +There are no breaking changes in this release, however, if moving from a version of Tyk older than 5.3.0 please read the explanation provided with [5.3.0 release](#TykOAS-v5.3.0). + + +#### Deprecations + +There are no deprecations in this release. + + +#### Upgrade Instructions + +When upgrading to 5.3.5, please follow the [detailed upgrade instructions](#upgrading-tyk). + + +

Dependencies

+{/* Required. Use this section to announce the following types of dependencies compatible with the release: +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +3rd party dependencies and tools */} + +With MongoDB 4.4 reaching [EOL](https://www.mongodb.com/legal/support-policy/lifecycles) in February 2024, we can no longer guarantee full compatibility with this version of the database. If you are [using MongoDB](/planning-for-production/database-settings#mongodb) we recommend that you upgrade to a version that we have tested with, as indicated [below](#3rdPartyTools-v5.3.5). + + +With PostgreSQL v11 reaching [EOL](https://www.postgresql.org/support/versioning/) in November 2023, we can no longer guarantee full compatibility with this version of the database. If you are [using PostgreSQL](/planning-for-production/database-settings#postgresql) we recommend that you upgrade to a version that we have tested with, as indicated [below](#3rdPartyTools-v5.3.5). + + +##### Compatibility Matrix For Tyk Components + +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.3.5| MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v2.0.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + + +
3rd Party Dependencies & Tools
+{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.21 | 1.21 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +#### Downloads + +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.3.5) + - ```bash + docker pull tykio/tyk-dashboard:v5.3.5 + ``` +- Helm charts + - [tyk-charts v2.0.0](/developer-support/release-notes/helm-chart#200-release-notes) + +

Changelog

+ No changes in this release. + +--- + +### 5.3.4 Release Notes + +#### Release Date August 26 2024 + +#### Breaking Changes +**Attention**: Please read this section carefully. +There are no breaking changes in this release, however, if moving from a version of Tyk older than 5.3.0 please read the explanation provided with [5.3.0 release](#TykOAS-v5.3.0). + +#### Deprecations +There are no deprecations in this release. + +#### Upgrade Instructions +When upgrading to 5.3.4 please follow the [detailed upgrade instructions](#upgrading-tyk). + + +#### Release Highlights +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.3.4) below. + +

Dependencies

+{/* Required. Use this section to announce the following types of dependencies compatible with the release: +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +3rd party dependencies and tools */} + +With MongoDB 4.4 reaching [EOL](https://www.mongodb.com/legal/support-policy/lifecycles) in February 2024, we can no longer guarantee full compatibility with this version of the database. If you are [using MongoDB](/planning-for-production/database-settings#mongodb) we recommend that you upgrade to a version that we have tested with, as indicated [below](#3rdPartyTools-v5.3.3). + + +With PostgreSQL v11 reaching [EOL](https://www.postgresql.org/support/versioning/) in November 2023, we can no longer guarantee full compatibility with this version of the database. If you are [using PostgreSQL](/planning-for-production/database-settings#postgresql) we recommend that you upgrade to a version that we have tested with, as indicated [below](#3rdPartyTools-v5.3.3). + + +##### Compatibility Matrix For Tyk Components +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.3.4 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v1.4.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + + +
3rd Party Dependencies & Tools
+{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.21 | 1.21 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.3.4) + - ```bash + docker pull tykio/tyk-dashboard:v5.3.4 + ``` +- Helm charts + - [tyk-charts v1.4](/developer-support/release-notes/helm-chart#140-release-notes) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Fixed +{/* This section should be a bullet point list of new features. Explain: + +- The purpose of the new feature +- How does the new feature benefit users? +- Link to documentation of the new feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +Fixed API’s stats not being shown when adding 2 or more tags in the Activity page and using Postgres + + + +Fixed 429 status codes not being shown on the Activity page when using Postgres + + + +Fixed wrong graphs and incorrect requests counter on Portal when using Postgres + + + +Fixed Error Breakdown issues with dates (it was showing errors that happened on different dates than the one that was actually displayed) + + +
    • +
+ +--- +### 5.3.3 Release Notes + +#### Release Date August 2nd 2024 + +#### Breaking Changes +**Attention**: Please read this section carefully. +There are no breaking changes in this release, however, if moving from a version of Tyk older than 5.3.0 please read the explanation provided with [5.3.0 release](#TykOAS-v5.3.0). + +#### Deprecations +There are no deprecations in this release. + +#### Upgrade Instructions +When upgrading to 5.3.3 please follow the [detailed upgrade instructions](#upgrading-tyk). + +#### Release Highlights + +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.3.3) below. + +

Dependencies

+{/* Required. Use this section to announce the following types of dependencies compatible with the release: +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +3rd party dependencies and tools */} + +With MongoDB 4.4 reaching [EOL](https://www.mongodb.com/legal/support-policy/lifecycles) in February 2024, we can no longer guarantee full compatibility with this version of the database. If you are [using MongoDB](/planning-for-production/database-settings#mongodb) we recommend that you upgrade to a version that we have tested with, as indicated [below](#3rdPartyTools-v5.3.3). + + +With PostgreSQL v11 reaching [EOL](https://www.postgresql.org/support/versioning/) in November 2023, we can no longer guarantee full compatibility with this version of the database. If you are [using PostgreSQL](/planning-for-production/database-settings#postgresql) we recommend that you upgrade to a version that we have tested with, as indicated [below](#3rdPartyTools-v5.3.3). + + +##### Compatibility Matrix For Tyk Components +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.3.3 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v1.4.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + + +
3rd Party Dependencies & Tools
+{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.21 | 1.21 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.3.3) + - ```bash + docker pull tykio/tyk-dashboard:v5.3.3 + ``` +- Helm charts + - [tyk-charts v1.4](/developer-support/release-notes/helm-chart#140-release-notes) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Added +{/* This section should be a bullet point list of new features. Explain: + +- The purpose of the new feature +- How does the new feature benefit users? +- Link to documentation of the new feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + +Fixed an issue where nested API endpoints, such as '/test' and '/test/abc', might incorrectly apply middleware from the parent path to the nested path. The fix ensures that API endpoint definitions are correctly ordered, preventing this middleware misapplication and ensuring both the HTTP method and URL match accurately. + +
    • +
+ +--- + +##### Fixed +{/* This section should be a bullet point list that describes the issues fixed in the release. For each fixed issue explain: +- What problem the issue caused +- How was the issue fixed +- Link to (new) documentation created as a result of a fix. For example, a new configuration parameter may have been introduced and documented for the fix +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +Addressed an issue in SSO where user permissions were not correctly applied, ensuring the Save API button is visible to all users in the Dashboard UI. + + + +Resolved a bug causing the Dashboard UI to display a blank page when creating a key for an API using static mTLS with dynamic JWT authentication. + + + +Addressed an issue where the Dashboard displayed an empty page when accessing Activity by Endpoint information after upgrading to Tyk 5.3.1. Users can now see all necessary information. + + +
    • +
+ +--- + +### 5.3.2 Release Notes + + +#### Release Date 5th June 2024 + + +#### Breaking Changes +**Attention**: Please read this section carefully. + + +There are no breaking changes in this release, however if moving from a version of Tyk older than 5.3.0 please read the explanation provided with [5.3.0 release](#TykOAS-v5.3.0). + + +#### Deprecations +There are no deprecations in this release. + + +#### Upgrade Instructions +When upgrading to 5.3.2 please follow the [detailed upgrade instructions](#upgrading-tyk). + + +#### Release Highlights +This release primarily focuses on bug fixes. +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.3.2) below. + + +

Dependencies

+{/* Required. Use this section to announce the following types of dependencies compatible with the release: + + +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. + + +3rd party dependencies and tools */} + + +With MongoDB 4.4 reaching [EOL](https://www.mongodb.com/legal/support-policy/lifecycles) in February 2024, we can no longer guarantee full compatibility with this version of the database. If you are [using MongoDB](/planning-for-production/database-settings#mongodb) we recommend that you upgrade to a version that we have tested with, as indicated [below](#3rdPartyTools-v5.3.2). + +With PostgreSQL v11 reaching [EOL](https://www.postgresql.org/support/versioning/) in November 2023, we can no longer guarantee full compatibility with this version of the database. If you are [using PostgreSQL](/planning-for-production/database-settings#postgresql) we recommend that you upgrade to a version that we have tested with, as indicated [below](#3rdPartyTools-v5.3.2). + +##### Compatibility Matrix For Tyk Components +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.3.2 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v1.4.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + + +
3rd Party Dependencies & Tools
+{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.21 | 1.21 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x LTS | 12.x - 16.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.3.2) + - ```bash + docker pull tykio/tyk-dashboard:v5.3.2 + ``` +- Helm charts + - [tyk-charts v1.4](/developer-support/release-notes/helm-chart#140-release-notes) + + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Fixed +{/* This section should be a bullet point list that describes the issues fixed in the release. For each fixed issue explain: +- What problem the issue caused +- How was the issue fixed +- Link to (new) documentation created as a result of a fix. For example, a new configuration parameter may have been introduced and documented for the fix +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +Resolved an issue in the `api/usage` endpoint where the Dashboard with PostgreSQL integration returned unfiltered results when one valid tag was used. Corrected the need for duplicating the same parameter as a workaround for filtering by multiple tags. Results are now properly filtered as expected, improving the accuracy and reliability of analytics data. + + + +Modified default OPA rules to prevent unauthorized admins from modifying other admins' passwords, mitigating potential 'rogue admin' behavior. Tyk Dashboard clients using custom OPA rules should update their rule set accordingly. Contact your assigned Tyk representative for assistance. + + + +Resolved an issue in the GQL schema editor for Data Graphs, where users couldn't utilize the 'Import Schema' button. Now, it's possible to import files containing GQL schemas into the Dashboard. + + + +Adjusted wording in Tyk's Dashboard UI to ensure inclusivity and clarity, removing any potentially oppressive language. + + + +Fixed an issue where API Templates were not correctly assigned to Tyk Organizations allowing the potential for accidental sharing of secret data between Organizations through use of the incorrect template. + + + +Addressed a potential issue when working with Tyk OAS APIs where request context variables are automatically made available to relevant Tyk and custom middleware. We have introduced a control in the Tyk OAS API definition to disable this access if required. + + + +Fixed an issue in the api/usage endpoint where Dashboard+Postgres returned unfiltered results with one valid tag, requiring duplication of the parameter as a workaround for multiple tags. Analytics now correctly filter results as expected. + + +
    • +
+ +--- + +### 5.3.1 Release Notes + +#### Release Date 24 April 2024 + +#### Breaking Changes +**Attention**: Please read this section carefully. + +There are no breaking changes in this release, however if moving from a version of Tyk older than 5.3.0 please read the explanation provided with [5.3.0 release](#TykOAS-v5.3.0). + +#### Deprecations +There are no deprecations in this release. + +#### Upgrade Instructions +When upgrading to 5.3.1, please follow the [detailed upgrade instructions](#upgrading-tyk). + + +#### Release Highlights +This release primarily focuses on bug fixes. +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.3.1) below. + +

Dependencies

+{/* Required. Use this section to announce the following types of dependencies compatible with the release: + +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. + +3rd party dependencies and tools */} + +With MongoDB 4.4 reaching [EOL](https://www.mongodb.com/legal/support-policy/lifecycles) in February 2024, we can no longer guarantee full compatibility with this version of the database. If you are [using MongoDB](/planning-for-production/database-settings#mongodb) we recommend that you upgrade to a version that we have tested with, as indicated [below](#3rdPartyTools-v5.3.1). + +##### Compatibility Matrix For Tyk Components +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.3.1 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v1.3.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +
3rd Party Dependencies & Tools
+{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.21 | 1.21 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 11.x - 15.x LTS | 11.x - 15.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.3.1) +- ```bash + docker pull tykio/tyk-dashboard:v5.3.1 + ``` +- Helm charts + - [tyk-charts v1.3](/developer-support/release-notes/helm-chart#130-release-notes) + +

Changelog

+##### Fixed + +
    +
  • + + +Improved the behavior of the Dashboard when searching for users to avoid transmitting sensitive information (user email addresses) in the request query parameters. Deprecated the `GET` method for the `/api/users/search` endpoint in favor of a `POST` method with the same logic but with parameters supplied in the request body. + + + +As Tyk Dashboard and Tyk Classic Portal do not accept cross origin requests we have removed the `Access-Control-Allow-Credentials` header from Dashboard API responses to prevent any potential misuse of the header by attackers. This allows simplification of the web application's security configuration. + + + +Implemented a randomised delay to obscure login response times, mitigating brute force attacks that rely on response time analysis. + + + +Fixed a bug where a user was still able to log into an Organization on the Tyk Dashboard after that Organization had been deleted. Now, when an Organization is deleted, it will not be offered as an option when logging in. + + + +Fixed an issue where access keys could accidentally also be printed to the Dashboard's stdout when a call was made to `/api/keys` to retrieve the keys. This has now been suppressed. + + + +The Endpoint Designer did not correctly display a GraphQL policy's allow or block list if a wildcard character (`*`) was used in the list's definition. This has been fixed and now, if the wildcard (`*`) is present in the allow/block list definition, the UI correctly displays the list of allowed/blocked fields. + + + +Fixed an issue that was preventing the OPA editor from being visible using the keyboard shortcut when using Microsoft Windows. + + + +Fixed an issue where common keyboard shortcuts (Cmd + X, A, C, V) were not working correctly when configuring the URL field for a UDG data source. + + + +Fixed an issue in the Tyk OAS API Designer where there was no input validation of the OAuth Introspection URL. The Gateway reported an HTTP 400 error when attempting to save an API with an illegal value, however the API Designer did not guide the user to the source of the error. Now there is automatic validation of the text entered in the Introspection URL field. + + + +Fixed an issue with the text editor in the Tyk OAS API Designer where the cursor was misaligned with where characters would be entered. We have replaced the text editor module throughout the Tyk Dashboard to use a more modern, supported library. + + + +The 'Top 5 Errors by Graph' bar chart in the Activity by Graph dashboard experienced display issues with long graph names and sometimes showed empty bars. This has been resolved, and the chart now displays accurately. + + + +Fixed a bug where some Tyk Dashboard analytics screens stopped working when the analytics aggregates collection grew too large. + + + +In [Tyk 5.2.2](#Changelog-v5.2.2) we fixed an issue when using MongoDB and Tyk Security Policies where Tyk could incorrectly grant access to an API after that API had been deleted from the associated policy. This introduced an unintended side-effect for users of DocumentDB such that they were unable to delete APIs from the persistent storage. We identified that this was due to the use of the `$expr` operator in the solution - and discovered that this is supported by MongoDB but not by DocumentDB. We have now reimplemented the fix and removed the limitation introduced for DocumentDB users. + + + +Addressed a bug where clearing the API cache from the Tyk Dashboard failed to invalidate the cache in distributed data plane gateways. + + +
    • +
+ +--- + +### 5.3.0 Release Notes + +#### Release Date 5 April 2024 + +#### Deployment Options for Tyk Dashboard + +##### Tyk Cloud +Tyk Dashboard 5.3.0 is available on Tyk Cloud since 5th April 2024. + +##### Self-Managed +This release is ready for installation on your own infrastructure. + +#### Breaking Changes + +**Attention: Please read this section carefully.** + +
Tyk OAS APIs Compatibility Caveats
+This upgrade transitions Tyk OAS APIs out of [Early Access](/developer-support/release-types/early-access-feature). + +- **Out of Early access** + - This means that from now on, all Tyk OAS APIs will be backwards compatible and in case of a downgrade from 5.3.X to 5.3.0, the Tyk OAS API definitions will always work. +- **Not Backwards Compatible** + - Tyk OAS APIs in Tyk Dashboard v5.3.0 are not [backwards compatible](https://tinyurl.com/3xy966xn). This means that the new Tyk OAS API format used by Tyk Gateway/Dashboard v5.3.X does not work with older versions of Tyk Gateway/Dashboard, i.e. you cannot export these API definitions from a v5.3.X Tyk Dashboard and import to an earlier version. + - The upgrade of Tyk OAS API definitions is **not reversible**, i.e. you cannot use version 5.3.X Tyk OAS API definitions with an older version of Tyk Dashboard. + - This means that if you wish to downgrade or revert to your previous version of Tyk, you will need to restore these API definitions from a backup. Please go to the [backup](#upgrade-instructions) section for detailed instructions on backup before upgrading to v5.3.0. + - When using MongoDB as your persistent data store, Tyk OAS APIs from v5.3.0 require a minimum version of MongoDB 5.0. + - If you are not using Tyk OAS APIs, Tyk will maintain backward compatibility standards. +- **Not Forward Compatible** + - Tyk OAS API Definitions prior to v5.3.0 are not [forward compatible](https://tinyurl.com/t3zz88ep) with Tyk Gateway v5.3.X. + - This means that any Tyk OAS APIs created in any previous release (4.1.0-5.2.x) cannot work with the new Tyk Dashboard v5.3.X without being migrated to its latest format. +- **MDCB deployment and Tyk OAS APIs** + - Tyk OAS APIs created in Tyk v5.3.0 will not be loaded by the data plane gateways if you are using MDCB v2.4 or older. This means that MDCB users already working with Tyk OAS APIs **must wait for the release of MDCB v2.5** before upgrading Tyk Gateway and Dashboard to v5.3.0. + - Tyk Dashboard v5.3.0 managing Tyk OAS APIs requires Tyk Gateway v5.3.0 and MDCB v2.5.X for proper functionality. Older versions of Tyk Gateway may experience compatibility issues with Tyk OAS API definitions from v5.3.0. +- **After upgrade (the good news)** + - If you had a Tyk OAS API prior to v5.3.0 then Tyk Dashboard will automatically update the API definition to latest format. + - This means that you do not have to do anything to make your Tyk OAS APIs compatible with the new 5.3.0 release as Tyk Dashboard will take care of that during start-up. + - As mentioned above, this upgrade of Tyk OAS API definitions is irreversible. + +**Important:** Please go to the [backup](#upgrade-instructions) section for essential instructions on how to backup before upgrading to v5.3.0 + +

Dependencies

+{/* Required. Use this section to announce the following types of dependencies compatible with the release: + +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. + +3rd party dependencies and tools */} + +With MongoDB 4.4 reaching [EOL](https://www.mongodb.com/legal/support-policy/lifecycles) in February 2024, we can no longer guarantee full compatibility with this version of the database and recommend upgrading to a version that we have tested with, as indicated [below](#3rdPartyTools-v5.3.0). + +##### Compatibility Matrix For Tyk Components +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Dashboard Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.3.0 | MDCB v2.5 | MDCB v2.5 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v1.3.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +
3rd Party Dependencies & Tools
+{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [GoLang](https://go.dev/dl/) | 1.21 | 1.21 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard | +| [PostgreSQL](https://www.postgresql.org/download/) | 11.x - 15.x LTS | 11.x - 15.x | Used by Tyk Dashboard | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas)| + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations +There are no deprecations in this release. + +

Upgrade Instructions

+**The following steps are essential to follow before upgrading** + +1. For Self Managed deployments - Backup Your environment using the [usual guidance](/developer-support/upgrading#tyk-upgrade-guides-for-different-deployment-models) documented with every release (this includes backup config file and database). +2. For all deployments - Backup all your API definitions (Tyk OAS API and Classic Definitions): + - For Tyk Cloud deployments - To perform the backup please use our guide for [exporting APIs and policies](/developer-support/upgrading#backup-apis-and-policies). + - For Self-Managed deployments - To perform the backup please use [Tyk Sync](/api-management/automations/sync). +4. Performing the upgrade - For all deployments, follow the instructions in the [upgrade guide](#upgrading-tyk) when upgrading Tyk. + +#### Release Highlights + +We are excited to announce the release of 5.3.0, packed with new features, improvements and bug fixes to enhance your experience with Tyk Dashboard. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.3.0) below. + +##### Tyk OAS Feature Maturity + +Tyk OAS is now out of [Early Access](/developer-support/release-types/early-access-feature) as we have reached feature maturity. You are now able to make use of the majority of Tyk's features from your Tyk OAS APIs, so they are a credible alternative to the legacy Tyk Classic APIs. +From Tyk 5.3.0 we support the following features when using Tyk OAS APIs with Tyk Dashboard: +- Security + - All Tyk-supported client-gateway authentication methods including custom auth plugins + - Automatic configuration of authentication from the OpenAPI description + - Gateway-upstream mTLS + - CORS + +- API-level (global) middleware including: + - Response caching + - Custom plugins for PreAuth, Auth, PostAuth, Post and Response hooks + - API-level rate limits + - Request transformation - headers + - Response transformation - headers + - Service discovery + - Internal API + +- Endpoint-level (per-path) middleware including: + - Request validation - headers and body (automatically configurable from the OpenAPI description) + - Request transformation - method, headers and body + - Response transformation - headers and body + - URL rewrite and internal endpoints + - Mock responses (automatically configurable from the OpenAPI description) + - Response caching + - Custom Go Post-Plugin + - Request size limit + - Virtual endpoint + - Allow and block listing + - Do-not-track + - Circuit breakers + - Enforced timeouts + - Ignore authentication + +- Observability + - Open Telemetry tracing + - Detailed log recording (include payload in the logs) + - Do-not-track endpoint + +- Governance + - API Versioning + - API Categories + - API Ownership + +##### API Templates + +Exclusively for Tyk OAS APIs, we are pleased to announce the introduction of API Templates: an API governance feature provided to streamline the process of creating APIs. An API template is an asset managed by Tyk Dashboard that is used as the starting point - a blueprint - from which you can create a new Tyk OAS API definition. With templates you can standardize configuration of your APIs more easily, combining your service-specific OpenAPI descriptions with enterprise requirements such as health endpoints, caching and authorization. + +##### Enhanced User Permissions + + Introducing allow list in field-based permissions via the Dashboard specifically tailored for GraphQL APIs. Users can now define granular access control for API key holders based on types and fields from a GraphQL schema. This feature enhances security and flexibility in managing API access, providing a more tailored and secure experience for users. + + ##### Global Header Management + + We've introduced global header management specifically for UDG, simplifying header configuration across all data sources. Users can now effortlessly add, adjust, and delete multiple global headers, ensuring consistency and efficiency throughout API management, ultimately saving developers time and effort + +##### GraphQL focused analytics +We have made the first step towards bringing our users GraphQL-focused monitoring capabilities. Users can now gain valuable insights into error trends and usage patterns for GraphQL APIs, when storing graph analytics in SQL databases. With the addition of popularity and error bar charts, users can delve deeper into their data, facilitating optimization and troubleshooting efforts. + +##### Redis v7.x Compatibility +We have upgraded Redis driver [go-redis](https://github.com/redis/go-redis) to v9. Subsequently, Tyk 5.3 is compatible with Redis v7.x. + +##### MongoDB v7.0.x Compatibility +We have upgraded `mongo-go` driver to [mongo-go v1.13.1](https://github.com/mongodb/mongo-go-driver/releases/tag/v1.13.1). It allows us to benefit from the bug fixes and enhancements released by MongoDB. We have also tested that both Tyk 5.0.x+ and Tyk 5.3 are compatible with MongoDB v7.0.x. + +#### Downloads +- [Docker Image to pull](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=&page_size=&ordering=&name=v5.3.0) +- ```bash + docker pull tykio/tyk-dashboard:v5.3.0 + ``` +- Helm charts + - [tyk-charts GH Repo](https://github.com/TykTechnologies/tyk-charts/releases) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. + +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Added +{/* This section should be a bullet point list of new features. Explain: + +- The purpose of the new feature +- How does the new feature benefit users? +- Link to documentation of the new feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +The following features have been added in 5.3.0 to bring Tyk OAS to feature maturity: + - Detailed log recording (include payload in the logs) + - Enable Open Telemetry tracing + - API-level header transforms (request and response) + - Endpoint-level cache + - Circuit breakers + - Track endpoint logs for inclusion in Dashboard aggregated data + - Do-not-track endpoint + - Enforced upstream timeouts + - Configure endpoint as Internal (not available externally) + - URL rewrite + - Per-endpoint request size limit + - Request transformation - method, header + - Response transformation - header + - Custom domain certificates + + + +Support for field-based permissions allow list has been added in the Dashboard. Users can now define which types and fields from a GraphQL schema an API key holder can access by simply putting a tick next to them in the policy/key definition screens. + + + +In this update, we've added support for API Categories for Tyk OAS APIs in the Tyk Dashboard, enhancing portfolio management by enabling efficient categorization and organization of APIs. + + + +We’ve extended the API ownership capabilities of Tyk Dashboard to Tyk OAS APIs. This feature allows you to manage visibility of APIs deployed on the Dashboard, streamlining governance processes and enhancing internal security. + + + +Extended Tyk Dashboard API to support CRUD operations on API Templates, enabling users to create, apply, and manage templates programmatically. + +Added Dashboard UI functionality for creation and management of API Templates, including the ability to create templates from existing Tyk OAS APIs. You can apply templates during API creation, including when importing OpenAPI documents. Access to API templates is controlled through the introduction of a new user permission. + + + +Now you can import the OpenAPI description from a file or URL when creating or updating your Tyk OAS APIs. + + + +Access the new Global Header Management feature directly through the Headers Management tab. Swiftly add and configure multiple global headers or remove them with a single click, ensuring they're forwarded to all GraphQL data sources. This enhancement streamlines header management, providing a more user-friendly experience. + + + +We’ve enabled basic Graph monitoring in the Dashboard. Due to the specificity of GQL APIs, monitoring them as you would REST, is not enough. One endpoint vs multiple endpoints, multiple queries/mutations vs HTTP methods, errors that happen not only in HTTP layer but also come back in response body - that all makes monitoring GQL slightly more complex than just looking at request and error rates. + +A new section of the Dashboard offers the following information: +- top 5 most popular graphs and operations requested within them within a specified period of time +- top 5 graphs with errors within a specified period of time +- summary of number of requests, number of successful responses, number of errors, average latency and last access date within a specified period of time for all graphs + + + +Tyk 5.3 integrates with [storage v1.2.2](https://github.com/TykTechnologies/storage), which updated mongo-go driver we use from v1.11.2 to [mongo-go v1.13.1](https://github.com/mongodb/mongo-go-driver/releases/tag/v1.13.1). It allows us to benefit from the bug fixes and enhancements released by MongoDB. We have also tested that Tyk 5.0.x+ is compatible with MongoDB v7.0.x + + + +Tyk 5.3 refactors Redis connection logic by using [storage v1.2.2](https://github.com/TykTechnologies/storage/releases/tag/v1.2.2), which integrates with [go-redis](https://github.com/redis/go-redis) v9. Subsequently, support now exists for Redis v7.0.x. + + +
    • +
+ + +##### Changed +{/* This should be a bullet-point list of updated features. Explain: + +- Why was the update necessary? +- How does the update benefit users? +- Link to documentation of the updated feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +Every Dashboard menu item can now be flagged as a favorite so that it is pinned to the top of the menu navigation bar for easier access. We've also made a few changes in styling, so that the navigation menu is nicer to look at. + + + +We have moved data source header management to a separate tab, so that it is easy to configure global headers that will be forwarded to all data sources by default. The data source configuration screen displays all headers that will be sent with the upstream request in read-only mode now and changes can be made by switching to Headers Management tab. + + + +We have updated Tyk Dashboard to use Go 1.21, matching the upgrade in Tyk Gateway 1.21. Remember to recompile any custom Go plugins with the matching version of Go to avoid incompatibility problems. + + + +If internal TIB is enabled in Dashboard and the TYK_IB_SESSION_SECRET environment variable is not set, it will be default to Dashboard admin_secret. It provides better security and user experience because SSO flow would not work if TYK_IB_SESSION_SECRET is not set. + + + +Tyk uses `mongo-go` as the default MongoDB driver from v5.3. This provides support for MongoDB 4.4.x, 5.0.x, 6.0.x and 7.0.x. If you are using older MongoDB versions e.g. 3.x, please set MongoDB driver to `mgo`. The [MongoDB supported versions](/planning-for-production/database-settings#supported-versions) page provides details on how to configure MongoDB drivers in Tyk. + + +
    • +
+ +##### Fixed +{/* This section should be a bullet point list that describes the issues fixed in the release. For each fixed issue explain: + +- What problem the issue caused +- How was the issue fixed +- Link to (new) documentation created as a result of a fix. For example, a new configuration parameter may have been introduced and documented for the fix +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +We fixed an issue where OPA rules were preventing users from importing an OpenAPI document as a UDG data source using the /api/data-graphs/data-sources/import endpoint. The endpoint has now been included into the correct user permission group and will be accessible for users who have `api:write` permissions. + + + +Fixed an issue where applying security policies to large numbers of APIs took a long time. We’ve implemented bulk processing in the validation step at the api/portal/policies/POLICY_ID endpoint, resulting in an 80% reduction in the time taken to apply a policy to 2000 APIs. + + + +Moved all HTML inline scripts to their own script files, to accommodate the content security policies that have been enabled, to increase security. + + + +Fixed an issue when importing reasonably large OpenAPI documents via the Dashboard would fail due to MongoDB storage limitation of 16 MB per document. + + + +Relaxed the strict validation for mock response so that the `Description` field is now optional for `response`, `responses` and `schema` within the OpenAPI description. Automatically configuring mock responses when using [Tyk OAS APIs](/api-management/traffic-transformation/mock-response#mock-responses-using-openapi-metadata) is now even easier. + + + +For Classic Portal cookies and Dashboard, use `SameSite = SameSiteLaxMode` so that SSO flows can be performed + + + +Remove the following unnecessary warning output when users use the `tyk-dashboard --version` command to check dashboard version. +> `WARN toth/tothic: no TYK_IB_SESSION_SECRET environment variable is set. The default cookie store is not available and any calls will fail. Ignore this warning if you are using a different store.` + + +
    • +
+ +##### Security Fixes +{/* This section should be a bullet point list that should be included when any security fixes have been made in the release, e.g. CVEs. For CVE fixes, consideration needs to be made as follows: +1. Dependency-tracked CVEs - External-tracked CVEs should be included on the release note. +2. Internal scanned CVEs - Refer to the relevant engineering and delivery policy. + +For agreed CVE security fixes, provide a link to the corresponding entry on the NIST website. For example: + +- Fixed the following CVEs: + - [CVE-2022-33082](https://nvd.nist.gov/vuln/detail/CVE-2022-33082) */} +
    +
  • + +Fixed the following high priority CVEs identified in the Tyk Dashboard, providing increased protection against security vulnerabilities: +- [CVE-2023-39325](https://nvd.nist.gov/vuln/detail/CVE-2023-39325) +- [CVE-2023-45283](https://nvd.nist.gov/vuln/detail/CVE-2023-45283) + +
    • +
+ + + +--- + +{/* */} + + + + +## 5.2 Release Notes +### 5.2.5 Release Notes + +**Release Date 19 Dec 2023** + +#### Breaking Changes + +**Attention**: Please read carefully this section. We have two topics to report: + +#### Early Access Features: +Please note that the `Tyk OAS APIs` feature, currently marked as *Early Access*, is subject to breaking changes in subsequent releases. Please refer to our [Early Access guide](/developer-support/release-types/early-access-feature) for specific details. Upgrading to a new version may introduce changes that are not backwards-compatible. Downgrading or reverting an upgrade may not be possible resulting in a broken installation. + +Users are strongly advised to follow the recommended upgrade instructions provided by Tyk before applying any updates. + +#### Deprecations +There are no deprecations in this release. + +#### Upgrade Instructions +If you are using a 5.2.x version, we advise you to upgrade ASAP to this latest release. If you are on an older version, you should skip 5.2.0 and upgrade directly to this release. Go to the [Upgrading Tyk](#upgrading-tyk) section for detailed upgrade Instructions. + +#### Release Highlights +Dashboard 5.2.5 was version bumped only, to align with Gateway 5.2.5. Subsequently, no changes were encountered in release 5.2.5. Gateway 5.2.5 was a critical patch release. For further information please see the release notes for Gateway [v5.2.5](/developer-support/release-notes/gateway) + +#### Downloads +- [Docker image to pull](https://hub.docker.com/layers/tykio/tyk-gateway/v5.2.5/images/sha256-c09cb03dd491e18bb84a0d9d4e71177eb1396cd5debef694f1c86962dbee10c6?context=explore) + +

Changelog

+Since this release was version bumped only to align with Gateway v5.2.5, no changes were encountered in this release. + +--- + + + +--- + +### 5.2.4 Release Notes + +**Release Date 7 Dec 2023** + +#### Breaking Changes + +**Attention**: Please read carefully this section. We have two topics to report: + +#### Early Access Features: +Please note that the `Tyk OAS APIs` feature, currently marked as *Early Access*, is subject to breaking changes in subsequent releases. Please refer to our [Early Access guide](/developer-support/release-types/early-access-feature) for specific details. Upgrading to a new version may introduce changes that are not backwards-compatible. Downgrading or reverting an upgrade may not be possible resulting in a broken installation. + +Users are strongly advised to follow the recommended upgrade instructions provided by Tyk before applying any updates. + +#### Deprecations +There are no deprecations in this release. + +#### Upgrade Instructions +If you are using a 5.2.x version, we advise you to upgrade ASAP to this latest release. If you are on an older version, you should skip 5.2.0 and upgrade directly to this release. Go to the [Upgrading Tyk](#upgrading-tyk) section for detailed upgrade Instructions. + +#### Release Highlights +This release primarily focuses on bug fixes. +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.2.4) below. + +#### Downloads +- [Docker image to pull](https://hub.docker.com/layers/tykio/tyk-dashboard/v5.2.4/images/sha256-8862e98c6ffd67d47b496275b228f4f8faae4359b9c8e42bcd8bd8a47d0c45e4?context=explore) + +

Changelog

+##### Fixed + +
    +
  • + + +Fixed two UI issues with the [OPA editor](/api-management/dashboard-configuration#using-the-open-policy-agent-in-the-dashboard) in the Tyk Dashboard to improve experience when using this feature. Scrolling beyond the end of the OPA window does not now start to scroll the API Designer window, and minimizing then re-expanding the OPA editor no longer limits the text to one line. + + + +Fixed minor issues in the Dashboard UI when configuring the user access controls for the Identity Management (TIB) and Real Time Notifications permissions. + + + +Fixed an issue where TLS 1.3 was not offered as an option in the "Minimum TLS version" dropdown in the API Designer. Also we gave better (human readable) names to the options, like TLS 1.0, TLS 1.1 etc. instead of their corresponding numbers 769, 770 etc. + + + +Fixed a situation where Tyk Dashboard could panic when using the mongo-go driver. + + + +Improved the error message that is returned when user tries to update a Tyk OAS API using a Tyk Classic API endpoint when `allow_unsafe_oas` is not enabled. + + +
    • +
+ +##### Added + +
    +
  • + +This prints the release version, git commit, Go version used, architecture and other build details. + +
    • +
+ +--- + +### 5.2.3 Release Notes + +**Release Date 21 Nov 2023** + +#### Breaking Changes + +**Attention**: Please read carefully this section. We have two topics to report: + +#### Early Access Features: +Please note that the `Tyk OAS APIs` feature, currently marked as *Early Access*, is subject to breaking changes in subsequent releases. Please refer to our [Early Access guide](/developer-support/release-types/early-access-feature) for specific details. Upgrading to a new version may introduce changes that are not backwards-compatible. Downgrading or reverting an upgrade may not be possible resulting in a broken installation. + +Users are strongly advised to follow the recommended upgrade instructions provided by Tyk before applying any updates. + +#### Deprecations +There are no deprecations in this release. + +#### Upgrade Instructions +If you are using a 5.2.x version, we advise you to upgrade ASAP to this latest release. If you are on an older version, you should skip 5.2.0 and upgrade directly to this release. Go to the [Upgrading Tyk](#upgrading-tyk) section for detailed upgrade Instructions. + +#### Release Highlights +This release primarily focuses on bug fixes. +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.2.3) below. + +#### Downloads +- [Docker image to pull](https://hub.docker.com/layers/tykio/tyk-dashboard/v5.2.3/images/sha256-7d61ed3ee3f03ff0e2f91be71a9113b90ef6637b1cef1f30d4c3e04ead09fa6a?context=explore) + +

Changelog

+##### Fixed + +
    +
  • + + +Fixed an issue where the [OPA editor](/api-management/dashboard-configuration#using-the-open-policy-agent-in-the-dashboard) was not resizable. The fix ensures the floating OPA editor is now resizable and the resizing operation is smooth, improving user experience. + + + +Fixed an issue where the [User Search](/api-management/user-management#search-users) was not working unless the full email address was entered. The fix restores the functionality of showing suggestions for names as they are typed in, improving user experience and search efficiency. + + + +Fixed an issue where Dashboard 4.1.0+ was unable to retrieve certificates from a Tyk Gateway with a version lower than 4.1.0. This was due to a change made in the 4.1 versions relating to the way certificate details are retrieved in the dashboard; in the newer versions, we can view more details of the certificates. Now you can use Tyk Dashboard with any version of the Tyk Gateway and still retrieve and view certificate details; the fix ensures smooth staged upgrades and prevents potential issues for customers who have weeks or months between upgrading components. + + + +Fixed an issue in the Tyk Classic API Designer where if you changed the protocol for an API (for example from HTTP to HTTPS) then the authentication mechanism would be automatically set to Authentication Token. + + + +Fixed an issue in the Classic API Designer where the 'use_standard_auth' value was constantly reverting to 'true' when editing an API with an [external OAuth flow](/api-management/client-authentication#integrate-with-external-authorization-server-deprecated). This fix ensures the 'use_standard_auth' value remains consistent, enabling the use of external OAuth via the Raw API editor. + + + +Fixed an issue with failed GraphQL subscriptions between the upstream and the Dashboard. When an upstream subscription was disconnected and later reconnected, the UI did not update to reflect the reconnection, preventing the seamless consumption of messages. Now the Dashboard can continue consuming messages after upstream reconnects. + + +
    • +
+ +--- + +### 5.2.2 Release Notes + +**Release Date 31 Oct 2023** + +#### Breaking Changes + +**Attention**: Please read carefully this section. We have two topics to report: + +#### Early Access Features: +Please note that the `Tyk OAS APIs` feature, currently marked as *Early Access*, is subject to breaking changes in subsequent releases. Please refer to our [Early Access guide](/developer-support/release-types/early-access-feature) for specific details. Upgrading to a new version may introduce changes that are not backwards-compatible. Downgrading or reverting an upgrade may not be possible resulting in a broken installation. + +Users are strongly advised to follow the recommended upgrade instructions provided by Tyk before applying any updates. + +#### Deprecations +There are no deprecations in this release. + +#### Upgrade Instructions +If you are using a 5.2.x version, we advise you to upgrade ASAP to this latest release. If you are on an older version, you should skip 5.2.0 and upgrade directly to this release. Go to the [Upgrading Tyk](#upgrading-tyk) section for detailed upgrade Instructions. + +#### Release Highlights +This release primarily focuses on bug fixes. +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.2.2) below. + +#### Downloads +- [Docker image to pull](https://hub.docker.com/layers/tykio/tyk-dashboard/v5.2.2/images/sha256-c6e701e270ebb2fed815483723375c454d0479ae41b5be2e1a6198b8d1e1a154?context=explore) + +

Changelog

+##### Added + +
    +
  • + +Added a new Dashboard configuration option `allow_unsafe_oas`. This permits the modification of Tyk OAS APIs via the Tyk Classic API endpoints. This is not a recommended action due to the risk of inconsistent behavior and potential for breaking changes while Tyk OAS is in [Early Access](/developer-support/release-types/early-access-feature). This is provided for early adopters and will be deprecated later, once Tyk OAS reaches full maturity. + +
    • +
+ +##### Fixed +
    +
  • + + +Fixed an issue when using MongoDB and [Tyk Security Policies](/api-management/policies#what-is-a-security-policy) where Tyk could incorrectly grant access to an API after that API had been deleted from the associated policy. This was due to the policy cleaning operation that is triggered when an API is deleted from a policy in a MongoDB installation. With this fix, the policy cleaning operation will not remove the final (deleted) API from the policy; Tyk recognizes that the API record is invalid and denies granting access rights to the key. + + + +Fixed an issue in the Tyk Dashboard where a user might not correctly inherit all permissions from their user group, and could incorrectly be granted visibility of Identity Management. + + + +Fixed an issue where Tyk would not store the *Policy Id* in the *API Definition* for a policy that did not exist. When using *JWT Authentication*, the *JWT Default Policy Id* is stored in the *API Definition*. If this policy had not been created in Tyk at the time the *API Definition* was created, Tyk Dashboard would invalidate the field in the *API Definition*. When the policy was later created, there would be no reference to it from the *API Definition*. This was a particular issue when using *Tyk Operator* to manage the creation of assets on Tyk. + + + +Fixed an issue in the Dashboard *Service Uptime* page where the number of success hits was being incorrectly reported as the total number of hits, inclusive of failures. After this fix, the *Success Column* displays only the number of success hits. + + + +Fixed the following high priority CVEs identified in the Tyk Dashboard, providing increased protection against security vulnerabilities: + +- [CVE-2022-33082](https://nvd.nist.gov/vuln/detail/CVE-2022-33082) +- [CVE-2022-28946](https://nvd.nist.gov/vuln/detail/CVE-2022-28946) +- [CVE-2021-23409](https://nvd.nist.gov/vuln/detail/CVE-2021-23409) +- [CVE-2021-23351](https://nvd.nist.gov/vuln/detail/CVE-2021-23351) +- [CVE-2023-28119](https://nvd.nist.gov/vuln/detail/CVE-2023-28119) +- [CVE-2022-21698](https://nvd.nist.gov/vuln/detail/CVE-2022-21698) +- [CVE-2020-26160](https://nvd.nist.gov/vuln/detail/CVE-2020-26160) +- [CVE-2019-19794](https://nvd.nist.gov/vuln/detail/CVE-2019-19794) +- [CVE-2010-0928](https://nvd.nist.gov/vuln/detail/CVE-2010-0928) +- [CVE-2007-6755](https://nvd.nist.gov/vuln/detail/CVE-2007-6755) +- [CVE-2018-5709](https://nvd.nist.gov/vuln/detail/CVE-2018-5709) + + + +Fixed an issue encountered with *Azure SAML2.0 Identity Provider* that was preventing users from authenticating. + + + +Fixed an issue encountered with the *API Designer* where fields defined in *Uptime_Tests.Check_List* were not correctly handled. Uptime tests can now be configured for *Tyk Classic APIs* using the *Raw API Definition* editor. + + + +Fixed a security vulnerability with the Tyk Dashboard API where the `api_version` and `api_id` query parameters were potential targets for SQL injection attack. + + +
    • +
+ +##### Updated + +
    +
  • + +On Tyk Dashboard's Licensing Statistics screen, we have renamed the License Limit to License Entitlement. We've also improved the experience when there is no limit in the license by hiding the License Entitlement line if no limit is set. + +
    • +
+ +--- + +### 5.2.1 Release Notes + +**Release Date 10 Oct 2023** + +#### Breaking Changes + +#### Early Access Features: +Please note that the `Tyk OAS APIs` feature, currently marked as *Early Access*, is subject to breaking changes in subsequent releases. Please refer to our [Early Access guide](/developer-support/release-types/early-access-feature) for specific details. Upgrading to a new version may introduce changes that are not backwards-compatible. Downgrading or reverting an upgrade may not be possible result in a broken installation. + +Users are strongly advised to follow the recommended upgrade instructions provided by Tyk before applying any updates. + +#### Deprecations +There are no deprecations in this release. + +#### Upgrade Instructions +If you are on a 5.2.0 we advise you to upgrade ASAP and if you are on an older version skip 5.2.0 and upgrade directly to this release. Go to the [Upgrading Tyk](#upgrading-tyk) section for detailed upgrade Instructions. + +#### Release Highlights +This release primarily focuses on bug fixes. +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.2.0) below. + +#### Downloads +- [Docker image to pull](https://hub.docker.com/layers/tykio/tyk-dashboard/v5.2.1/images/sha256-2f9d8af0e57f7fe4afb618dcf34772c001104dc0ec62a27541d12dc9ae90d5c8?context=explore) + +

Changelog

+##### Added + +
    +
  • + +Added support to Tyk Dashboard API so that Tyk Sync can fully support Tyk OAS API Definitions; this will be enabled from Tyk Sync version 1.4.1. + +
    • +
+ +##### Fixed + +
    +
  • + +Fixed a bug in the *Tyk Dashboard* UI where pagination in the APIs screen was breaking for API of type GraphQL/UDG. This resulted in the page failing to load data and displaying a 'No data to display' message. + +
    • + +
  • + +Fixed an issue where the 'Add GraphQL Operation' checkbox in the GraphQL data source configuration screen couldn't be disabled, even when no operation was added. Now, its state can be adjusted based on the presence of GraphQL operations and variables. + +
    • +
+ +--- + +### 5.2.0 Release Notes + +**Release Date 29 Sep 2023** + +#### Breaking Changes + +**Attention**: Please read carefully this section. We have two topics to report: + +#### Early Access Features: +Please note that the `Tyk OAS APIs` feature, currently marked as *Early Access*, is subject to breaking changes in subsequent releases. Please refer to our [Early Access guide](/developer-support/release-types/early-access-feature) for specific details. Upgrading to a new version may introduce changes that are not backwards-compatible. Downgrading or reverting an upgrade may not be possible resulting in a broken installation. + +Users are strongly advised to follow the recommended upgrade instructions provided by Tyk before applying any updates. + +#### Deprecations +There are no deprecations in this release. + +#### Release Highlights + +We're thrilled to bring you some exciting enhancements and crucial fixes to improve your experience with Tyk Dashboard. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.2.0) below. + +Configure Caching Timeouts Per API Endpoint and Enable Advanced Caching Options From Within Dashboard + +We’ve added the ability to [configure](/api-management/response-caching#configuring-the-middleware-in-the-tyk-oas-api-definition) per-endpoint timeouts for Tyk’s response cache, giving you increased flexibility to tailor your APIs to your upstream services. While doing this, we’ve also fixed a longstanding issue within the *Tyk Dashboard* so that you can configure more of the [advanced caching](/api-management/response-caching#configuring-the-middleware-in-the-api-designer) options from within the UI. + +##### Added Body Transform Middleware to Tyk OAS API Definition + +With this release, we are adding the much requested *Body Transformations* to *Tyk OAS API Definition*. You can now [configure](/api-management/gateway-config-tyk-oas#transformbody) middleware for both [request](/api-management/traffic-transformation/request-body) and [response](/api-management/traffic-transformation/response-body) *Body Transformations* and - as a *Tyk Dashboard* user - you’ll be able to do so from within our simple and elegant API Designer tool. Visually test and preview *Body Transformations* from within the API Designer. + +##### Track Usage Of License APIs, Gateways And Distributed Data Planes Over Time + +Within the Dashboard UI, we’ve enhanced the *Licensing* information page, so that you can visualise your usage of licensed APIs, *Gateways* and distributed *Data Planes* over time. This allows the visualisation of deployed and active APIs using a range of different types of interactive charts. + + +#### Downloads + +Tyk Dashboard 5.2 - [Docker image to pull](https://hub.docker.com/layers/tykio/tyk-dashboard/v5.2.0/images/sha256-28ff62e1e1208d02fec44cf84c279a5f780207ccbb7c3bdef23d1bf8fc6af3b8?context=explore) + + +#### API Changes + +The following is a list of API changes in this release. Please visit our [Postman collection](https://www.postman.com/tyk-technologies/workspace/tyk-public-workspace/overview) for further information on our APIs. + +
    +
  • + +Added a new [endpoint](/tyk-dashboard-api), */system/stats*, to provide insight and operational statistics on total and active APIs deployed. The endpoint's flexible date filtering options, equip users to obtain comprehensive insights into usage trends. + +
    • +
+ + +

Changelog

+##### Added + +
    +
  • + + +Added support for API developers to easily [configure](/api-management/gateway-config-tyk-oas#transformbody) both request and response *Body Transformations* for more precise data management when working with *Tyk OAS* APIs. Define input data, craft transformation templates and test them against specific inputs for reliable customization. + + + +Adding a new [data source](/api-management/data-graph#3-configure-datasource-details) is simpler when working with *UDG*. The default value for the *data source name* is pre-filled, saving time. The *data source name* is pre-filled in the format *fieldName_typeName*, with *typeName* being the name of any GraphQL type. + + + +Added a new [endpoint](/tyk-dashboard-api), */system/stats*, to provide insight and operational statistics on total and active APIs deployed. The endpoint's flexible date filtering options, equip users to obtain comprehensive insights into usage trends. + + +
    • +
+ +##### Changed +
    +
  • + + +Improved the flow when creating an API within the *API Designer* so that you remain on the same screen after saving. This means you can continue editing without having to navigate back to the screen to make subsequent changes. + + + +Updated the [screen](/api-management/data-graph#connect-datasource) for configuring and saving *UDG* data sources. The *Save* button has been replaced with *Save & Update API* button and users no longer need to click *Update* at the top of the screen to persist changes. Saving a *UDG* data source is now simpler and quicker. + + + +Updated the *Dashboard* with enhanced API usage monitoring. Users now benefit from an insightful chart on the *Licensing Statistics* page, detailing: maximum, minimum and average counts of created and active APIs. Flexible date filtering, license limit reference lines and the ability to toggle between line and bar graphs empower users to monitor usage effortlessly, ensuring license adherence. + + + +A new chart has been introduced on the *License Statistics* page that presents the number of deployed *Data Planes*. This addition enables users to easily monitor their *Data Plane* usage and nearness to their contract limits. + + +
    • +
+ +##### Fixed + +
    +
  • + + +Fixed an issue where *advanced_cache_config* data was absent in the *Raw Editor*. This fix now ensures that *advanced_cache_config* can be configured. Furthermore, API modifications in the *Designer* no longer lead to data loss, safeguarding cache configuration consistency. The UI now offers a clear view of advanced cache settings, including the new *Timeout* field and *Cache* response codes fields. + + + +Fixed an issue with *JWT claim names* containing spaces. Previously 403 errors were raised when using tokens containing such claims. + + + +Fixed an issue where *popular endpoints* data was not displayed in *Tyk Dashboard* with *SQL aggregated analytics* enabled. Users can now view *popular endpoints* when viewing *Traffic Activity* per API or filtering by API with *SQL aggregated analytics* enabled. + + + +Fixed a potential security vulnerability where *static* or *dynamic mTLS* requests with expired certificates could be proxied upstream. + + + +Fixed an issue in the *API Activity* dashboard where users were unable to view request analytics for a specific date. Subsequently, users can now make informed decisions based on access to this data. + + + +Fixed an issue where the [Enforced Timeout](/planning-for-production/ensure-high-availability/enforced-timeouts) configuration parameter of an API endpoint accepted negative values, without displaying validation errors. With this fix, users receive clear feedback and prevent unintended configurations. + + + +Fixed an issue in *Tyk Dashboard* where duplicate APIs could be created with the same names and listen paths if you clicked multiple times on the *save* button in the API Designer. Now, this is not possible anymore and there is no risk of creating multiple APIs with the same name. + + + +Fixed an issue with *MongoDB* connection strings. To ensure consistent compatibility with both *mgo* and *mongo-go* drivers, users should now utilize URL-encoded values within the *MongoDB* connection string's username and password fields when they contain characters like "?", "@`". This resolves the need for different handling across *MongoDB* drivers. + + +
    • +
+ +--- + +## 5.1 Release Notes +### 5.1.0 Release Notes + +#### Release Date 23 June 2023 + +#### Breaking Changes +**Attention warning*: Please read carefully this section. We have two topics to report: + +###### Golang Version upgrade +Our Dashboard is using [Golang 1.19](https://tip.golang.org/doc/go1.19) programming language starting with the 5.1 release. This brings improvements to the code base and allows us to benefit from the latest features and security enhancements in Go. Don’t forget that, if you’re using GoPlugins, you'll need to [recompile](/api-management/plugins/golang#upgrading-your-tyk-gateway) these to maintain compatibility with the latest Gateway. + +###### Tyk OAS APIs +To provide a superior experience with OAS APIs, we have made some changes which include various security fixes, improved validation etc. Upgrading to v5.1 from v4.x.x may be irreversible, rollback to v4.x.x could break your OAS API definitions. For this reason, we recommend making a database backup so you can always restore from the backup (of v4.X.X) in case you encounter a problem during the upgrade. Please refer to our guides for detailed information on [upgrading Tyk](/developer-support/upgrading) and [how to back up tyk](/developer-support/faq#tyk-configuration) + +#### Deprecation +There are no deprecations in this release. + +#### Upgrade Instructions +Go to the [Upgrading Tyk](#upgrading-tyk) section for detailed upgrade instructions. + +#### Release Highlights + +##### Dashboard Analytics for API Ownership + +When we implemented Role Based Access Control and API Ownership in Tyk +Dashboard, we unlocked great flexibility for you to assign different roles to +different users and user groups with visibility and control over different +collections of APIs on your Gateway. Well, from 5.1 we have added a new Role, +which layers on top of the existing β€œAnalytics” role and can be used to restrict +a user’s access, within the Dashboard Analytics screens, to view only the +statistics from APIs that they own; we’ve called this β€œOwned Analytics”. Due to +the way the analytics data are aggregated (to optimize storage), a user granted +this role will not have access to the full range of charts. Take a look at the +documentation for a full description of this new [user role](/api-management/user-management#user-permissions). + +##### Import API examples from within the Dashboard + +In 5.0 we introduced the possibility to import API examples manually or via +[_Tyk Sync_](/api-management/automations/sync). We have now extended this feature and it is now possible to do this without +leaving the Dashboard. When having an empty β€œData Graphs” section you will be +presented with 3 icon buttons with one of them offering you to import an Example +API. + +If you already have Data Graphs in your Dashboard you can either click on +the β€œImport” button or click on the β€œAdd Data Graphβ€œ button and select β€œUse +example data graphβ€œ on the next screen. The examples UI will present you with a +list of available examples. You can navigate to the details page for every +example and import it as well from the same page. + +##### Improved nested GraphQL stitching + +Before this release, it was only possible to implement nested GraphQL stitching +(GraphQL data source inside another data source) by using a REST data source and +providing the GraphQL body manually. We have now extended the GraphQL data source so +that you can provide a custom operation and therefore access arguments or object +data from parent data sources. + +To use this feature you will only need to check the β€œAdd GraphQL operationβ€œ checkbox when creating a GraphQL data source. + +##### Import UDG API from OAS 3.0.0 + +We added a [Dashboard API Endpoint](/api-management/data-graph#automatically-creating-rest-udg-configuration-based-on-oas-specification) that is capable of taking an OAS 3.0.0 document and converting it into a UDG API. + +This will generate the full schema as well as the data sources that are defined inside the OAS document. + +##### Changed default RPC pool size for MDCB deployments + +We have reduced the default RPC pool size from 20 to 5. This can reduce the CPU and +memory footprint in high throughput scenarios. Please monitor the CPU and memory +allocation of your environment and adjust accordingly. You can change the pool +size using [slave_options.rpc_pool_size](/tyk-oss-gateway/configuration#slave_optionsrpc_pool_size) + +#### Downloads + +[docker image to pull](https://hub.docker.com/layers/tykio/tyk-dashboard/v5.1/images/sha256-8cde3c6408b9a34daa508a570539ca6cd9fcb8ee5c4790abe907eaecddc1bd9b?context=explore) + + +#### Changelog + +##### Added + +- Added two endpoints to the dashboard to support the retrieval of example API definitions. One for fetching all examples and another for fetching a single example. +- Added a way to display UDG examples from the [tyk-examples](https://github.com/TykTechnologies/tyk-examples) repository in the Dashboard UI +- Added screens in Dashboard New Graph flow, that allows users to choose between creating a graph from scratch or importing one of our example graphs +- Added a screen to display details of a UDG example API +- Added a feature to display a full [_Tyk Sync_](/api-management/automations/sync) command that will allow a user to import an example UDG into their Dashboard +- Added `/examples` endpoint to Dashboard API that returns a list of available API examples that can later be imported into the Dashboard `GET /api/examples` +- Added `/data-graphs/data-sources/import` endpoint to Dashboard API that transforms an OpenAPI document into UDG config and publishes it in Dashboard `POST /api/data-graphs/data-sources/import` +- Added query param `apidef=true` to example detail endpoint in Dashboard API to retrieve the API definition of an example +- Added new `owned_analytics` user permission which restricts the user's access only to analytics relating to APIs they own. These are the _API Activity Dashboard Requests_ and _Average Errors Over Time_ charts in the Tyk Dashboard. Note that it is not currently possible to respect API Ownership in other aggregated charts + +##### Changed + +- Tyk Dashboard updated to Go 1.19 +- Updated npm package dependencies of Dashboard, to address critical and high CVEs +- Changed the field mapping tickbox description in GUI to be 'Use default field mapping' + +##### Fixed + +- Fixed an issue when using custom authentication with multiple authentication methods. Custom authentication could not be selected to provide the base identity +- Fixed an issue where the login URL was displayed as undefined when creating a TIB Profile using LDAP as a provider +- Fixed an issue where it was not possible to download Activity by API or Activity by Key from the Dashboard when using PostgreSQL for the analytics store +- Fixed an issue where a new user could be stuck in a password reset loop in the dashboard if TYK_DB_SECURITY_FORCEFIRSTLOGINPWRESET was enabled +- Fixed an issue where the `ssl_force_common_name_check` flag was disappearing. The flag was disappearing after being updated via dashboard UI raw API editor and a subsequent page reload. It was also disappearing when updating the API Definition via the GW/DB API. +- Fixed an issue where a user could update their email address to match that of another user within the same organization +- Fixed an issue where users without `user:write` permission were able to update their permissions through manipulation of Dashboard API calls +- Fixed an issue where the versions endpoint returned APIs that were not owned by the logged-in user +- Fixed an issue where the log browser showed analytics for APIs not owned by the logged-in user +- Fixed an issue that prevented non-admin users from seeing _Endpoint Popularity_ data in the Tyk Dashboard +- Fixed an issue where additional data was returned when requesting analytics with p=-1 query when using SQL for the analytics store +- Fixed an issue so that filtering by API now respects API Ownership in three Dashboard charts. + + - Gateway Dashboard - API Activity Dashboard - Requests + - Activity by API - Traffic Activity per API + - Errors - Average Errors Over Time + +- Fixed an issue so that the Log Browser now respects API Ownership. A user will now only be able to see logs for the APIs that they are authorized to view +- Fixed filters for the Log Browser, Errors - Average Errors Over Time and API Activity Dashboard - Requests so that a user can only select from versions of APIs for which they have visibility +- Fixed UI bug so that data graphs created with multiple words are [sluggified](https://www.w3schools.com/django/ref_filters_slugify.php#:~:text=Definition%20and%20Usage,ASCII%20characters%20and%20hyphens%20(%2D).), i.e. spaces are replaced with a hyphen `-` +- Fixed an issue with routing, which was sending the user to a blank screen while creating a new Data Graph or importing an example API + +## 5.0 Release Notes +### 5.0.15 Release Notes + +#### Release Date 24 October 2024 + +#### Release Highlights + +This is a version bump to align with Gateway v5.0.15, no changes have been implemented in this release. + +#### Breaking Changes + +There are no breaking changes in this release. + +

Upgrade instructions

+If you are upgrading to 5.0.15, please follow the detailed [upgrade instructions](#upgrading-tyk). + +

Changelog

+No changes in this release. + + +--- + +

5.0.14 Release Notes

+#### Release Date 18th September 2024 + +#### Upgrade Instructions + +This release is not tightly coupled with Tyk Gateway v5.0.14, so you do not have to upgrade both together. + + +Go to the [Upgrading Tyk](/developer-support/release-notes/gateway#upgrading-tyk) section for detailed upgrade instructions. + + +#### Release Highlights + +This release fixes some display issues in Tyk Dashboard and Tyk Classic Portal when using PostgreSQL. + +

Changelog

+##### Fixed + +
    +
  • + + +Fixed an issue where API statistics were not being shown when using PostgreSQL and adding two or more tags in the Activity page + + + +Fixed an issue where HTTP 429 status codes were not being shown on the Activity page when using PostgreSQL + + + +Fixed wrong graphs and incorrect requests counter on Tyk Classic Portal when using PostgreSQL + + + +Fixed Error Breakdown issue showing errors that happened on different dates than selected date + + +
    • +
+ +--- + +### 5.0.13 Release Notes +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.13) + +--- + +### 5.0.12 Release Notes +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.12) + +--- + +### 5.0.11 Release Notes +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.11) + +--- + +### 5.0.10 Release Notes +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.10) + +--- + +### 5.0.9 Release Notes +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.9) + +--- + +### 5.0.8 Release Notes +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.8) + +--- + +### 5.0.7 Release Notes +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.7). + +--- + +### 5.0.6 Release Notes +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.6). + +--- + +### 5.0.5 Release Notes +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.5). + +--- + +### 5.0.4 Release Notes +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.4). + +--- + +### 5.0.3 Release Notes +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.3). + +--- + +### 5.0.2 Release Notes + +##### Release Date 29 May 2023 + +##### Release Highlights + +###### Support for MongoDB 5 and 6 +From Tyk 5.0.2, we added support for MongoDB 5.0.x and 6.0.x. To enable this, you have to set new Dashboard config option driver to *mongo-go*. +The driver setting defines the driver type to use for MongoDB. It can be one of the following values: +- [mgo](https://github.com/go-mgo/mgo) (default): Uses the *mgo* driver. This driver supports MongoDB versions `<= v4.x` (lower or equal to v4.x). You can get more information about this driver in the [mgo](https://github.com/go-mgo/mgo) GH repository. To allow users more time for migration, we will update our default driver to the new driver, *mongo-go*, in next major release. +- [mongo-go](https://github.com/mongodb/mongo-go-driver): Uses the official MongoDB driver. This driver supports MongoDB versions >= v4.x (greater or equal to v4.x). You can get more information about this driver in [mongo-go-driver](https://github.com/mongodb/mongo-go-driver) GH repository. + +See how to [Choose a MongoDB driver](/planning-for-production/database-settings#choose-a-mongodb-driver) + +**Note: Tyk Pump 1.8.0 and MDCB 2.2 releases have been updated to support the new driver option** + +##### Downloads + +[docker image to pull](https://hub.docker.com/layers/tykio/tyk-dashboard/v5.0.2/images/sha256-fe3009c14ff9096771d10995a399a494389321707e951a3c46f944afd28d18cd?context=explore) + + +
Changelog
+###### Fixed +- Fixed a bug on migration of a portal catalog with deleted policy to SQL +- Fixed: Redirect unregistered user to new page when SSOOnlyForRegisteredUsers is set to true + +--- + +### 5.0.1 Release Notes + +##### Release Date 25 Apr 2023 + +##### Release Highlights +This release primarily focuses on bug fixes. +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.0.1) below. + +##### Downloads +- [docker image to pull](https://hub.docker.com/layers/tykio/tyk-dashboard/v5.0.1/images/sha256-013d971fc826507702f7226fa3f00e1c7e9d390fc0fb268bed42e410b126e89d?context=explore) + +
Changelog
+###### Added +- Improved security for people using the Dashboard by adding the Referrer-Policy header with the value `no-referrer` +- Added ability to select the plugin driver within the Tyk OAS API Designer + +###### Changed +- When creating a new API in the Tyk OAS API Designer, caching is now disabled by default + +###### Fixed +- Fixed a bug where a call to the `/hello` endpoint would unnecessarily log `http: superfluous response.WriteHeader call` +- Fixed a bug where the Dashboard was showing *Average usage over time* for all Developers, rather than just those relevant to the logged in developer +- Fixed a bug where logged in users could see Identity Management pages, even if they didn't have the rights to use these features +- Fixed a bug that prevented Tyk Dashboard users from resetting their own passwords +- Fixed issue with GraphQL proxy headers added via UI +- Fixed a bug where the Dashboard would not allow access to any screens if a logged in user didn’t have access to the APIs resource regardless of other access rights +- Fixed a bug on the key management page where searching by `key_id` did not work - you can now initiate the search by pressing enter after typing in the `key_id` +- Fixed a bug where Dashboard API could incorrectly return HTTP 400 when deleting an API +- Fixed UDG UI bug that caused duplicate data source creation on renaming +- Fixed schema validation for custom domain in Tyk OAS API definition +- Fixed a bug where the left menu did not change when Dashboard language was changed +- Fixed a bug that caused the Dashboard to report errors when decoding multiple APIs associated with a policy +- Fixed a bug where it was not possible to disable the Use Scope Claim option when using JWT authentication +- Fixed a bug in the default OPA rule that prevented users from resetting their own password +- Fixed a bug where authToken data was incorrectly stored in the JWT section of the authentication config when a new API was created + +--- + +### 5.0.0 Release Notes + +#### Release Date 28 Mar 2023 + +#### Release Highlights + +##### Improved OpenAPI support + +Tyk Dashboard has been enhanced with **all the custom middleware options** for Tyk OAS APIs, so **for the first time** you can configure your custom middleware from the Dashboard; this covers the full suite of custom middleware from pre- to post- and response plugins. We’ve got support for middleware bundles, Go plugins and Tyk Virtual Endpoints, all within the new and improved Tyk Dashboard UI. + +[Versioning your Tyk OAS APIs](/api-management/api-versioning) is easier than ever, with the Tyk OSS Gateway now looking after the maintenance of the list of versions associated with the base API for you; we’ve also added a new endpoint on the Tyk API that will return details of the versions for a given API. + +Tyk Dashboard hasn’t been left out, we’ve implemented a brand new version management UI for Tyk OAS APIs, to make it as easy as possible for you to manage those API versions as you develop and extend your API products with Tyk. + +We’ve improved support for [OAS Mock Responses](/api-management/traffic-transformation/mock-response), with the Tyk OAS API definition now allowing you to register multiple Mock Responses in a single API, providing you with increased testing flexibility. + +Another new feature in the Tyk OAS API Designer is that you can now update (PATCH) your existing Tyk OAS APIs through the Dashboard API without having to resort to curl. That should make life just that little bit easier. +Of course, we’ve also addressed some bugs and usability issues as part of our ongoing ambition to make Tyk OAS API the best way for you to create and manage your APIs. + +##### GraphQL and Universal Data Graph improvements + +This release is all about making things easier for our users with GraphQL and Universal Data Graph. + +In order to get our users up and running with a working Universal Data Graph quickly, we’ve created a repository of examples that anyone can import into their Dashboard or Gateway and see what Universal Data Graph is capable of. Import can be done in two ways: +- manually, by simply copying a Tyk API definition from GitHub - [TykTechnologies/tyk-examples](https://github.com/TykTechnologies/tyk-examples): A repository containing example API definitions and policies for Tyk products. +- via command line [using tyk-sync](/api-management/data-graph#udg-examples) + +To make it easier for our users to find their way to Universal Data Graph, we’ve also given it its own space in the Dashboard. From now on you can find UDG under Data Graphs section of the menu. + +It also got a lot easier to turn a Kafka topic into a GraphQL subscription. Using our new Dashboard API endpoint, users will be able to transform their AsyncAPI documentation into Universal Data Graph definition with a single click. Support for OAS coming soon as well! + +With this release we are also giving our users [improved headers for GQL APIs](/api-management/graphql#graphql-apis-headers). It is now possible to use context variables in request headers and persist headers needed for introspection separately for improved security. + +Additionally we’ve added Dashboard support for introspection control on policy and key level. It is now possible to allow or block certain consumers from being able to introspect any graph while creating a policy or key via Dashboard. + +#### Downloads + +[docker image to pull](https://hub.docker.com/layers/tykio/tyk-dashboard/v5.0/images/sha256-3d736b06b023e23f406b1591f4915b3cb15a417fcb953d380eb8b4d71829f20f?tab=vulnerabilities) + +

Changelog

+##### Added +- Numerous UX improvements +- New UI for custom middleware for Tyk OAS APIs +- Significantly improved Tyk OAS API versioning user experience +- It now possible to use PATCH method to modify Tyk OAS APIs via the Dashboard API +- Now you can turn a Kafka topic into a GraphQL subscription by simply [importing your AsyncAPI definition](/api-management/dashboard-configuration#data-graphs-api) +- Way to control access to introspection on policy and key level + +##### Changed +- Universal Data Graph moved to a separate dashboard section + +--- + +## 4.3 Release Notes +### 4.3.0 Release Notes + +#### Release Highlights + +##### Tyk OAS APIs - Versioning via the Dashboard + +Tyk v4.3 adds API versioning to the Dashboard UI, including: + +- Performing CRUD operations over API versions +- Navigate seamlessly between versions +- A dedicated manage versions screen +- easily identify the default version and the base API. + +##### Importing OAS v3 via the Dashboard + +Importing OpenAPI v3 documents in order to generate Tyk OAS API definition is now fully supported in our Dashboard UI. Our UI automatically detects the version of your OpenAPI Document, and will suggest options that you can pass or allow Tyk to read from the provided document, in order to configure the Tyk OAS API Definition. Such as: + +- custom upstream URL +- custom listen path +- authentication mechanism +- validation request rules and limit access only to the defined paths. + +[Importing OAS v3 via the Dashboard](/api-management/gateway-config-managing-oas#importing-an-openapi-description-to-create-an-api) + +##### Updated the Tyk Dashboard version of Golang, to 1.16. + +**Our Dashboard is using Golang 1.16 version starting with 4.3 release. This version of the Golang release deprecates x509 commonName certificates usage. This will be the last release where it's still possible to use commonName, users need to explicitly re-enable it with an environment variable.** + +The deprecated, legacy behavior of treating the CommonName field on X.509 certificates as a host name when no Subject Alternative Names are present is now disabled by default. It can be temporarily re-enabled by adding the value x509ignoreCN=0 to the GODEBUG environment variable. + +Note that if the CommonName is an invalid host name, it's always ignored, regardless of GODEBUG settings. Invalid names include those with any characters other than letters, digits, hyphens and underscores, and those with empty labels or trailing dots. + + +#### Changelog + +##### Added + +- Added an option for using multiple header/value pairs when configuring GraphQL API with a protected upstream and persisting those headers for future use. +- Added documentation on how edge endpoints Dashboard configuration can be used by users to add tags for their API Gateways. +- When retrieving the Tyk OAS API Definition of a versioned API, the base API ID is passed on the GET request as a header: `x-tyk-base-api-id`. +- If Edge Endpoints Dashboard configuration is present, when users add segment/tags to the Tyk OAS API Definition, their corresponding URLs are populated in the servers section of the OAS document. +- Listen path field is now hidden from the API Designer UI, when the screen presents a versioned or internal API. + +##### Changed + +- Extended existing `x-tyk-gateway` OAS documentation and improved the markdown generator to produce a better-formatted documentation for `x-tyk-gateway` schema. +- Complete change of Universal Data Graph configuration UI. New UI is now fully functional and allows configuration of all existing datasources (REST, GraphQL and Kafka). +- Changed look & feel of request logs for GraphQL Playground. It is now possible to filter the logs and display only the information the user is interested in. + +##### Fixed + +- Fixed: OAS API definition showing management gateway URL even if segment tags are present in cloud. From now on OAS servers section would be filled with edge endpoint URLs if configured. +- Adding a path that contains a path parameter, doesn’t throw an error anymore on the Dashboard UI, and creates default path parameter description in the OAS. + +#### Updated Versions + +Tyk Dashboard 4.3 ([docker images](https://hub.docker.com/r/tykio/tyk-dashboard/tags?page=1&name=4.3.0)) + +#### Upgrade process + +Follow the [standard upgrade guide](/developer-support/upgrading), there are no breaking changes in this release. + +If you want switch from MongoDB to SQL, you can [use our migration tool](/planning-for-production/database-settings#migrating-from-an-existing-mongodb-instance), but keep in mind that it does not yet support the migration of your analytics data. + + + +Note: Upgrading the Golang version implies that all the Golang custom plugins that you are using need to be recompiled before migrating to 4.3 version of the Gateway. Check our docs for more details [Golang Plugins](/api-management/plugins/golang). + + + +## 4.2 Release Notes +### 4.2.0 Release Notes + +#### Release Highlights + +##### GraphQL Federation improvements + +###### Changed GUI in Universal Data Graph configuration section. + +A new GUI introduces enhancements to the user experience and more consistent user journey for UDG. +This change does not yet cover all possible use cases and is released with a feature flag. To enable the new GUI, analytics.conf needs the following setting: + +``` +"ui": { + "dev": true +} +``` expandable + +What’s possible with this change: +- Importing GraphQL schema created outside of Tyk (formats accepted .json, .graphql, .grahqls) +- Creating GraphQL schema in Tyk using schema editor +- Hide/Unhide schema editor to focus on graphical representation of the schema +- Resizing schema editor to adjust workspace look & feel to user preferences +- Improved search in schema editor (search and search & replace available) +- Quick link to UDG documentation from schema editor + +> Note: Full configuration of new Universal Data Graph is not yet possible in the GUI, however any UDGs created earlier will not be broken and will work as previously. + +##### Changes to federation entities +###### Defining the base entity +Entities must be defined with the `@key` directive. The fields argument must reference a field by which the entity can be uniquely identified. Multiple primary keys are possible. For example: + +Subgraph 1 (base entity): +``` +type MyEntity @key(fields: "id") @key(fields: "name") { + id: ID! + name: String! +} +``` + Attempting to extend a non-entity with an extension that includes the @key directive or attempting to extend a base entity with an extension that does not include the @key directive will both result in errors. + +###### Entity stubs + +Entities cannot be shared types (be defined in more than one single subgraph). +If one subgraph references a base entity (an entity defined in another subgraph), that reference must be declared as a stub (stubs look like an extension without any new fields in federation v1). This stub would contain the minimal amount of information to identify the entity (referencing exactly one of the primary keys on the base entity regardless of whether there are multiple primary keys on the base entity). For example, a stub for MyEntity from Subgraph 1 (defined above): + +Subgraph 2 (stub) +``` +extend type MyEntity @key(fields: "id") { + id: ID! @external +} +``` expandable + +###### Supergraph extension orphans +It is now possible to define an extension for a type in a subgraph that does not define the base type. +However, if an extension is unresolved (an extension orphan) after an attempted federation, the federation will fail and produce an error. + +###### Improved Dashboard UI and error messages +GraphQL-related (for example when federating subgraphs into a supergraph) errors in the Dashboard UI will show a lean error message with no irrelevant prefixes or suffixes. + +Changed the look & feel of request logs in Playground tab for GraphQL APIs. New component presents all logs in a clearer way and is easier to read for the user + +###### Shared types +Types of the same name can be defined in more than one subgraph (a shared type). This will no longer produce an error if each definition is identical. +Shared types cannot be extended outside of the current subgraph, and the resolved extension must be identical to the resolved extension of the shared type in all other subgraphs (see subgraph normalization notes). Attempting to extend a shared type will result in an error. +The federated supergraph will include a single definition of a shared type, regardless of how many times it has been identically defined in its subgraphs. + +###### Subgraph normalization before federation +Extensions of types whose base type is defined in the same subgraph will be resolved before an attempt at federation. A valid example involving a shared type: + +Subgraph 1: +``` +enum Example { + A, + B +} + +extend enum Example { + C +} +``` + +Subgraph 2: +``` +enum Example { + A, + B, + C +} +``` expandable + +The enum named β€œExample” defined in Subgraph 1 would resolve to be identical to the same-named enum defined in Subgraph 2 before federation takes place. The resulting supergraph would include a single definition of this enum. + +###### Validation +Union members must be both unique and defined. +Types must have bodies, e.g., enums must contain at least one value; inputs, interfaces, or objects must contain at least one field + +##### OpenAPI +Added support for the Request Body Transform middleware, for new Tyk OAS API Definitions. + +##### Universal Data Graph + +Added support for Kafka as a data source in Universal Data Graph. Configuration allows the user to provide multiple topics and broker addresses. + +#### Changelog + +##### Added +- Added support for Kafka as a data source in Universal Data Graph. +- Added support for the Request Body Transform middleware for OAS based APIs + +##### Changed +- Improved GraphQL Dashboard UI error messages +- Changed GUI in Universal Data Graph +- Changed look & feel of request logs in Playground tab for GraphQL APIs. + +##### Fixed +- Fixed an issue with key lookup where keys were not being found when using the search field +- Fixed an issue with object types dropdown in Universal Data Graph config, where it wasn’t working correctly when object type UNION was chosen +- Fixed an issue in Universal Data Graph which prevented users from injecting an argument value or parameter value in the domain part of the defined data source upstream URL + +#### Updated Versions + +Tyk Dashboard 4.2 + + +#### Upgrade process + +Follow the [standard upgrade guide](/developer-support/upgrading), there are no breaking changes in this release. + +If you want switch from MongoDB to SQL, you can [use our migration tool](/planning-for-production/database-settings#migrating-from-an-existing-mongodb-instance), but keep in mind that it does not yet support the migration of your analytics data. + +## 4.1 Release Notes +### 4.1.0 Release Notes + +#### Release Highlights + +##### OpenAPI as a native API definition format +Tyk has always had a proprietary specification for defining APIs. From Tyk v4.1 we now support defining APIs using the Open API Specification (OAS) as well, which can offer significant time and complexity savings. [This is an early access capability](/developer-support/release-types/early-access-feature). + +As we extend our OAS support, we would very much like your feedback on how we can extend and update to best meet your needs: . + +This capability is available in both the open source and paid versions of Tyk. See our [Tyk OAS documentation](/api-management/gateway-config-tyk-oas) for more details. + + +##### MDCB Synchroniser + +Tyk Gateway v4.1 enables an improved synchroniser functionality within Multi Data Center Bridge (MDCB) v2.0. Prior to this release, the API keys, certificates and OAuth clients required by worker Gateways were synchronised from the controller Gateway on-demand. With Gateway v4.1 and MDCB v2.0 we introduce proactive synchronisation of these resources to the worker Gateways when they start up. + +This change improves resilience in case the MDCB link or controller Gateway is unavailable, because the worker Gateways can continue to operate independently using the resources stored locally. There is also a performance improvement, with the worker Gateways not having to retrieve resources from the controller Gateway when an API is first called. + +Changes to keys, certificates and OAuth clients are still synchronised to the worker Gateways from the controller when there are changes and following any failure in the MDCB link. + +##### Go Plugin Loader +When upgrading your Tyk Installation you need to re-compile your plugin with the new version. At the moment of loading a plugin, the Gateway will try to find a plugin with the name provided in the API definition. If none is found then it will fallback to search the plugin file with the name: `{plugin-name}_{Gw-version}_{OS}_{arch}.so` + +From v4.1.0 the plugin compiler automatically names plugins with the above naming convention. It enables you to have one directory with different versions of the same plugin. For example: + +- `plugin_v4.1.0_linux_amd64.so` +- `plugin_v4.2.0_linux_amd64.so` + +So, if you upgrade from Tyk v4.1.0 to v4.2.0 you only need to have the plugins compiled for v4.2.0 before performing the upgrade. + +#### Changelog + +##### Added +- Added support for new OAS api definition format, and new API creation screens +- Dashboard boostrap instalation script extended to support SQL databases +- Added `TYK_DB_OMITCONFIGFILE` option for Tyk Dashboard to ignore the values in the config file and load its configuration only from environment variables and default values +- Added a new config option `identity_broker.ssl_insecure_skip_verify` that will allow customers using the embedded TIB to use IDPs exposed with a self signed certificate. Not intended to be used in production, only for testing and POC purposes. +- Added option to configure certificates for Tyk Dashboard using [environment variables](/tyk-dashboard/configuration#http_server_optionscertificates). + +##### Changed +- Detailed information about certificates can be viewed from certificates listing page +- Dashboard APIs GQL Playground now shows additional information about certificates +- Dashboard will now use default version of GraphiQL Playground which can switch between light and dark modes for more accessibility +- Banner for resyncing GraphQL schema has been given a new, more accessible look in line with the rest of Dashboard design + +##### Fixed +- Fixed an issue with key lookup where keys were not being found when using the search field +- Fixed an issue with object types dropdown in Universal Data Graph config, where it wasn’t working correctly when object type UNION was chosen +- Fixed an issue in Universal Data Graph which prevented users from injecting an argument value or parameter value in the domain part of the defined data source upstream URL + +#### Updated Versions +Tyk Dashboard 4.1 +Tyk MDCB 2.0.1 + +#### Upgrade process + +Follow the [standard upgrade guide](/developer-support/upgrading), there are no breaking changes in this release. + +If you want switch from MongoDB to SQL, you can [use our migration tool](/planning-for-production/database-settings#migrating-from-an-existing-mongodb-instance), but keep in mind that it does not yet support the migration of your analytics data. + +## 4.0 Release Notes +### 4.0.0 Release Notes + +#### Release Highlights + +##### GraphQL federation + +As we know, ease-of-use is an important factor when adopting GraphQL. Modern enterprises have dozens of backend services and need a way to provide a unified interface for querying them. Building a single, monolithic GraphQL server is not the best option. It is hard to maintain and leads to a lot of dependencies and over-complication. + +To remedy this, Tyk 4.0 offers GraphQL federation that allows the division of GraphQL implementation across multiple backend services, while still exposing them all as a single graph for the consumers. Subgraphs represent backend services and define a distinct GraphQL schema. A subgraph can be queried directly, as a separate service or federated in the Tyk Gateway into a larger schema of a supergraph – a composition of several subgraphs that allows execution of a query across multiple services in the backend. + +[Federation docs](/api-management/graphql#overview-1) + +[Subgraphs and Supergraphs docs](/api-management/graphql#subgraphs-and-supergraphs) + +##### GraphQL subscriptions + +Subscriptions are a way to push data from the server to the clients that choose to listen to real-time messages from the server, using the WebSocket protocol. There is no need to enable subscriptions separately; Tyk supports them alongside GraphQL as standard. + +With release 4.0, users can federate GraphQL APIs that support subscriptions. Federating subscriptions means that events pushed to consumers can be enriched with information from other federated graphs. + +[Subscriptions docs](/api-management/graphql#graphql-subscriptions) + +##### SQL database support +The other major capability in Tyk 4.0 is that the Tyk Dashboard can store its data in a SQL relational database. + +Until now, Tyk Dashboard has used MongoDB for storing everything from data such as APIs, policies and users through to analytics and logs. MongoDB is still a great storage choice for most projects. However, not all users have MongoDB as part of their tech stack. Some are in heavily regulated industries which means adding it would be a pain. For others, the document storage type and lack of proper ACID transaction support may not be the best solution. These users can now choose a SQL database solution instead. + +From version 4.0, Tyk Dashboard and Tyk Pump will support four data storage layers, which can be configured separately, each with a different officially supported database solution (if needed). All data stored in SQL databases will provide the same information in the Dashboard that MongoDB did. + +While SQL support for Tyk products does not depend on specific database features, with this release, we will provide official support for [PostgreSQL DB for production purposes](/planning-for-production/database-settings#postgresql), and SQLite for development and PoC environments. Note that SQL support is available for self-managed setups only. + +As part of SQL support we are also providing tooling to perform seamless migration of your Dashboard data from Mongo to SQL. However, at the moment migration of analytics data is not supported. +[MongoDB to SQL migration docs](/planning-for-production/database-settings#migrating-from-an-existing-mongodb-instance) + +#### Changelog +- Now it is possible to configure GraphQL upstream authentication, in order for Tyk to work with its schema +- JWT scopes now support arrray and comma delimiters +- Go plugins can be attached on per-endpoint level, similar to virtual endpoints + +#### Updated Versions +Tyk Dashboard 4.0 +Tyk Pump 1.5 + +#### Upgrade process + +Follow the [standard upgrade guide](/developer-support/upgrading), there are no breaking changes in this release. + +If you want switch from MongoDB to SQL, you can [use our migration tool](/planning-for-production/database-settings#migrating-from-an-existing-mongodb-instance), but keep in mind that it does not yet support the migration of your analytics data. + +## 3.2 Release Notes +### 3.2.0 Release Notes + +#### Release Notes + +##### Bring your own Identity Provider - Dynamic Client Registration now available! + +DCR is a protocol of the Internet Engineering Task Force put in place to set standards in the dynamic registration of clients with authorization servers. This feature is a way for you to integrate your Tyk Developer Portal with an external identity provider such as Keycloak, Gluu, Auth0 or Okta. +The portal developer won't notice a difference. However, when they create the app via Tyk Developer portal, Tyk will dynamically register that client on your authorization server. This means that it is the Authorization Server that will issue the Client ID and Client Secret for the app. + +Check our DCR docs [here](/tyk-developer-portal/tyk-portal-classic/dynamic-client-registration) + +We also took this opportunity to give a refresh to the portal settings UI so let us know if you like it! + +##### GraphQL and UDG improvements + +We've updated the GraphQL functionality of our [Universal Data Graph](/api-management/data-graph#overview). You’re now able to deeply nest GraphQL & REST APIs and stitch them together in any possible way. + +Queries are now possible via WebSockets and Subscriptions are coming in the next Release (3.3.0). + +You're also able to configure [upstream Headers dynamically](/api-management/data-graph#header-forwarding), that is, you’re able to inject Headers from the client request into UDG upstream requests. For example, it can be used to access protected upstreams. + +We've added an easy to use URL-Builder to make it easier for you to inject object fields into REST API URLs when stitching REST APIs within UDG. + +Query-depth limits can now be configured on a per-field level. + +If you’re using GraphQL upstream services with UDG, you’re now able to forward upstream error objects through UDG so that they can be exposed to the client. + + +##### Extendable Tyk Dashboard permissions system + +The Tyk Dashboard permission system can now be extended by writing custom rules using an Open Policy Agent (OPA). The rule engine works on top of the Tyk Dashboard API, which means you can control not only access rules, but also the behavior of all Dashboard APIs (except your public developer portal). You can find more details about OPA [here](/api-management/dashboard-configuration#extend-permissions-using-open-policy-agent-opa). + +In addition, you can now create your own custom permissions using the Additional Permissions API or by updating `security.additional_permissions` map in the Tyk Dashboard config, and writing Opa rule containing logic for the new permission. + +#### Changelog + +In addition to the above, version 3.2 includes all the fixes that are part of 3.0.5 +https://github.com/TykTechnologies/tyk/releases/tag/v3.0.5 + +#### Updated Versions +Tyk Dashboard 3.2 + +#### Upgrade process +If you already have GraphQL or UDG APIs you need to follow this [upgrade guide](/api-management/graphql#migrating-to-32). + +## 3.1 Release Notes +### 3.1.0 Release Notes + +#### Release Highlights + +##### Identity Management UX and SAML support +You will notice that the experience for creating a new profile in the Identity management section of the dashboard was changed to a β€˜wizard’ approach which reduces the time it takes to get started and configure a profile. +In addition, users are now able to use SAML for the dashboard and portal login, whether you use TIB(Tyk Identity Broker) internally or externally of the dashboard. + +This follows the recent changes that we have made to embed TIB (Tyk Identity Broker)in the dashboard. See 3.0 [release notes](/developer-support/release-notes/dashboard#tyk-identity-broker-now-built-in-to-the-dashboard) for more information regarding this. + +To learn more [see the documentation](/api-management/external-service-integration) + +##### UDG (Universal Data Graph) & GraphQL +###### Schema Validation + +For any GraphQL API that is created via Dashboard or through our API, the GraphQL schema is now validated before saving the definition. Instant feedback is returned in case of error. + +###### Sync / Update schema with upstream API (Proxy Only Mode) + +If you’ve configured just a proxy GraphQL API, you can now keep in sync the upstream schema with the one from the API definition, just by clicking on the `Get latest version` button on the `Schema` tab from API Designer + +Docs [here](/api-management/graphql#syncing-gql-schema) + +###### Debug logs + +You can now see what responses are being returned by the data sources used while configuring a UDG (universal data graph). These can be seen by calling the `/api/debug` API or using the playground tab within API designer. + +The data that will be displayed will show information on the query before and after the request to a data source happens, as follows: + +Before the request is sent: + +Example log message: "`Query.countries: preSendHttpHook executed”. Along with this message, the log entry will contain the following set of fields: Typename, Fieldname and Upstream url; + + +After the request is sent: + +Example log message: "Query.countries: postReceiveHttpHook executed”. Along with this message, the log entry will contain the following set of fields: Typename, Filename, response body, status code. + +Example: + +```{"typename": "Query", "fielname": "countries", "response_body": "{\"data\":{}}", "status_code": 200}``` + +Docs [here](/api-management/graphql#graphql-playground) + +##### Portal +###### GraphQL Documentation + +Documentation for the GraphQL APIs that you are exposing to the portal is available now through a GraphQL Playground UI component, same as on the playground tab of API Designer. + +Also to overcome the CORS issues that you might encounter while testing documentation pages on the portal, we have pre-filled the CORS settings section in API Designer with explicit values from the start. All you need to do is to check the β€œEnable CORS” option. + +###### Portal - API key is hidden in email +You now have the option to hide the API key in the email generated after you approve the key request for a developer. + +[Docs here](/tyk-developer-portal/tyk-portal-classic/key-requests) + + +#### Changelog +The 3.1 version includes the fixes that are part of 3.0.1. +https://github.com/TykTechnologies/tyk/releases/tag/v3.0.1 + + +#### Updated Versions +- Tyk Dashboard 3.1 + +## 3.0 Release Notes +### 3.0.0 Release Notes + +#### Release Highlights + +##### Version changes and LTS releases + +We have bumped our major Tyk Gateway version from 2 to 3, a long overdue change as we’ve been on version 2 for 3 years. We have also changed our Tyk Dashboard major version from 1 to 3, and from now on it will always be aligned with the Tyk Gateway for major and minor releases. The Tyk Pump has also now updated to 1.0, so we can better indicate major changes in future. + +Importantly, such a big change in versions does not mean that we going to break backward compatibility. More-over we are restructuring our internal release strategy to guarantee more stability and to allow us to deliver all Tyk products at a faster pace. We aim to bring more clarity to our users on the stability criteria they can expect, based on the version number. +Additionally we are introducing Long Term Releases (also known as LTS). + +Read more about this changes in our blog post: https://tyk.io/blog/introducing-long-term-support-some-changes-to-our-release-process-product-versioning/ + +##### New Look and Feel + +We have a brand new look to our Tyk Dashboard. About half a year ago, we made some changes to our visual branding to better express our love for creativity and great UX. Those changes started with our website and now we are also incorporating these visual changes into the UI of our products. We do this to keep our brand consistent across the whole Tyk experience and to enhance your experience using our products. + +See our updated [Tutorials](/tyk-self-managed) section. + +##### Universal Data Graph and GraphQL + +Tyk now supports GraphQL **natively**. This means Tyk doesn’t have to use any external services or process for any GraphQL middleware. You can securely expose existing GraphQL APIs using our GraphQL core functionality. + +In addition to this you can also use Tyk’s integrated GraphQL engine to build a Universal Data Graph. The Universal Data Graph (UDG) lets you expose existing services as one single combined GraphQL API. + +All this without even have to build your own GraphQL server. If you have existing REST APIs all you have to do is configure the UDG and Tyk has done the work for you. + +With the Universal Data Graph (UDG), Tyk becomes the central integration point for all your internal and external APIs. +It also benefits from the full set of capabilities included with your Tyk installationβ€”meaning your data graph is secure from the start and can take advantage of a wide range of out-of-the-box middleware to power your graph. + +Read more about the [GraphQL](/api-management/graphql) and [Universal Data Graph](/api-management/data-graph#overview) + + +##### Policies and Keys UX changes + +We have a lot to update you on with our UX & UI revamp, but one thing we want to highlight here are the updates to the policies and keys Dashboard pages. We know there was confusion in the way we set policies and keys up in the Tyk Dashboard, so we redesigned the UI workflow to make it less error-prone, simpler and more intuitive when you create, view and edit security policies and keys. + +When you create, view or edit a key the steps are in a more logical order. We’ve removed the long form that needed to be filled out and replaced it with tabs so you can find and enter information easily. We’ve also grouped all information within each API so you know the exact set up of each of your access rights without any confusion. The new workflow should allow tasks to be completed faster and more efficiently. + +See updated tutorials on how to [create a policy](/api-management/gateway-config-managing-classic#secure-an-api) and [keys](/api-management/gateway-config-managing-classic#access-an-api) + +We also have a [blog post](https://tyk.io/blog/the-transformation-of-policies-and-keys/) that explains what we've done, and why we did it. + + +##### Tyk Identity broker now built-in to the Dashboard + +Previously you had to run a separate process to setup SSO (single sign on). Now this functionality is built-in to the dashboard and got UI revamp. So now you can just start the dashboard, and via UI, create a SSO flow, without installing 3-rd party components. Including SSO via social logins, OpenID Connect and LDAP (with SAML coming very soon!) including integration with the Dashboards RBAC and your Identity Provider. + +See [updated flow details](/api-management/external-service-integration#what-is-tyk-identity-broker-tib) + + +##### Using external secret management services + +Want to reference secrets from a KV store in your API definitions? We now have native Vault & Consul integration. You can even pull from a tyk.conf dictionary or environment variable file. + +[Read more](/tyk-configuration-reference/kv-store) + + +##### Co-Process Response Plugins + +We added a new middleware hook allowing middleware to modify the response from the upstream. Using response middleware you can transform, inspect or obfuscate parts of the response body or response headers, or fire an event or webhook based on information received by the upstream service. + +At the moment the Response hook is supported for [Python and gRPC plugins](/api-management/plugins/rich-plugins#coprocess-dispatcher---hooks). + + +##### Enhanced Gateway health check API + +Now the standard Health Check API response include information about health of the dashboard, redis and mdcb connections. +You can configure notifications or load balancer rules, based on new data. For example, you can be notified if your Tyk Gateway can’t connect to the Dashboard (or even if it was working correctly with the last known configuration). + +[Read More](/planning-for-production/ensure-high-availability/health-check) + +##### Enhanced Detailed logging +Detailed logging is used in a lot of the cases for debugging issues. Now as well as enabling detailed logging globally (which can cause a huge overhead with lots of traffic), you can enable it for a single key, or specific APIs. + +New detailed logging changes are available only to our Self-Managed customers currently. + +[Read More](/api-management/troubleshooting-debugging#capturing-detailed-logs) + +##### Better Redis failover + +Now, if Redis is not available, Tyk will be more gracefully handle this scenario, and instead of simply timing out the Redis connection, will dynamically disable functionality which depends on redis, like rate limits or quotas, and will re-enable it back once Redis is available. The Tyk Gateway can even be started without Redis, which makes possible scenarios, such as when the Gateway proxies Redis though itself, like in a Redis Sentinel setup. + +##### Weight-Based Load Balancing + +The Tyk Dashboard now allows you to control weighting of the upstreams, when using load balancing functionality. For example now you can configure Tyk to send 20% of traffic to one upstream, with 80% to another upstream service. + +This enables you to perform Canary or A/B tests of their APIs and services. Similarly, if caches require warming, then we can send a low % of traffic to these services, and when confident that they can handle the load, start incrementally sending a higher % of traffic to these services. + +[Read More](/planning-for-production/ensure-high-availability/load-balancing#configure-load-balancing-and-weighting-via-the-dashboard) + +##### Ability to shard analytics to different data-sinks + +In a multi-org deployment, each organization, team, or environment might have their preferred analytics tooling. At present, when sending analytics to the Tyk Pump, we do not discriminate analytics by org - meaning that we have to send all analytics to the same database - e.g. MongoDB. Now the Tyk Pump can be configured to send analytics for different organizations to different places. E.g. Org A can send their analytics to MongoDB + DataDog. But Org B can send their analytics to DataDog + expose the Prometheus metrics endpoint. + +It also becomes possible to put a blocklist in-place, meaning that some data sinks can receive information for all orgs, whereas others will not receive OrgA’s analytics if blocked. + +This change requires updating to new Tyk Pump 1.0 + +[Read More](/api-management/tyk-pump#tyk-pump-configuration) + +##### 404 Error logging - unmatched paths + +Concerned that client’s are getting a 404 response? Could it be that the API definition or URL rewrites have been misconfigured? Telling Tyk to track 404 logs, will cause the Tyk Gateway to produce error logs showing that a particular resource has not been found. + +The feature can be enabled by setting the config `track_404_logs` to `true` in the gateway's config file. + + +#### Changelog +- Fixed the bug when tokens created with non empty quota, and quota expiration set to `Never`, were treated as having unlimited quota. Now such tokens will stop working, once initial quota is reached. + +#### Updated Versions + +- Tyk Dashboard 3.0 +- Tyk Pump 1.0 + +#### Upgrading From Version 2.9 + +No specific actions required. +If you are upgrading from version 2.8, please [read this guide](/developer-support/release-notes/archived#upgrading-from-version-28) + + +## Further Information + +### Upgrading Tyk +Please refer to the [upgrading Tyk](/developer-support/upgrading) page for further guidance on the upgrade strategy. + +### API Documentation +{/* Required. Update the link to the Gateway "tyk-gateway-api" or dashboard "tyk-dashboard-api" and the Postman collection + +If there were changes in any of Tyk’s API docs: + +- Have API endpoints been documented in the release note summary and changelog? +- Has a link to the endpoint documentation being included? +- Has the benefit of the new/updated endpoint been explained in the release highlights and changelog? */} +- [OpenAPI Document](/tyk-dashboard-api) +- [Postman Collection](https://www.postman.com/tyk-technologies/workspace/tyk-public-workspace/overview) + +### FAQ + +Please visit our [Developer Support](/developer-support/community) page for further information relating to reporting bugs, upgrading Tyk, technical support and how to contribute. + + + diff --git a/developer-support/release-notes/gateway.mdx b/developer-support/release-notes/gateway.mdx new file mode 100644 index 00000000..d726693c --- /dev/null +++ b/developer-support/release-notes/gateway.mdx @@ -0,0 +1,5104 @@ +--- +title: "Tyk Gateway Release Notes" +'og:description': "Release notes documenting updates, enhancements, and changes for Tyk Gateway." +tags: ['Tyk Gateway', 'Release notes', 'changelog'] +sidebarTitle: "Gateway" +--- + +**Open Source** ([Mozilla Public License](https://github.com/TykTechnologies/tyk/blob/master/LICENSE.md)) + +**This page contains all release notes for Gateway displayed in a reverse chronological order** + +## Support Lifetime + +Our minor releases are supported until our next minor comes out. + +--- +## 5.8 Release Notes + +### 5.8.2 Release Notes + +#### Release Date 1st July 2025 + +#### Release Highlights + +This patch release contains fixes to some bugs experienced when using MDCB and distributed data planes. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.8.2) below. + +#### Breaking Changes + +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.8.2 | MDCB v2.8.1 | MDCB v2.8.1 | +| | Operator v1.2.0 | Operator v0.17 | +| | Sync v2.1.0 | Sync v2.1.0 | +| | Helm Chart v3.0 | Helm all versions | +| | EDP v1.13 | EDP all versions | +| | Pump v1.12.0 | Pump all versions | +| | TIB (if using standalone) v1.7.0 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------ | :---------------------- | :---------------------- | :-------- | +| [Go](https://go.dev/dl/) | 1.23 | 1.23 | [Go plugins](/api-management/plugins/golang) must be built using Go 1.23 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3)| v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations + +There are no deprecations in this release. + +

Upgrade instructions

+If you are upgrading to 5.8.2, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.8.2) + - ```bash expandable + docker pull tykio/tyk-gateway:v5.8.2 + ``` +- Helm charts + - [tyk-charts v3.0.0](/developer-support/release-notes/helm-chart#300-release-notes) + +- [Source code tarball of Tyk Gateway v5.8.2](https://github.com/TykTechnologies/tyk/releases/tag/v5.8.2) + +

Changelog

+##### Fixed + +
    +
  • + + +Resolved an issue introduced in Tyk 5.7.1 where Gateways in distributed Data Planes failed to cache TLS certificates correctly in the local Redis, resulting in potential service disruptions if MDCB became unavailable. Data plane gateways now reliably serve HTTPS and mTLS traffic even if MDCB is unavailable. + + + +We've fixed an issue where RPC connections remained stale when DNS records changed (such as ELB IP updates), leading to timeout errors. Based on direct customer reports, we've enhanced DNS resolution so all connections in the RPC pool now properly reconnect when endpoint IPs change. This eliminates service disruptions during infrastructure updates and ensures more resilient connectivity. + + + +Fixed a bug where a timeout in an RPC call to MDCB would lead to policies not being synchronised to the data plane. + + +
    • +
+ +--- + +### 5.8.1 Release Notes + +#### Release Date 9 May 2025 + +#### Release Highlights + +This patch release contains various bug fixes. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.8.1) below. + +#### Breaking Changes + +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.8.1 | MDCB v2.8.1 | MDCB v2.8.1 | +| | Operator v1.2.0 | Operator v0.17 | +| | Sync v2.1.0 | Sync v2.1.0 | +| | Helm Chart v3.0 | Helm all versions | +| | EDP v1.13 | EDP all versions | +| | Pump v1.12.0 | Pump all versions | +| | TIB (if using standalone) v1.7.0 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------ | :---------------------- | :---------------------- | :-------- | +| [Go](https://go.dev/dl/) | 1.23 | 1.23 | [Go plugins](/api-management/plugins/golang) must be built using Go 1.23 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3)| v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations + +There are no deprecations in this release. + +

Upgrade instructions

+If you are upgrading to 5.8.1, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.8.1) + - ```bash + docker pull tykio/tyk-gateway:v5.8.1 + ``` +- Helm charts + - [tyk-charts v3.0.0](/developer-support/release-notes/helm-chart#300-release-notes) + +- [Source code tarball of Tyk Gateway v5.8.1](https://github.com/TykTechnologies/tyk/releases/tag/v5.8.1) + +

Changelog

+##### Fixed + +
    +
  • + + +Addressed an issue for UDG APIs where caching led to the forwarding of stale values for headers that contained content variables towards the upstream of the UDG apis. + + + +Resolved an issue where requests could be routed incorrectly due to inverted prioritisation of dynamically declared paths over those with similar static paths. Now, statically declared paths take priority in the path matching algorithm, so if API1 has listen path `/path/{param}/endpoint` and API2 has listen path `/path/specific/endpoint` a request to `/path/specific/endpoint/resource` will be correctly routed to API2. + + + +Fixed an issue where an [enforced timeout](/planning-for-production/ensure-high-availability/enforced-timeouts) set for a specific API endpoint could be overruled by the configured [proxy_default_timeout](/tyk-oss-gateway/configuration#proxy_default_timeout). Now if an endpoint-level timeout is set then this will be honoured, regardless of any default timeout that is configured. + + + +Resolved a race condition in self-managed deployments which occasionally lead to fewer Gateways registering with the Dashboard than the number that had been licensed. Now Tyk Self-Managed deployments will allow the licensed number of Gateways to register and serve traffic. + + + +Resolved a bug where `allowed_types` from multiple policies were incorrectly merged using intersection logic. Policies now correctly merge fields to allow access to any fields listed across the applied policies. + + +
    • +
+ +### 5.8.0 Release Notes + +#### Release Date 28 March 2025 + +#### Release Highlights + +With Tyk 5.8.0 we are delighted to unlock the power and flexibility of Tyk OAS for all users, with full feature parity with the legacy Tyk Classic API definition. We are also bringing other updates and improvements, delivering more control, flexibility, and performance. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.8.0) below. + +##### Full support for Gateway configuration using Tyk OAS + +We have completed the journey with Tyk OAS that started in Tyk 4.1 - and now anything that you can configure using the Tyk Classic API definition is also available in the Tyk OAS API definition. Tyk OAS is now the recommended API style for all REST services, with Tyk Classic recommended for use only for GraphQL and TCP services. + +With Tyk OAS we combine the industry standard OpenAPI description with the Tyk Vendor Extension, which encapsulates all of the Tyk Gateway settings that cannot be inferred from the OpenAPI Specification (OAS). You can keep your service description (OAS) as source of truth and update the OpenAPI description part of a Tyk OAS API independently from the Tyk Vendor Extension - no need to unpick distributed vendor extensions from your OAS. For more details, please see the [documentation](/api-management/gateway-config-introduction). + +#### Breaking Changes + +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.8.0 | MDCB v2.8.0 | MDCB v2.8.0 | +| | Operator v1.2.0 | Operator v0.17 | +| | Sync v2.1.0 | Sync v2.1.0 | +| | Helm Chart v3.0 | Helm all versions | +| | EDP v1.13 | EDP all versions | +| | Pump v1.12.0 | Pump all versions | +| | TIB (if using standalone) v1.7.0 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------ | :---------------------- | :---------------------- | :-------- | +| [Go](https://go.dev/dl/) | 1.23 | 1.23 | [Go plugins](/api-management/plugins/golang) must be built using Go 1.23 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3)| v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations + +There are no deprecations in this release. + +

Upgrade instructions

+If you are upgrading to 5.8.0, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.8.0) + - ```bash + docker pull tykio/tyk-gateway:v5.8.0 + ``` +- Helm charts + - [tyk-charts v3.0.0](/developer-support/release-notes/helm-chart#300-release-notes) + +- [Source code tarball of Tyk Gateway v5.8.0](https://github.com/TykTechnologies/tyk/releases/tag/v5.8.0) + +

Changelog

+##### Added + +
    +
  • + + +In Tyk 5.8.0, we have added configuration of the following features into the Tyk OAS API definition, so that anything you can configure for a REST API via Tyk Classic you can also configure using Tyk OAS: + +- IP access control +- API-Level request size limit +- API-level ignore endpoint case +- Skip rate limit middleware +- Skip quota middleware +- Skip quota reset on key creation +- Custom analytics tags +- Custom analytics retention period +- Custom analytics plugins +- Preserve client Host header +- Gateway HTTP settings +- Upstream uptime testing +- Upstream load balancing +- Upstream SSL configuration +- Upstream authentication: HMAC request signing +- Event handling: custom JS handler +- Event handling: custom log Handler +- Batch requests + + + +Tyk Gateway now supports transaction logs, providing structured access logs for API requests. This improves debugging and observability without the overhead of enabling debug mode in production. Logs can be output in JSON format and customized via a template, ensuring flexibility while maintaining performance. Find more details in our [Transaction Logs documentation](/api-management/logs-metrics#enabling-api-request-access-logs-in-tyk-gateway). + + + +We have added GODEBUG flags to enable deprecated insecure ciphers by default for backward compatibility. Existing users will not be affected. New users or those who wish to override these settings can do so at runtime using environment variables. + + +
    • +
+ +##### Changed + +
    +
  • + + +Tyk Gateway now runs on Golang 1.23, bringing security and performance improvements. Key changes include: + +- unbuffered Timer/Ticker channels +- removal of 3DES cipher suites +- updates to X509KeyPair handling. + +**You may need to adjust your setup for compatibility**. For more detail please see the official Go [release notes](https://go.dev/doc/go1.23). + + + +We have updated the library that supports JSON schema validation in the Tyk Classic Validate JSON middleware. This introduces improved error messaging when a request does not match the expected schema, reporting where the error exists in the request payload. + + + +Modified the default values of allow_explicit_policy_id and enable_duplicate_slugs to true in all example configuration files, ensuring consistency and alignment with recommended settings. + + +
    • +
+ + +##### Fixed + +
    +
  • + + +We have fixed an issue where authentication was incorrectly handled for the Internal API when URL Rewrite middleware was used to redirect a request using the `tyk://` protocol. This fix ensures that when API A redirects to API B, authentication with API B will use the method configured for API B, improving access control and preventing access denials. Users can now rely on the expected authentication flow, providing a predictable experience when routing to internal APIs. + + + +Resolved initialization errors that caused unnecessary error logging during gateway startup, improving PID file handling and Redis connection state management. + + + +Fixed an issue where the gateway stopped processing traffic when restarted while MDCB was unavailable. Instead of entering β€œemergency” mode and loading APIs and policies from the Redis backup, the gateway remained unresponsive, continuously attempting to reconnect. With this fix, the gateway detects connection failure and enters emergency mode, ensuring traffic processing resumes even when MDCB is down. + + + +Improved the performance of ctx.GetOASDefinition() in custom plugins by replacing the deep copy operation with a more efficient cloning method. This optimization reduces memory usage by 95% and CPU consumption by 46%, significantly speeding up API definition retrieval. + +Thanks to @sebkehr for identifying this issue and providing valuable feedback to enhance Tyk's performance. + + + +Multi-value response headers were previously lost after synchronization with coprocess middleware, as only the first value was retained. This has been resolved, ensuring all response headers are properly synchronized and preserved + + + +Resolved an issue where the gateway incorrectly selected the OAuth upstream authentication flow when both client credentials and password flows were configured. The gateway now correctly respects the allowedAuthorizeTypes setting, ensuring the intended authentication flow is used. + + +
    • +
+ +--- +## 5.7 Release Notes + +### 5.7.3 Release Notes + +#### Release Date 05 June 2025 + +#### Release Highlights + +This patch release contains a bug fix. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.7.3) below. + +

Dependencies

+##### Compatibility Matrix For Tyk Components + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.7.3 | MDCB v2.7.2 | MDCB v2.4.2 | +| | Operator v1.1.0 | Operator v0.17 | +| | Sync v2.0.2 | Sync v1.4.3 | +| | Helm Chart v2.2 | Helm all versions | +| | EDP v1.12 | EDP all versions | +| | Pump v1.11.1 | Pump all versions | +| | TIB (if using standalone) v1.6.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------ | :---------------------- | :---------------------- | :-------- | +| [Go](https://go.dev/dl/) | 1.22 | 1.22 | [Go plugins](/api-management/plugins/golang) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3)| v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations + +There are no deprecations in this release. + +

Upgrade instructions

+If you are upgrading to 5.7.3, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.7.3) + - ```bash + docker pull tykio/tyk-gateway:v5.7.3 + ``` +- Helm charts + - [tyk-charts v2.2.0](/developer-support/release-notes/helm-chart#220-release-notes) + +- [Source code tarball of Tyk Gateway v5.7.3](https://github.com/TykTechnologies/tyk/releases/tag/v5.7.3) + +

Changelog

+##### Fixed + +
    +
  • + +Resolved an issue introduced in Tyk 5.7.1 where Gateways in distributed Data Planes failed to cache TLS certificates correctly in the local Redis, resulting in potential service disruptions if MDCB became unavailable. Data plane gateways now reliably serve HTTPS and mTLS traffic even if MDCB is unavailable. + +
    • +
+ +--- + +### 5.7.2 Release Notes + +#### Release Date 19 February 2025 + +#### Release Highlights + +This patch release contains a bug fix. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.7.2) below. + +#### Breaking Changes +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.7.2 | MDCB v2.7.2 | MDCB v2.4.2 | +| | Operator v1.1.0 | Operator v0.17 | +| | Sync v2.0.2 | Sync v1.4.3 | +| | Helm Chart v2.2 | Helm all versions | +| | EDP v1.12 | EDP all versions | +| | Pump v1.11.1 | Pump all versions | +| | TIB (if using standalone) v1.6.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------ | :---------------------- | :---------------------- | :-------- | +| [Go](https://go.dev/dl/) | 1.22 | 1.22 | [Go plugins](/api-management/plugins/golang) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3)| v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations +There are no deprecations in this release. + + +

Upgrade instructions

+If you are upgrading to 5.7.2, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.7.2) + - ```bash + docker pull tykio/tyk-gateway:v5.7.2 + ``` +- Helm charts + - [tyk-charts v2.2.0](/developer-support/release-notes/helm-chart#220-release-notes) + +- [Source code tarball of Tyk Gateway v5.7.2](https://github.com/TykTechnologies/tyk/releases/tag/v5.7.2) + +

Changelog

+##### Fixed + +
    +
  • + +Fixed an issue where the gateway stopped processing traffic when restarted while MDCB was unavailable. Instead of entering β€œemergency” mode and loading APIs and policies from the Redis backup, the gateway remained unresponsive, continuously attempting to reconnect. With this fix, the gateway detects connection failure and enters emergency mode, ensuring traffic processing resumes even when MDCB is down. + +
    • +
+ +--- + +### 5.7.1 Release Notes + +#### Release Date 31 December 2024 + +#### Release Highlights + +This release focuses mainly on bug fixes. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.7.1) below. + +#### Breaking Changes +{/* Required. Use the following statement if there are no breaking changes, or explain if there are */} + +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.7.1 | MDCB v2.7.2 | MDCB v2.4.2 | +| | Operator v1.1.0 | Operator v0.17 | +| | Sync v2.0.1 | Sync v1.4.3 | +| | Helm Chart v2.2 | Helm all versions | +| | EDP v1.12 | EDP all versions | +| | Pump v1.11.1 | Pump all versions | +| | TIB (if using standalone) v1.6.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------ | :---------------------- | :---------------------- | :-------- | +| [Go](https://go.dev/dl/) | 1.22 | 1.22 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3)| v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations +{/* Required. Use the following statement if there are no deprecations, or explain if there are */} +There are no deprecations in this release. + + +

Upgrade instructions

+If you are upgrading to 5.7.1, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.7.1) + - ```bash + docker pull tykio/tyk-gateway:v5.7.1 + ``` +- Helm charts + - [tyk-charts v2.2.0](/developer-support/release-notes/helm-chart#220-release-notes) + +- [Source code tarball of Tyk Gateway v5.7.1](https://github.com/TykTechnologies/tyk/releases/tag/v5.7.1) + +

Changelog

+##### Fixed + +
    +
  • + + +Resolved an issue where the response body could be only partially recorded in the traffic log if a custom response plugin modified the payload. This was due to Tyk using the original, rather than the modified, content-length of the response when identifying the data to include in the traffic log. + + + +Fixed a bug that prevented the control plane Gateway from loading APIs that use custom plugin bundles. The control plane Gateway is used to register OAuth clients and generate access tokens so this could result in an API being loaded to the data plane Gateways but clients unable to obtain access tokens. This issue was introduced in v5.3.1 as a side-effect of a change to address a potential security issue where APIs could be loaded without their custom plugins. + + + +Addressed an issue where shared loggers caused debug logs to misidentify the middleware source, complicating debugging. Log entries now correctly indicate which middleware generated the log, ensuring clearer and more reliable diagnostics + + + +Fixed an issue where a malformed listen path could cause the Gateway to crash. Now, such listen paths are properly validated, and if validation fails, an error is logged, and the API is skippedβ€”preventing Gateway instability. + + + +Resolved a bug that prevented upstream server-sent events (SSE) from being sent when OpenTelemetry was enabled, and fixed a gateway panic that occurred when detailed recording was active while SSE was in use. This ensures stable SSE streaming in configurations with OpenTelemetry. + + + +Resolved an issue where API access keys remained valid even if all associated policies were deleted. The Gateway now attempts to apply all linked policies to the key when it is presented with a request. Warning logs are generated if any policies cannot be applied (for example, if they are missing). If no linked policy can be applied, the Gateway will reject the key to ensure no unauthorized access. + + + +Resolved an issue where APIs using the Transfer-Encoding: chunked header alongside URL Rewrite or Validate Request middleware would lose the response payload body. The payload now processes correctly, ensuring seamless functionality regardless of header configuration. + + + +OAuth 2.0 access tokens can now be issued even when data plane gateways are disconnected from the control plane. This is achieved by saving OAuth clients locally within the data plane when they are pulled from RPC. + + + +Tyk now supports RSA-PSS signed JWTs (PS256, PS384, PS512), enhancing security while maintaining backward compatibility with RS256. No configuration changes are neededβ€”just use RSA public keys, and Tyk will validate both algorithms seamlessly. + + + +Resolved a problem in the request size limit middleware that caused GET and DELETE requests to fail validation.The middleware incorrectly expected a request body (payload) for these methods and blocked them when none was present. + + + +Fixed an issue where GraphQL queries using variables for custom scalar types, such as UUID, failed due to incorrect input handling. Previously, the query would return an error when a variable was used but worked when the value was directly embedded in the query. This update ensures that variables for custom scalar types are correctly inferred and processed, enabling seamless query execution. + + +
    • +
+ +--- + +### 5.7.0 Release Notes + +#### Release Date 03 December 2024 + +#### Release Highlights + +We are thrilled to announce new updates and improvements in Tyk 5.7.0, bringing more control, flexibility, and performance. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.7.0) below. + +##### Tyk Streams - asynchronous API management with Tyk + +Tyk is now entering the asynchronous API management space with a bang by delivering Tyk Streams to our users! +Many API management solutions fail to fully support event-driven architectures, causing fragmented management, inconsistent security practices, and increased operational complexity. With event-driven architectures on the rise recently, keeping everything under control and enforcing standards at the organizational level has become a challenge. + +**Tyk Streams** is an event streaming solution available within the Tyk API Management Platform, which applies proven API management principles to simplify event and streams handling. +This release brings capabilities to stream data and events using Kafka, Websocket, SSE and HTTP protocols. It also becomes possible to mediate the message format between Avro and JSON on the fly. + +- Merge together various sources of events to present to consumers as a unified stream. +- Apply authentication and authorization to streams of messages, just as you do for your RESTful APIs +- Expose async APIs via Tyk Portal, so that they are easily discoverable + +All of this possible in self-managed and k8s deployments of Tyk! + +#### Breaking Changes +{/* Required. Use the following statement if there are no breaking changes, or explain if there are */} + +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.7.0 | MDCB v2.7.2 | MDCB v2.4.2 | +| | Operator v1.1.0 | Operator v0.17 | +| | Sync v2.0.1 | Sync v1.4.3 | +| | Helm Chart v2.2 | Helm all versions | +| | EDP v1.12 | EDP all versions | +| | Pump v1.11.1 | Pump all versions | +| | TIB (if using standalone) v1.6.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------ | :---------------------- | :---------------------- | :-------- | +| [Go](https://go.dev/dl/) | 1.22 | 1.22 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3)| v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations +{/* Required. Use the following statement if there are no deprecations, or explain if there are */} +In 5.7.0, we have deprecated the dedicated [External OAuth](/api-management/client-authentication#integrate-with-external-authorization-server-deprecated) (Tyk Classic: `external_oauth`, Tyk OAS: `server.authentication.securitySchemes.externalOAuth`) and [OpenID Connect](/api-management/client-authentication#integrate-with-openid-connect-deprecated) (Tyk Classic: `auth_configs.oidc`, Tyk OAS: `server.authentication.oidc`) authentication methods. We advise users to switch to [JWT Authentication](/basic-config-and-security/security/authentication-authorization/json-web-tokens). + + +

Upgrade instructions

+If you are upgrading to 5.7.0, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.7.0) + - ```bash + docker pull tykio/tyk-gateway:v5.7.0 + ``` +- Helm charts + - [tyk-charts v2.2.0](/developer-support/release-notes/helm-chart#220-release-notes) + +- [Source code tarball of Tyk Gateway v5.7.0](https://github.com/TykTechnologies/tyk/releases/tag/v5.7.0) + +

Changelog

+##### Added + +
    +
  • + + +Added to Streams analytics capability to capture and report common error scenarios, including broker connectivity issues and standard HTTP errors, ensuring comprehensive request tracking for Streams-processed requests. + + + +Connected the new OAS validator to the /streams endpoint, adding proper error handling and validation responses for invalid stream configurations. + + + +Extended the OAS validator to include Streams configuration validation, enforcing allowlisted components and validating nested broker configurations while implementing schema validation for Streams configurations. + + + +Introduced a new validator derived from the existing OAS schema, adapting it for Streams validation with modified requirements for upstreamURL and x-tyk-streaming fields. This validator is now used by both the Dashboard API streams endpoint and streams configuration validator. + + + +Refined streams logging behavior to match Tyk's logging patterns, reducing unnecessary log output and improving log clarity. + + + +Implemented allowlist-based validation for components in streams configurations, replacing the previous blocklist approach. Supported components now include Kafka, WebSocket, SSE, and HTTP for both inputs and outputs (including broker combinations), along with JSON-Avro bidirectional conversion processors, while other components like scanners, caches, and buffers are blocked by default. This validation is enforced consistently across Gateway, Dashboard API, and UI. + + +
    • +
+ +##### Fixed + +
    +
  • + + +When using Tyk Streams and sending input via http, the requests sometimes timed out causing a problem for the consumers. The issue has been fixed and now inputs via http for Tyk Streams work as intended. + + + +Fixed a backwards compatibility issue with Tyk OAS API schema validation. When downgrading from a Tyk version, schema validation could fail if new fields had been added to the Tyk OAS API definition. This change relaxes the strictness of validation to allow additional properties. + + + +Resolved a bug where path-based permissions in policies were not preserved when policies were combined, potentially omitting URL values and incorrectly restricting access. The updated behavior ensures that URL access rights from all applicable policies are merged, regardless of policy order, allowing seamless enforcement of combined permissions. + + + +Fixed a routing issue that caused incorrect API matching when dealing with APIs that lacked a trailing slash, used custom domains, or had similar listen path patterns. Previously, the router prioritized APIs with longer subdomains and shorter listen paths, leading to incorrect matches when listen paths shared prefixes. This fix ensures accurate API matching, even when subdomains and listen paths overlap. + + + +Fixed an issue that caused increased memory consumption when proxying large response payloads. The Gateway now handles large payloads more efficiently in terms of speed and memory usage. + + +
    • +
+ +## 5.6 Release Notes + +### 5.6.1 Release Notes + +#### Release Date 18 October 2024 + +#### Release Highlights + +{/* Required. Use similar ToV to previous release notes. For example for a patch release: +This release primarily focuses on bug fixes. +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-vX.Y.0) below. */} + +This patch release for Tyk Gateway addresses critical stability issues for users running Tyk Gateway within the data +plane, connecting to the control plane or Tyk Hybrid. Affected users should upgrade immediately to version 5.6.1 to +avoid service interruptions and ensure reliable operations with the control plane or Tyk Hybrid. + +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.6.1) below. + +#### Breaking Changes + +{/* Required. Use the following statement if there are no breaking changes, or explain if there are */} + +There are no breaking changes in this release. + +

Dependencies

+{/* Required. Use this section to announce the following types of dependencies compatible with the release: + +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. + +3rd party dependencies and tools */} + +##### Compatibility Matrix For Tyk Components + +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :--------------- | :-------------------------------- | :----------------------- | +| 5.6.1 | MDCB v2.7.1 | MDCB v2.4.2 | +| | Operator v1.0.0 | Operator v0.17 | +| | Sync v2.0 | Sync v1.4.3 | +| | Helm Chart v2.1 | Helm all versions | +| | EDP v1.11 | EDP all versions | +| | Pump v1.11 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------- | :--------------- | :------------------- | :------------------------------------------------------------------------------------------- | +| [Go](https://go.dev/dl/) | 1.22 | 1.22 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the +ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations + +{/* Required. Use the following statement if there are no deprecations, or explain if there are */} + +There are no deprecations in this release. + +{/* Optional section! +Used to share and notify users about our plan to deprecate features, configs etc. +Once you put an item in this section, we must keep this item listed in all the following releases till the deprecation happens. */} +{/* ###### Future deprecations */} + +

Upgrade instructions

+If you are upgrading to 5.6.1, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.6.1) + - ```bash + docker pull tykio/tyk-gateway:v5.6.1 + ``` +- Helm charts + + - [tyk-charts v2.1.0](/developer-support/release-notes/helm-chart#210-release-notes) + +- [Source code tarball of Tyk Gateway v5.6.1](https://github.com/TykTechnologies/tyk/releases/tag/v5.6.1) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. + +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Fixed + +{/* This section should be a bullet point list that describes the issues fixed in the release. For each fixed issue explain: + +- What problem the issue caused +- How was the issue fixed +- Link to (new) documentation created as a result of a fix. For example, a new configuration parameter may have been introduced and documented for the fix +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + +In version 5.6.0, Tyk Gateway could encounter a panic when attempting to reconnect to the control plane after it was +restarted. This patch version has resolved this issue, ensuring stable connectivity between the gateway and control +plane following reconnections and reducing the need for manual intervention. + +
    • +
+ +{/* ##### Security Fixes +This section should be a bullet point list that should be included when any security fixes have been made in the release, e.g. CVEs. For CVE fixes, consideration needs to be made as follows: +1. Dependency-tracked CVEs - External-tracked CVEs should be included on the release note. +2. Internal scanned CVEs - Refer to the relevant engineering and delivery policy. + +For agreed CVE security fixes, provide a link to the corresponding entry on the NIST website. For example: + +- Fixed the following CVEs: + - [CVE-2022-33082](https://nvd.nist.gov/vuln/detail/CVE-2022-33082) + + +
    +
  • + +Fixed the following high priority CVEs identified in the Tyk Gateway, providing increased protection against security vulnerabilities: +- [CVE-2024-6104](https://nvd.nist.gov/vuln/detail/CVE-2024-6104) + +
    • +
*/} + +{/* Required. use 3 hyphens --- between release notes of every patch (minors will be on a separate page) */} + +### 5.6.0 Release Notes + +#### Release Date 10 October 2024 + + + +**Important Update**

Date: 12 October 2024
Topic: Gateway panic when +reconnecting to MDCB control plane or Tyk Cloud
Workaround: Restart Gateway
Affected Product: Tyk +Gateway as an Edge Gateway
Affected versions: v5.6.0, v5.3.6, and v5.0.14
Issue Description:
+ +

We have identified an issue affecting Tyk Gateway deployed as a data plane connecting to the Multi-Data Center Bridge (MDCB) control plane or Tyk Cloud. In the above mentioned Gateway versions a panic may occur when gateway reconnect to the control plane after the control plane is restarted.

+ +

Our engineering team is actively working on a fix, and a patch (versions 5.6.1, 5.3.7, and 5.0.15) will be released soon.

+ +Recommendations:
+
    +
  • For users on versions 5.5.0, 5.3.5, and 5.0.13
    +We advise you to delay upgrading to the affected versions (5.6.0, 5.3.6, or 5.0.14) until the patch is available.
    • + +
  • For users who have already upgraded to 5.6.0, 5.3.6, or 5.0.14 and are experiencing a panic in the gateway:
    +Restarting the gateway process will restore it to a healthy state. If you are operating in a *Kubernetes* environment, Tyk Gateway instance should automatically restart, which ultimately resolves the issue.
    • +
+ +

We appreciate your understanding and patience as we work to resolve this. Please stay tuned for the upcoming patch release, which will address this issue.

+ + + + +#### Release Highlights + +{/* Required. Use similar ToV to previous release notes. For example for a patch release: +This release primarily focuses on bug fixes. +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-vX.Y.0) below. */} + +We are thrilled to announce new updates and improvements in Tyk 5.6.0, bringing more control, flexibility, and +performance. For a comprehensive list of changes, please refer to the detailed +[changelog](#Changelog-v5.6.0) below. + +##### Per endpoint Rate Limiting for clients + +Building on the [per-endpoint upstream rate +limits](/api-management/rate-limit#api-level-rate-limiting) introduced in Tyk 5.5.0 we have +now added [per-endpoint client +rate limits](/api-management/rate-limit#key-level-rate-limiting). This new feature allows +for more granular control over client consumption of API resources by associating the rate limit with the access key, +enabling you to manage and optimize API usage more effectively. + +##### Gateway logs in JSON format + +You can now output Tyk Gateway system logs in JSON format. This allows for easier integration with logging systems and +more structured log data. + +##### Go upgrade to 1.22 + +We’ve upgraded the Tyk Gateway to Golang 1.22, bringing improved performance, better security, and enhanced stability to +the core system. + +#### Breaking Changes + +{/* Required. Use the following statement if there are no breaking changes, or explain if there are */} + +There are no breaking changes in this release. + +

Dependencies

+{/* Required. Use this section to announce the following types of dependencies compatible with the release: + +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. + +3rd party dependencies and tools */} + +##### Compatibility Matrix For Tyk Components + +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :--------------- | :-------------------------------- | :----------------------- | +| 5.6.0 | MDCB v2.7.1 | MDCB v2.4.2 | +| | Operator v1.0.0 | Operator v0.17 | +| | Sync v2.0 | Sync v1.4.3 | +| | Helm Chart v2.1 | Helm all versions | +| | EDP v1.11 | EDP all versions | +| | Pump v1.11 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------- | :--------------- | :------------------- | :------------------------------------------------------------------------------------------- | +| [Go](https://go.dev/dl/) | 1.22 | 1.22 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the +ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations + +{/* Required. Use the following statement if there are no deprecations, or explain if there are */} + +There are no deprecations in this release. + +{/* Optional section! +Used to share and notify users about our plan to deprecate features, configs etc. +Once you put an item in this section, we must keep this item listed in all the following releases till the deprecation happens. */} +{/* ###### Future deprecations */} + +

Upgrade instructions

+If you are upgrading to 5.6.0, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.6.0) + - ```bash + docker pull tykio/tyk-gateway:v5.6.0 + ``` +- Helm charts + + - [tyk-charts v2.1.0](/developer-support/release-notes/helm-chart#210-release-notes) + +- [Source code tarball of Tyk Gateway v5.6.0](https://github.com/TykTechnologies/tyk/releases/tag/v5.6.0) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. + +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Added + +{/* This section should be a bullet point list of new features. Explain: + +- The purpose of the new feature +- How does the new feature benefit users? +- Link to documentation of the new feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + +Building on the [per-endpoint upstream rate +limits](/api-management/rate-limit#api-level-rate-limiting) introduced in Tyk 5.5.0 we have +added [per-endpoint client +rate limits](/api-management/rate-limit#key-level-rate-limiting). This new feature +provided users with more precise control over API resource consumption by linking rate limits to access keys, allowing +for better management and optimization of API usage. + +
    • +
  • + +The Tyk Gateway now supports logging in JSON format. To enable this feature, set the environment variable +`TYK_GW_LOGFORMAT` to `json`. If a different value is provided, the logs will default to the standard format. This +enhancement allows for improved log processing and integration with various monitoring tools. + +
    • +
+ +##### Changed + +{/* This should be a bullet-point list of updated features. Explain: + +- Why was the update necessary? +- How does the update benefit users? +- Link to documentation of the updated feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + +The Tyk Gateway and Tyk Dashboard have been upgraded from Golang 1.21 to Golang 1.22, bringing enhanced performance, +strengthened security, and access to the latest features available in the new Golang release. + +
    • +
+ +##### Fixed + +{/* This section should be a bullet point list that describes the issues fixed in the release. For each fixed issue explain: + +- What problem the issue caused +- How was the issue fixed +- Link to (new) documentation created as a result of a fix. For example, a new configuration parameter may have been introduced and documented for the fix +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +We have enhanced the initial synchronization of Data Plane gateways with the Control Plane to ensure more reliable +loading of policies and APIs on start-up. A synchronous initialization process has been implemented to avoid sync +failures and reduce the risk of service disruptions caused by failed loads. This update ensures smoother and more +consistent syncing of policies and APIs in distributed deployments. + + + +We have fixed an issue where the quota limit was not being consistently respected during request spikes, especially in +deployments with multiple gateways. The problem occurred when multiple gateways cached the current and remaining quota +counters at the end of quota periods. To address this, a distributed lock mechanism has been implemented, ensuring +coordinated quota resets and preventing discrepancies across gateways. + + + +We have fixed an issue where API-level rate limits set in multiple policies were not correctly applied to the same key. +With this update, when multiple policies configure rate limits for a key, the key will now receive the highest rate +limit from the combined policies, ensuring proper enforcement of limits. + + + +We have addressed a performance regression where key creation for policies with a large number of APIs (100+) became +significantly slower in Tyk 4.0.13/5.0.1. The operation, which previously took around 1.5 seconds, has been taking over +20 seconds since versions 4.0.13/5.0.1. This issue has been resolved by optimizing Redis operations during key creation, +restoring the process to the previous duration, even with a large number of APIs in the policy. + + +
    • +
+ +##### Security Fixes + +{/* This section should be a bullet point list that should be included when any security fixes have been made in the release, e.g. CVEs. For CVE fixes, consideration needs to be made as follows: +1. Dependency-tracked CVEs - External-tracked CVEs should be included on the release note. +2. Internal scanned CVEs - Refer to the relevant engineering and delivery policy. + +For agreed CVE security fixes, provide a link to the corresponding entry on the NIST website. For example: + +- Fixed the following CVEs: + - [CVE-2022-33082](https://nvd.nist.gov/vuln/detail/CVE-2022-33082) */} + +
    +
  • + +Fixed the following high priority CVEs identified in the Tyk Gateway, providing increased protection against security +vulnerabilities: + +- [CVE-2024-6104](https://nvd.nist.gov/vuln/detail/CVE-2024-6104) + +
    • +
+ +{/* Required. use 3 hyphens --- between release notes of every patch (minors will be on a separate page) */} + +{/* Repeat the release notes section above for every patch here */} + +{/* The footer of the release notes page. It contains a further information section with details of how to upgrade Tyk, +links to API documentation and FAQs. You can copy it from the previous release. */} + +## 5.5 Release Notes + +### 5.5.2 Release Notes + +#### Release Date 03 October 2024 + +#### Release Highlights +This release replaces Tyk Gateway 5.5.1 which was accidentally released as a non-distroless image. + + +#### Breaking Changes + +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.5.2 | MDCB v2.7 | MDCB v2.4.2 | +| | Operator v0.18 | Operator v0.17 | +| | Sync v1.5 | Sync v1.4.3 | +| | Helm Chart v2.0.0 | Helm all versions | +| | EDP v1.10 | EDP all versions | +| | Pump v1.11 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------ | :---------------------- | :---------------------- | :-------- | +| [Go](https://go.dev/dl/) | 1.21 | 1.21 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3)| v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations +{/* Required. Use the following statement if there are no deprecations, or explain if there are */} +There are no deprecations in this release. + +

Upgrade instructions

+If you are upgrading to 5.5.2, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.5.2) + - ```bash + docker pull tykio/tyk-gateway:v5.5.2 + ``` +- Helm charts + - [tyk-charts v2.0.0](/developer-support/release-notes/helm-chart#200-release-notes) +- [Source code tarball of Tyk Gateway v5.5.2](https://github.com/TykTechnologies/tyk/releases/tag/v5.5.2) + +--- + +### 5.5.1 Release Notes + +#### Release Date 26 September 2024 + +#### Release Highlights +This release fixes some issues related to the way that Tyk performs URL path matching, introducing two new Gateway configuration options to control path matching strictness. + +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.5.1) below. + +#### Breaking Changes + +There are no breaking changes in this release. + +

Dependencies

+##### Compatibility Matrix For Tyk Components + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.5.1 | MDCB v2.7 | MDCB v2.4.2 | +| | Operator v0.18 | Operator v0.17 | +| | Sync v1.5 | Sync v1.4.3 | +| | Helm Chart v2.0.0 | Helm all versions | +| | EDP v1.10 | EDP all versions | +| | Pump v1.11 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------ | :---------------------- | :---------------------- | :-------- | +| [Go](https://go.dev/dl/) | 1.21 | 1.21 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3)| v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations +{/* Required. Use the following statement if there are no deprecations, or explain if there are */} +There are no deprecations in this release. + +

Upgrade instructions

+If you are upgrading to 5.5.1, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.5.1) + - ```bash + docker pull tykio/tyk-gateway:v5.5.1 + ``` +- Helm charts + - [tyk-charts v2.0.0](/developer-support/release-notes/helm-chart#200-release-notes) +- [Source code tarball of Tyk Gateway v5.5.1](https://github.com/TykTechnologies/tyk/releases/tag/v5.5.1) + +

Changelog

+##### Added + +
    +
  • + +We have introduced two new options in the `http_server_options` [Gateway configuration](/tyk-oss-gateway/configuration#http_server_options) that will enforce prefix and/or suffix matching when Tyk performs checks on whether middleware or other logic should be applied to a request: + +- `enable_path_prefix_matching` ensures that the start of the request path must match the path defined in the API definition +- `enable_path_suffix_matching` ensures that the end of the request path must match the path defined in the API definition +- combining `enable_path_prefix_matching` and `enable_path_suffix_matching` will ensure an exact (explicit) match is performed + +These configuration options provide control to avoid unintended matching of paths from Tyk's default *wildcard* match. Use of regex special characters when declaring the endpoint path in the API definition will automatically override these settings for that endpoint. + +Tyk recommends that exact matching is employed, but both options default to `false` to avoid introducing a breaking change for existing users. + +The example Gateway configuration file `tyk.conf.example` has been updated to set the recommended *exact matching* with: + + - `http_server_options.enable_path_prefix_matching = true` + - `http_server_options.enable_path_suffix_matching = true` + - `http_server_options.enable_strict_routes = true` + +
    • +
+ +##### Fixed + +
    +
  • + +Fixed an issue when using granular [Path-Based Permissions](/api-management/policies#secure-your-apis-by-method-and-path) in access policies and keys that led to authorization incorrectly being granted to endpoints if an invalid regular expression was configured in the key/policy. Also fixed an issue where path-based parameters were not correctly handled by Path-Based Permissions. Now Tyk's authorization check correctly handles both of these scenarios granting access only to the expected resources. + +
    • +
  • + +Fixed an issue where a parameterized endpoint URL (e.g. `/user/{id}`) would be invoked if a request is made that omits the parameter. For example, a request to `/user/` will now be interpreted as a request to `/user` and not to `/user/{id}`. + +
    • +
+ +--- + +### 5.5.0 Release Notes + +#### Release Date 12 August 2024 + +#### Release Highlights +{/* Required. Use similar ToV to previous release notes. For example for a patch release: +This release primarily focuses on bug fixes. +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-vX.Y.0) below. */} +We are thrilled to introduce Tyk Gateway 5.5, bringing advanced rate-limiting capabilities, enhanced certificate authentication, and performance optimizations. For a comprehensive list of changes, please refer to the [changelog](#Changelog-v5.5.0) below. + +##### Per Endpoint Rate Limiting + +Now configure rate limits at the endpoint level for both [Tyk OAS](/api-management/rate-limit#tyk-oas-api-definition) and [Tyk Classic APIs](/api-management/rate-limit#tyk-classic-api-definition), providing granular protection for upstream services against overloading and abuse. + +##### Root CA Support for Client Certificates + +Simplify certificate management with support for root Certificate Authority (CA) certificates, enabling clients to authenticate using certificates signed by the [configured root CA](/basic-config-and-security/security/mutual-tls/client-mtls#faq). + +##### Optimised AST Document Handling + +Experience improved performance with optimised creation and usage of Abstract Syntax Tree (AST) documents in our GQL library, reducing memory usage and enhancing efficiency. + +#### Breaking Changes +{/* Required. Use the following statement if there are no breaking changes, or explain if there are */} +Docker images are now based on [distroless](https://github.com/GoogleContainerTools/distroless). No shell is shipped in the image. + +

Dependencies

+{/* Required. Use this section to announce the following types of dependencies compatible with the release: + +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. + +3rd party dependencies and tools */} + +##### Compatibility Matrix For Tyk Components +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.5.0 | MDCB v2.7 | MDCB v2.4.2 | +| | Operator v0.18 | Operator v0.17 | +| | Sync v1.5 | Sync v1.4.3 | +| | Helm Chart v1.6 | Helm all versions | +| | EDP v1.10 | EDP all versions | +| | Pump v1.11 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools +{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------ | :---------------------- | :---------------------- | :-------- | +| [Go](https://go.dev/dl/) | 1.21 | 1.21 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3)| v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations +{/* Required. Use the following statement if there are no deprecations, or explain if there are */} +There are no deprecations in this release. + +{/* Optional section! +Used to share and notify users about our plan to deprecate features, configs etc. +Once you put an item in this section, we must keep this item listed in all the following releases till the deprecation happens. */} +{/* ###### Future deprecations */} + +

Upgrade instructions

+If you are upgrading to 5.5.0, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.5.0) + - ```bash + docker pull tykio/tyk-gateway:v5.5.0 + ``` +- Helm charts + - [tyk-charts v1.6](/developer-support/release-notes/helm-chart#160-release-notes) +- [Source code tarball of Tyk Gateway v5.5.0](https://github.com/TykTechnologies/tyk/releases/tag/v5.5.0) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. + +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Added +{/* This section should be a bullet point list of new features. Explain: + +- The purpose of the new feature +- How does the new feature benefit users? +- Link to documentation of the new feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +We've added support for you to register Certificate Authority (CA) certificates in your API definitions when using static mutual TLS (mTLS). Tyk can now authenticate clients presenting certificates signed by the registered root CA, simplifying certificate management for multiple clients sharing a common CA. + + + +Optimised the creation and usage of AST documents in our GQL library to reduce significant memory allocations caused by pre-allocations during initial creation. These optimizations free up resources more efficiently, minimising performance penalties with increased requests to the Gateway. + + + +Introduced new more granular controls for request rate limiting. Rate limits can now be configured at the endpoint level in Tyk OAS and Tyk Classic API definitions. + + + +When using the URL path to indicate the API version (for example `/v1/my-api`) it is common to strip the version identifier (e.g. `/v1`) from the path before proxying the request to the upstream. If the client doesn't provide any version identifier this could lead to an invalid target URL and failed requests, rather than correctly redirecting to the default version. We have introduced an optional configuration `url_versioning_pattern` where you can specify a regex that Tyk will use to identify if the URL contains a version identifier and avoiding the accidental stripping of valid upstream path. + + +
    • +
+ +##### Fixed +{/* This section should be a bullet point list that describes the issues fixed in the release. For each fixed issue explain: + +- What problem the issue caused +- How was the issue fixed +- Link to (new) documentation created as a result of a fix. For example, a new configuration parameter may have been introduced and documented for the fix +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +Fixed an issue when using Tyk OAS APIs where nested API endpoints, such as '/test' and '/test/abc', might incorrectly apply middleware from the parent path to the nested path. The fix ensures that API endpoint definitions are correctly ordered so that the standard behaviour of Tyk is followed, whereby path matching is performed starting from the longest path, preventing middleware misapplication and ensuring both the HTTP method and URL match accurately. + + + +Previously, key creation or reset led to an exponential number of Redis `DeleteRawKey` commands; this was especially problematic for access lists with over 100 entries. The key creation sequence now runs only once, eliminating redundant deletion of non-existent keys in Redis. This optimization significantly reduces deletion events, enhancing performance and stability for larger access lists. + + + +Addressed a bug that caused Server Side Event (SSE) streaming responses to be considered for caching, which required buffering the response and prevented SSE from being correctly proxied. + + + +Resolved an issue where Host and Latency fields (Total and Upstream) were not correctly reported for Tyk Gateways in MDCB data planes. The fix ensures accurate Host values and Latency measurements are now captured and displayed in the generated traffic logs. + + +
    • +
+ + +##### Security Fixes +{/* This section should be a bullet point list that should be included when any security fixes have been made in the release, e.g. CVEs. For CVE fixes, consideration needs to be made as follows: +1. Dependency-tracked CVEs - External-tracked CVEs should be included on the release note. +2. Internal scanned CVEs - Refer to the relevant engineering and delivery policy. + +For agreed CVE security fixes, provide a link to the corresponding entry on the NIST website. For example: + +- Fixed the following CVEs: + - [CVE-2022-33082](https://nvd.nist.gov/vuln/detail/CVE-2022-33082) */} + +
    +
  • + +Fixed the following high priority CVEs identified in the Tyk Gateway, providing increased protection against security vulnerabilities: +- [CVE-2023-39325](https://nvd.nist.gov/vuln/detail/CVE-2023-39325) +- [CVE-2023-45283](https://nvd.nist.gov/vuln/detail/CVE-2023-45283) + +
    • +
+ +{/* Required. use 3 hyphens --- between release notes of every patch (minors will be on a separate page) */} +--- + +{/* Repeat the release notes section above for every patch here */} + + +{/* The footer of the release notes page. It contains a further information section with details of how to upgrade Tyk, +links to API documentation and FAQs. You can copy it from the previous release. */} +## 5.4 Release Notes +### 5.4.0 Release Notes + +#### Release Date 2 July 2024 + +#### Breaking Changes +{/* Required. Use the following statement if there are no breaking changes, or explain if there are */} +**Attention: Please read this section carefully** + +We have fixed a bug in the way that Tyk calculates the [key-level rate limit](/api-management/rate-limit#key-level-rate-limiting) when multiple policies are applied to the same key. This fix alters the logic used to calculate the effective rate limit and so may lead to a different rate limit being applied to keys generated from your existing policies. See the [change log](#fixed) for details of the change. + +

Dependencies

+{/* Required. Use this section to announce the following types of dependencies compatible with the release: + +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. + +3rd party dependencies and tools */} + +##### Compatibility Matrix For Tyk Components +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.4.0 | MDCB v2.6 | MDCB v2.4.2 | +| | Operator v0.18 | Operator v0.17 | +| | Sync v1.5 | Sync v1.4.3 | +| | Helm Chart v1.5.0 | Helm all versions | +| | EDP v1.9 | EDP all versions | +| | Pump v1.10.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +The above table needs reviewing and updating if necessary + +##### 3rd Party Dependencies & Tools +{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------ | :---------------------- | :---------------------- | :-------- | +| [Go](https://go.dev/dl/) | 1.19 (GQL), 1.21 (GW) | 1.19 (GQL), 1.21 (GW) | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3)| v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +**The above table needs reviewing and updating if necessary** + +#### Deprecations +{/* Required. Use the following statement if there are no deprecations, or explain if there are */} +There are no deprecations in this release. + +{/* Optional section! +Used to share and notify users about our plan to deprecate features, configs etc. +Once you put an item in this section, we must keep this item listed in all the following releases till the deprecation happens. */} +{/* ###### Future deprecations */} + +

Upgrade instructions

+If you are upgrading to 5.4.0, please follow the detailed [upgrade instructions](#upgrading-tyk). + +Add upgrade steps here if necessary. + +#### Release Highlights +{/* Required. Use similar ToV to previous release notes. For example for a patch release: +This release primarily focuses on bug fixes. +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-vX.Y.0) below. */} +We're thrilled to introduce exciting enhancements in Tyk Gateway 5.4, aimed at improving your experience with Tyk Gateway. For a comprehensive list of changes, please refer to the change log below. + +##### Enhanced Rate Limiting Strategies + +We've introducing a [Rate Limit Smoothing](/api-management/rate-limit#rate-limit-smoothing) option for the spike arresting Redis Rate Limiter to give the upstream time to scale in response to increased request rates. + +##### Fixed MDCB Issue Relating To Replication Of Custom Keys To Dataplanes + +Resolved an issue encountered in MDCB environments where changes to custom keys made via the Dashboard were not properly replicated to data planes. The issue impacted both key data and associated quotas, in the following versions: + +- 5.0.4 to 5.0.12 +- 5.1.1 and 5.1.2 +- 5.2.0 to 5.2.6 +- 5.3.0 to 5.3.2 + +###### Action Required +Customers should clear their edge Redis instances of any potentially affected keys to maintain data consistency and ensure proper synchronization across their environments. Please refer to the item in the [fixed](#fixed) section of the changelog for recommended actions. + +##### Fixed Window Rate Limiter + +Ideal for persistent connections with load-balanced gateways, the [Fixed Window Rate Limiter](/api-management/rate-limit#fixed-window-rate-limiter) algorithm mechanism ensures fair handling of requests by allowing only a predefined number to pass per rate limit window. It uses a simple shared counter in Redis so requests do not need to be evenly balanced across the gateways. + +##### Event handling with Tyk OAS + +We’ve added support for you to [register webhooks](/api-management/gateway-events#event-handling-with-webhooks) with your Tyk OAS APIs so that you can handle events triggered by the Gateway, including circuit breaker and quota expiry. You can also assign webhooks to be fired when using the new [smoothing rate limiter](/api-management/rate-limit#rate-limit-smoothing) to notify your systems of ongoing traffic spikes. + +##### Enhanced Header Handling in GraphQL APIs + +Introduced a features object in API definitions for GQL APIs, including the `use_immutable_headers` attribute. This allows advanced header control, enabling users to add new headers, rewrite existing ones, and selectively remove specific headers. Existing APIs will have this attribute set to `false` by default, ensuring no change in behavior. For new APIs, this attribute is true by default, facilitating smoother migration and maintaining backward compatibility. + +#### Downloads +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.4.0) + - ```bash + docker pull tykio/tyk-gateway:v5.4.0 + ``` +- Helm charts + - [tyk-charts v1.5](/developer-support/release-notes/helm-chart#150-release-notes) +- [Source code tarball of Tyk Gateway v5.4.0](https://github.com/TykTechnologies/tyk/releases/tag/v5.4.0) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. + +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Added +{/* This section should be a bullet point list of new features. Explain: + +- The purpose of the new feature +- How does the new feature benefit users? +- Link to documentation of the new feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +Introduced a [Fixed Window Rate Limiting](/api-management/rate-limit#fixed-window-rate-limiter) mechanism to handle rate limiting for load balancers with keep-alives. This algorithm allows the defined number of requests to pass for every rate limit window and blocks any excess requests. It uses a simple shared counter in Redis to count requests. It is suitable for situations where traffic towards Gateways is not balanced fairly. To enable this rate limiter, set `enable_fixed_window_rate_limiter` in the gateway config or set the environment variable `TYK_GW_ENABLEFIXEDWINDOWRATELIMITER=true`. + + + +Implemented [Rate Limit Smoothing](/api-management/rate-limit#rate-limit-smoothing) as an extension to the existing Redis Rate Limiter to gradually adjust the rate based on smoothing configuration. Two new Gateway events have been created (`RateLimitSmoothingUp` and `RateLimitSmoothingDown`) which will be triggered as smoothing occurs. These can be used to assist with auto-scaling of upstream capacity during traffic spikes. + + + +We've added the `use_immutable_headers` option to the GraphQL API configuration, offering advanced header transformation capabilities. When enabled, users can add new headers, rewrite existing ones, and selectively remove specific headers, allowing granular control without altering the original request. Existing APIs will default to `false`, maintaining current behavior until ready for upgrade. + + + +Introduced an option for users to manually provide GQL schemas when creating APIs in Tyk, eliminating the dependency on upstream introspection. This feature enables the creation and editing of GQL APIs in Tyk even when upstream introspection is unavailable, providing flexibility for schema management as upstream configurations evolve over time. + + + +The new GraphQL engine, version 3-preview, is now available in Tyk Gateway. It can be used for any GQL API by using the following enum in raw API definition: *"version": "3-preview"*. This experimental version offers optimized GQL operation resolution, faster response times, and a more efficient data loader. It is currently not recommended for production use and will be stabilised in future releases, eventually becoming the default for new GQL APIs in Tyk. + + + +Enhanced request headers handling in API definitions for GQL APIs by introducing a *features* object. Users can now set the `use_immutable_headers` attribute, which defaults to false for existing APIs, ensuring no change in header behavior. For new APIs, this attribute is `true` by default, facilitating smoother migration and maintaining backwards compatibility. + + + +We’ve added some more features to the Tyk OAS API, moving closer to full parity with Tyk Classic. In this release we’ve added controls that allow you: to enable or prevent generation of traffic logs at the API-level and to enable or prevent the availability of session context to middleware. We’ve also added the facility to register webhooks that will be fired in response to Gateway events. + + +
    • +
+ +##### Fixed +{/* This section should be a bullet point list that describes the issues fixed in the release. For each fixed issue explain: + +- What problem the issue caused +- How was the issue fixed +- Link to (new) documentation created as a result of a fix. For example, a new configuration parameter may have been introduced and documented for the fix +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +Resolved a critical issue affecting MDCB environments, where changes to custom keys made via the dashboard were not properly replicated to data planes. This affected both the key data and associated quotas. This issue was present in versions: +- 5.0.4 to 5.0.12 +- 5.1.1 and 5.1.2 +- 5.2.0 to 5.2.6 +- 5.3.0 to 5.3.2 + +**Action Required** + +Customers are advised to clear their edge Redis instances of any keys that might have been affected by this bug to ensure data consistency and proper synchronization across their environments. There are several methods available to address this issue: + +1. **Specific Key Deletion via API**: To remove individual buggy keys, you can use the following API call: + +```bash +curl --location --request DELETE 'http://tyk-gateway:{tyk-hybrid-port}/tyk/keys/my-custom-key' \ --header 'X-Tyk-Authorization: {dashboard-key}' +``` + +Replace `{tyk-hybrid-port}`, `my-custom-key` and `{dashboard-key}` with your specific configuration details. This method is safe and recommended for targeted removals without affecting other keys. + +2. **Bulk Key Deletion Using Redis CLI**: For environments with numerous affected keys, you might consider using the Redis CLI to remove keys en masse: + +```bash +redis-cli --scan --pattern 'apikey-*' | xargs -L 1 redis-cli del +redis-cli --scan --pattern 'quota-*' | xargs -L 1 redis-cli del +``` + +This method can temporarily impact the performance of the Redis server, so it should be executed during a maintenance window or when the impact on production traffic is minimal. + +3. **Complete Redis Database Flush**: If feasible, flushing the entire Redis database offers a clean slate: + +```bash +redis-cli FLUSHALL ASYNC +``` expandable + +**Implications** +Regardless of the chosen method, be aware that quotas will be reset and will need to resynchronize across the system. This may temporarily affect reporting and rate limiting capabilities. + + + +Addressed an issue with service discovery where an IP returned by Consul wasn't parsed correctly on the Gateway side, leading to unexpected errors when proxying requests to the service. Typically, service discovery returns valid domain names, which did not trigger the issue. + + + +Fixed an issue where GQL Open Telemetry semantic conventions attribute names that lacked the 'graphql' prefix, deviating from the community standard. All attributes now have the correct prefix. + + + +Corrected an issue where GraphQL OTel attributes were missing from spans when request validation failed in cases where `detailed_tracing` was set to `false`. Traces now include GraphQL attributes (operation name, type, and document), improving debugging for users. + + + +Fixed a gateway panic issue observed by users when using the *Persist GQL* middleware without defined arguments. The gateway will no longer throw panics in these cases. + + + +Fixed an issue with GraphQL API's Cross-Origin Resource Sharing (CORS) configuration, which previously caused the API to fail in respecting CORS settings. This resulted in an inability to proxy requests to upstream servers and handle OPTIONS/CORS requests correctly. With this fix, users can now seamlessly make requests, including OPTIONS method requests, without encountering the previously reported error. + + + +Fixed an issue where the Gateway did not respect API domain settings when there was another API with the same listen path but no domain. This could lead to the custom domain API not functioning correctly, depending on the order in which APIs were loaded. APIs with custom domains are now prioritised before those without custom domains to ensure that the custom domain is not ignored. + + + +Addressed a problem with nested field mapping in UDG for GraphQL (GQL) operations. Previously, querying a single nested field caused an error, while including another *normal* field from the same level allowed the query to succeed. This issue has been fixed to ensure consistent behavior regardless of the query composition. + + + +Fixed a long-standing bug in the algorithm used to determine the effective rate limit when multiple policies are applied to a key. If more than one policy is applied to a key then Tyk will apply the highest request rate permitted by any of the policies that defines a rate limit. + +Rate limits in Tyk are defined using two elements: `rate`, which is the number of requests and `per`, which is the period over which those requests can be sent. So, if `rate` is 90 and `per` is 30 seconds for a key, Tyk will permit a maximum of 90 requests to be made using the key in a 30 second period, giving an effective maximum of 180 requests per minute (or 3 rps). + +Previously, Tyk would take the highest `rate` and the highest `per` from the policies applied to a key when determining the effective rate limit. So, if policy A had `rate` set to 90 and `per` set to 30 seconds (3rps) while policy B had `rate` set to 100 and `per` set to 10 seconds (10rps) and both were applied to a key, the rate limit configured in the key would be: `rate = 100` and `per = 30` giving a rate of 3.33rps. + +With the fix applied in Tyk 5.4.0, the Gateway will now apply the highest effective rate to the key - so in this example, the key would take the rate limit from policy B: `rate = 100` and `per = 10` (10rps). + +Note that this corrected logic is applied when access keys are presented in API requests. If you are applying multiple policies to keys, there may be a change in the effective rate limit when using Tyk 5.4.0 compared with pre-5.4.0 versions. + + +
    • +
+ + +##### Security Fixes +{/* This section should be a bullet point list that should be included when any security fixes have been made in the release, e.g. CVEs. For CVE fixes, consideration needs to be made as follows: +1. Dependency-tracked CVEs - External-tracked CVEs should be included on the release note. +2. Internal scanned CVEs - Refer to the relevant engineering and delivery policy. + +For agreed CVE security fixes, provide a link to the corresponding entry on the NIST website. For example: + +- Fixed the following CVEs: + - [CVE-2022-33082](https://nvd.nist.gov/vuln/detail/CVE-2022-33082) */} + +
    +
  • + +Fixed the following high priority CVEs identified in the Tyk Gateway, providing increased protection against security vulnerabilities: +- [CVE-2023-39325](https://nvd.nist.gov/vuln/detail/CVE-2023-39325) +- [CVE-2023-45283](https://nvd.nist.gov/vuln/detail/CVE-2023-45283) + +
    • +
+ +{/* Required. use 3 hyphens --- between release notes of every patch (minors will be on a separate page) */} +--- + +{/* Repeat the release notes section above for every patch here */} + + +{/* The footer of the release notes page. It contains a further information section with details of how to upgrade Tyk, +links to API documentation and FAQs. You can copy it from the previous release. */} + +## 5.3 Release Notes + +### 5.3.11 Release Notes + +#### Release Date 7 May 2025 + +#### Release Highlights + +This patch release contains various bug fixes. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.3.11) below. + +#### Breaking Changes + +This release has no breaking changes. + +#### Dependencies + +##### Compatibility Matrix For Tyk Components + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.3.11 | MDCB v2.8.0 | MDCB v2.8.0 | +| | Operator v1.2.0 | Operator v0.17 | +| | Sync v2.1.0 | Sync v2.1.0 | +| | Helm Chart v3.0 | Helm all versions | +| | EDP v1.13 | EDP all versions | +| | Pump v1.12.0 | Pump all versions | +| | TIB (if using standalone) v1.7.0 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------ | :---------------------- | :---------------------- | :-------- | +| [Go](https://go.dev/dl/) | 1.23 | 1.23 | [Go plugins](/api-management/plugins/golang) must be built using Go 1.23 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3)| v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations + +There are no deprecations in this release. + +

Upgrade instructions

+If you are upgrading to 5.3.11, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.3.11) + - ```bash + docker pull tykio/tyk-gateway:v5.3.11 + ``` +- Helm charts + - [tyk-charts v3.0.0](/developer-support/release-notes/helm-chart#300-release-notes) + +- [Source code tarball of Tyk Gateway 5.3.11](https://github.com/TykTechnologies/tyk/releases/tag/v5.3.11) + +

Changelog

+##### Added + +
    +
  • + +We have added GODEBUG flags to enable deprecated insecure ciphers by default for backward compatibility. Existing users will not be affected. New users or those who wish to override these settings can do so at runtime using environment variables. + +
    • +
+ +##### Fixed + +
    +
  • + + +Addressed an issue for UDG APIs where caching led to the forwarding of stale values for headers that contained content variables towards the upstream of the UDG apis. + + + +Resolved an issue where requests could be routed incorrectly due to inverted prioritisation of dynamically declared paths over those with similar static paths. Now, statically declared paths take priority in the path matching algorithm, so if API1 has listen path `/path/{param}/endpoint` and API2 has listen path `/path/specific/endpoint` a request to `/path/specific/endpoint/resource` will be correctly routed to API2. + + + +Fixed an issue where an [enforced timeout](/planning-for-production/ensure-high-availability/enforced-timeouts) set for a specific API endpoint could be overruled by the configured [proxy_default_timeout](/tyk-oss-gateway/configuration#proxy_default_timeout). Now if an endpoint-level timeout is set then this will be honoured, regardless of any default timeout that is configured. + + + +Resolved a race condition in self-managed deployments which occasionally lead to fewer Gateways registering with the Dashboard than the number that had been licensed. Now Tyk Self-Managed deployments will allow the licensed number of Gateways to register and serve traffic. + + + +Resolved a bug where Gateway pods in Kubernetes would enter a crash loop on restart if MDCB was down. The issue occurred due to the HTTP router failing to initialize properly during cold start. This fix ensures stable Gateway recovery even when MDCB is offline. + + + +Multi-value response headers were previously lost after synchronization with coprocess middleware, as only the first value was retained. This has been resolved, ensuring all response headers are properly synchronized and preserved + + +
    • +
+ + +### 5.3.10 Release Notes + +#### Release Date 19 February 2025 + +#### Release Highlights + +In this release, we upgraded the Golang version to `v1.23` for security enhancement and fixed an API authentication issue with redirects. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.3.10) below. + +#### Breaking Changes + +This release has no breaking changes. + +#### Dependencies + +##### Compatibility Matrix For Tyk Components + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :--------------- | :------------------------------------------------------------------ | :----------------------- | +| 5.3.10 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v2.0.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------- | :--------------------- | :--------------------- | :------------------------------------------------------------------------------------------ | +| [Go](https://go.dev/dl/) | 1.23 (GW) | 1.23 (GW) | [Go plugins](/api-management/plugins/golang) must be built using Go 1.23 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the +ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations + +There are no deprecations in this release + +#### Upgrade Instructions + +If you are upgrading to 5.3.10, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.3.10) + - ```bash + docker pull tykio/tyk-gateway:v5.3.10 + ``` +- Helm charts + - [tyk-charts v2.0.0](/developer-support/release-notes/helm-chart#200-release-notes) +- [Source code tarball of Tyk Gateway 5.3.10](https://github.com/TykTechnologies/tyk/releases/tag/v5.3.10) + +

Changelog

+##### Fixed + +
    +
  • + + +Fixed an issue where the gateway stopped processing traffic when restarted while MDCB was unavailable. Instead of entering "emergency" mode and loading APIs and policies from the Redis backup, the gateway remained unresponsive, continuously attempting to reconnect. +With this fix, the gateway detects connection failure and enters `emergency` mode, ensuring traffic processing resumes even when MDCB is down. + + + +Tyk Gateway now runs on Golang 1.23, bringing security and performance improvements. Key changes include unbuffered Timer/Ticker channels, removal of 3DES cipher suites, and updates to X509KeyPair handling. Users may need to adjust their setup for compatibility. + + + +This fix ensures that when API A redirects to API B using the tyk:// scheme, API B will now correctly authenticate using its own credentials, improving access control and preventing access denials. Users can now rely on the expected authentication flow without workarounds, providing a smoother experience when integrating APIs. + + +
    • +
+ +### 5.3.9 Release Notes + +#### Release Date 31 December 2024 + +#### Release Highlights + +This release contains bug fixes. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.3.9) below. + +#### Breaking Changes + +This release has no breaking changes. + +#### Dependencies + +##### Compatibility Matrix For Tyk Components + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :--------------- | :------------------------------------------------------------------ | :----------------------- | +| 5.3.9 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v2.0.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------- | :--------------------- | :--------------------- | :------------------------------------------------------------------------------------------ | +| [Go](https://go.dev/dl/) | 1.22 (GW) | 1.22 (GW) | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the +ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations + +There are no deprecations in this release + +#### Upgrade Instructions + +If you are upgrading to 5.3.9, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.3.9) + - ```bash + docker pull tykio/tyk-gateway:v5.3.9 + ``` +- Helm charts + - [tyk-charts v2.0.0](/developer-support/release-notes/helm-chart#200-release-notes) +- [Source code tarball of Tyk Gateway v5.3.9](https://github.com/TykTechnologies/tyk/releases/tag/v5.3.9) + +

Changelog

+##### Fixed + +
    +
  • + + +Resolved an issue where the response body could be only partially recorded in the traffic log if a custom response plugin modified the payload. This was due to Tyk using the original, rather than the modified, content-length of the response when identifying the data to include in the traffic log. + + + +Fixed a bug that prevented the control plane Gateway from loading APIs that use custom plugin bundles. The control plane Gateway is used to register OAuth clients and generate access tokens so this could result in an API being loaded to the data plane Gateways but clients unable to obtain access tokens. This issue was introduced in v5.3.1 as a side-effect of a change to address a potential security issue where APIs could be loaded without their custom plugins. + + + +Addressed an issue where shared loggers caused debug logs to misidentify the middleware source, complicating debugging. Log entries now correctly indicate which middleware generated the log, ensuring clearer and more reliable diagnostics + + + +Resolved an issue where APIs using the Transfer-Encoding: chunked header alongside URL Rewrite or Validate Request middleware would lose the response payload body. The payload now processes correctly, ensuring seamless functionality regardless of header configuration. + + + +Resolved an issue where API access keys remained valid even if all associated policies were deleted. The Gateway now attempts to apply all linked policies to the key when it is presented with a request. Warning logs are generated if any policies cannot be applied (for example, if they are missing). If no linked policy can be applied, the Gateway will reject the key to ensure no unauthorized access. + + + +Fixed a routing issue that caused incorrect API matching when dealing with APIs that lacked a trailing slash, used custom domains, or had similar listen path patterns. Previously, the router prioritized APIs with longer subdomains and shorter listen paths, leading to incorrect matches when listen paths shared prefixes. This fix ensures accurate API matching, even when subdomains and listen paths overlap. + + + +Fixed an issue where a malformed listen path could cause the Gateway to crash. Now, such listen paths are properly validated, and if validation fails, an error is logged, and the API is skippedβ€”preventing Gateway instability. + + + +Fixed an issue where GraphQL queries using variables for custom scalar types, such as UUID, failed due to incorrect input handling. Previously, the query would return an error when a variable was used but worked when the value was directly embedded in the query. This update ensures that variables for custom scalar types are correctly inferred and processed, enabling seamless query execution. + + + +Resolved a bug that prevented upstream server-sent events (SSE) from being sent when OpenTelemetry was enabled, and fixed a gateway panic that occurred when detailed recording was active while SSE was in use. This ensures stable SSE streaming in configurations with OpenTelemetry. + + + +OAuth 2.0 access tokens can now be issued even when data plane gateways are disconnected from the control plane. This is achieved by saving OAuth clients locally within the data plane when they are pulled from RPC. + + + +Tyk now supports RSA-PSS signed JWTs (PS256, PS384, PS512), enhancing security while maintaining backward compatibility with RS256. No configuration changes are neededβ€”just use RSA public keys, and Tyk will validate both algorithms seamlessly. + + + +Resolved a problem in the request size limit middleware that caused GET and DELETE requests to fail validation.The middleware incorrectly expected a request body (payload) for these methods and blocked them when none was present. + + +
    • +
+ +--- + +### 5.3.8 Release Notes + +#### Release Date 07 November 2024 + +#### Release Highlights + +This release focuses mainly on bug fixes. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.3.8) below. + +#### Breaking Changes + +This release has no breaking changes. + +#### Dependencies + +{/* Required. Use this section to announce the following types of dependencies compatible with the release: +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +3rd party dependencies and tools */} + +##### Compatibility Matrix For Tyk Components + +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :--------------- | :------------------------------------------------------------------ | :----------------------- | +| 5.3.8 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v2.0.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------- | :--------------------- | :--------------------- | :------------------------------------------------------------------------------------------ | +| [Go](https://go.dev/dl/) | 1.22 (GW) | 1.22 (GW) | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the +ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations + +This is an advanced notice that the dedicated External OAuth, OpenID Connect (OIDC) authentication options, and SQLite support will be deprecated starting in version 5.7.0. We recommend that users of the [External OAuth](/api-management/client-authentication#integrate-with-external-authorization-server-deprecated) and [OpenID Connect](/api-management/client-authentication#integrate-with-openid-connect-deprecated) methods migrate to Tyk's dedicated [JWT Auth](/basic-config-and-security/security/authentication-authorization/json-web-tokens) method. Please review your API configurations, as the Gateway logs will provide notifications for any APIs utilizing these methods. + + +#### Upgrade Instructions + +If you are upgrading to 5.3.8, please follow the detailed [upgrade instructions](#upgrading-tyk). + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.3.8) + - ```bash + docker pull tykio/tyk-gateway:v5.3.8 + ``` +- Helm charts + - [tyk-charts v2.0.0](/developer-support/release-notes/helm-chart#200-release-notes) +- [Source code tarball of Tyk Gateway v5.3.8](https://github.com/TykTechnologies/tyk/releases/tag/v5.3.8) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Added +
    +
  • + +A deprecation notice for External OAuth and OpenID Connect (OIDC) authentication mechanisms has been implemented in the Gateway logs starting from version 5.3.8. This provides advanced notification to users regarding any APIs configured with these authentication methods in preparation for future upgrades where these middleware options may be removed in version 5.7.0. + +
    • +
+ +##### Fixed + +
    +
  • + + +This update fixes a bug that caused increased memory usage when proxying large response payloads that was introduced in version 5.3.1, restoring memory requirements to the levels seen in version 5.0.6. Users experiencing out-of-memory errors with 1GB+ file downloads will notice improved performance and reduced latency. + + + +We resolved an issue that caused path-based permissions in policies to be lost when policies were combined, potentially omitting URL values and restricting access based on the merge order. It ensures that all applicable policies merge their allowed URL access rights, regardless of the order in which they are applied. + + + +A backwards compatibility issue in the way that the Gateway handles Tyk OAS API definitions has been addressed by reducing the strictness of validation against the expected schema. Since Tyk version 5.3, the Gateway has enforced strict validation, potentially causing problems for users downgrading from newer versions. With this change, Tyk customers can move between versions seamlessly, ensuring their APIs remain functional and avoiding system performance issues. + + + +This update resolves an issue where API keys could be lost if the [keyspace synchronization](/api-management/mdcb#synchroniser-feature-with-mdcb) between control and data planes was interrupted. The solution now enforces a resynchronization whenever a connection is re-established between MDCB and the data plane, ensuring key data integrity and seamless API access. + + +
    • +
+ +--- + +### 5.3.7 Release Notes + +#### Release Date 22 October 2024 + +#### Release Highlights + +This patch release for Tyk Gateway addresses critical stability issues for users running Tyk Gateway within the data +plane, connecting to the control plane or Tyk Hybrid. Affected users should upgrade immediately to version 5.3.7 to +avoid service interruptions and ensure reliable operations with the control plane or Tyk Hybrid. + +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.3.7) below. + +#### Breaking Changes + +There are no breaking changes in this release. + +#### Deprecations + +There are no deprecations in this release. + +#### Upgrade Instructions + +When upgrading to 5.3.7 please follow the [detailed upgrade instructions](#upgrading-tyk). + +#### Dependencies + +{/* Required. Use this section to announce the following types of dependencies compatible with the release: +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +3rd party dependencies and tools */} + +##### Compatibility Matrix For Tyk Components + +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :--------------- | :------------------------------------------------------------------ | :----------------------- | +| 5.3.7 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v2.0.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------- | :--------------- | :------------------- | :------------------------------------------------------------------------------------------ | +| [Go](https://go.dev/dl/) | 1.22 | 1.22 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the +ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.3.7) + - ```bash + docker pull tykio/tyk-gateway:v5.3.7 + ``` +- Helm charts + - [tyk-charts v2.0.0](/developer-support/release-notes/helm-chart#200-release-notes) +- [Source code tarball of Tyk Gateway v5.3.7](https://github.com/TykTechnologies/tyk/releases/tag/v5.3.7) + +

Changelog

+##### Fixed + + {/* This section should be a bullet point list that describes the issues fixed in the release. For each fixed issue explain: + - What problem the issue caused + - How was the issue fixed + - Link to (new) documentation created as a result of a fix. For example, a new configuration parameter may have been introduced and documented for the fix + - For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + +In version 5.3.6, Tyk Gateway could encounter a panic when attempting to reconnect to the control plane after it was restarted. This patch version has resolved this issue, ensuring stable connectivity between the gateway and control plane following reconnections and reducing the need for manual intervention. + +
    • +
+ +{/* ##### Security Fixes +This section should be a bullet point list that should be included when any security fixes have been made in the release, e.g. CVEs. For CVE fixes, consideration needs to be made as follows: +1. Dependency-tracked CVEs - External-tracked CVEs should be included on the release note. +2. Internal scanned CVEs - Refer to the relevant engineering and delivery policy. + +For agreed CVE security fixes, provide a link to the corresponding entry on the NIST website. For example: + +- Fixed the following CVEs: + - [CVE-2022-33082](https://nvd.nist.gov/vuln/detail/CVE-2022-33082) */} + +--- + +### 5.3.6 Release Notes + +#### Release Date 04 October 2024 + + + +**Important Update**

Date: 12 October 2024
Topic: Gateway panic when +reconnecting to MDCB control plane or Tyk Cloud
Workaround: Restart Gateway
Affected Product: Tyk +Gateway as an Edge Gateway
Affected versions: v5.6.0, v5.3.6, and v5.0.14
Issue Description:
+ +

We have identified an issue affecting Tyk Gateway deployed as a data plane connecting to the Multi-Data Center Bridge (MDCB) control plane or Tyk Cloud. In the above mentioned Gateway versions a panic may occur when gateway reconnect to the control plane after the control plane is restarted.

+ +

Our engineering team is actively working on a fix, and a patch (versions 5.6.1, 5.3.7, and 5.0.15) will be released soon.

+ +Recommendations:
+
    +
  • For users on versions 5.5.0, 5.3.5, and 5.0.13
    +We advise you to delay upgrading to the affected versions (5.6.0, 5.3.6, or 5.0.14) until the patch is available.
    • + +
  • For users who have already upgraded to 5.6.0, 5.3.6, or 5.0.14 and are experiencing a panic in the gateway:
    +Restarting the gateway process will restore it to a healthy state. If you are operating in a *Kubernetes* environment, Tyk Gateway instance should automatically restart, which ultimately resolves the issue.
    • +
+ +

We appreciate your understanding and patience as we work to resolve this. Please stay tuned for the upcoming patch release, which will address this issue.

+ + + + +#### Release Highlights + +This release primarily focuses on bug fixes. For a comprehensive list of changes, please refer to the detailed +[changelog](#Changelog-v5.3.6) below. + +#### Breaking Changes + +Docker images are now based on [distroless](https://github.com/GoogleContainerTools/distroless). No shell is shipped in +the image. + +If moving from an version of Tyk older than 5.3.0 please read the explanation provided with [5.3.0 release](#TykOAS-v5.3.0). + +#### Deprecations + +There are no deprecations in this release. + +#### Upgrade Instructions + +When upgrading to 5.3.6 please follow the [detailed upgrade instructions](#upgrading-tyk). + +#### Dependencies + +{/* Required. Use this section to announce the following types of dependencies compatible with the release: +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +3rd party dependencies and tools */} + +##### Compatibility Matrix For Tyk Components + +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :--------------- | :------------------------------------------------------------------ | :----------------------- | +| 5.3.6 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v2.0.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------- | :--------------- | :------------------- | :------------------------------------------------------------------------------------------ | +| [Go](https://go.dev/dl/) | 1.22 | 1.22 | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.22 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the +ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.3.6) + - ```bash + docker pull tykio/tyk-gateway:v5.3.6 + ``` +- Helm charts + - [tyk-charts v2.0.0](/developer-support/release-notes/helm-chart#200-release-notes) +- [Source code tarball of Tyk Gateway v5.3.6](https://github.com/TykTechnologies/tyk/releases/tag/v5.3.6) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Changed + +{/* This should be a bullet-point list of updated features. Explain: + +- Why was the update necessary? +- How does the update benefit users? +- Link to documentation of the updated feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + +The Tyk Gateway has been upgraded from Golang 1.21 to Golang 1.22, bringing enhanced performance, strengthened security, +and access to the latest features available in the new Golang release. + +
    • + +
  • + +In this release, we've enhanced the security of the Tyk Gateway image by changing the build process to support +[distroless](https://github.com/GoogleContainerTools/distroless) containers. This significant update addresses critical +CVEs associated with Debian, ensuring a more secure and minimal runtime environment. Distroless containers reduce the +attack surface by eliminating unnecessary packages, which bolsters the security of your deployments. + +
    • + +
+ +##### Fixed + +
    +
  • + + +We have resolved an issue where custom [response plugins](/api-management/plugins/plugin-types#response-plugins) were not being +triggered for Tyk OAS APIs. This fix ensures that all custom plugins are invoked as expected when using Tyk OAS APIs. + + + +We have enhanced the initial synchronization of Data Plane gateways with the Control Plane to ensure more reliable +loading of policies and APIs on start-up. A synchronous initialization process has been implemented to avoid sync +failures and reduce the risk of service disruptions caused by failed loads. This update ensures smoother and more +consistent syncing of policies and APIs in distributed deployments. + + + +We have fixed an issue where the quota limit was not being consistently respected during request spikes, especially in +deployments with multiple gateways. The problem occurred when multiple gateways cached the current and remaining quota +counters at the end of quota periods. To address this, a distributed lock mechanism has been implemented, ensuring +coordinated quota resets and preventing discrepancies across gateways. + + + +We have addressed a performance regression identified in Tyk Gateway versions 4.0.13 and later, where key creation for +policies with a large number of APIs (100+) became significantly slower. The operation, which previously took around 1.5 +seconds in versions 4.0.0 to 4.0.12, was taking over 20 seconds in versions 4.0.13 and beyond. This issue has been +resolved by optimizing Redis operations during key creation, restoring the process to its expected speed of +approximately 1.5 seconds, even with a large number of APIs in the policy. + + +
    • +
+ +##### Security Fixes + +{/* This section should be a bullet point list that should be included when any security fixes have been made in the release, e.g. CVEs. For CVE fixes, consideration needs to be made as follows: +1. Dependency-tracked CVEs - External-tracked CVEs should be included on the release note. +2. Internal scanned CVEs - Refer to the relevant engineering and delivery policy. + +For agreed CVE security fixes, provide a link to the corresponding entry on the NIST website. For example: + +- Fixed the following CVEs: + - [CVE-2022-33082](https://nvd.nist.gov/vuln/detail/CVE-2022-33082) */} + +
    +
  • + +Fixed the following high priority CVEs identified in the Tyk Gateway, providing increased protection against security +vulnerabilities: + +- [CVE-2024-6104](https://nvd.nist.gov/vuln/detail/CVE-2024-6104) + +
    • +
+ +--- + +### 5.3.5 Release Notes + +#### Release Date 26 September 2024 + +#### Release Highlights + +This release fixes some issues related to the way that Tyk performs URL path matching, introducing two new Gateway +configuration options to control path matching strictness. For a comprehensive list of changes, please refer to the +detailed [changelog](#Changelog-v5.3.5) below. + +#### Breaking Changes + +There are no breaking changes in this release, however if moving from an version of Tyk older than 5.3.0 please read the +explanation provided with [5.3.0 release](#TykOAS-v5.3.0). + +#### Deprecations + +There are no deprecations in this release. + +#### Upgrade Instructions + +When upgrading to 5.3.5 please follow the [detailed upgrade instructions](#upgrading-tyk). + +#### Dependencies + +{/* Required. Use this section to announce the following types of dependencies compatible with the release: +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +3rd party dependencies and tools */} + +##### Compatibility Matrix For Tyk Components + +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :--------------- | :------------------------------------------------------------------ | :----------------------- | +| 5.3.5 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v2.0.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------- | :--------------------- | :--------------------- | :------------------------------------------------------------------------------------------ | +| [Go](https://go.dev/dl/) | 1.19 (GQL), 1.21 (GW) | 1.19 (GQL), 1.21 (GW) | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the +ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.3.5) + - ```bash + docker pull tykio/tyk-gateway:v5.3.5 + ``` +- Helm charts + - [tyk-charts v2.0.0](/developer-support/release-notes/helm-chart#200-release-notes) +- [Source code tarball of Tyk Gateway v5.3.5](https://github.com/TykTechnologies/tyk/releases/tag/v5.3.5) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Added + +
    +
  • + +We have introduced two new options in the `http_server_options` [Gateway +configuration](/tyk-oss-gateway/configuration#http_server_options) that will enforce prefix and/or suffix matching +when Tyk performs checks on whether middleware or other logic should be applied to a request: + +- `enable_path_prefix_matching` ensures that the start of the request path must match the path defined in the API + definition +- `enable_path_suffix_matching` ensures that the end of the request path must match the path defined in the API + definition +- combining `enable_path_prefix_matching` and `enable_path_suffix_matching` will ensure an exact (explicit) match is + performed + +These configuration options provide control to avoid unintended matching of paths from Tyk's default _wildcard_ match. +Use of regex special characters when declaring the endpoint path in the API definition will automatically override these +settings for that endpoint. Tyk recommends that exact matching is employed, but both options default to `false` to avoid +introducing a breaking change for existing users. + +The example Gateway configuration file `tyk.conf.example` has been updated to set the recommended exact matching with: + +- `http_server_options.enable_path_prefix_matching = true` +- `http_server_options.enable_path_suffix_matching = true` +- `http_server_options.enable_strict_routes = true` + +
    • +
+ +##### Fixed + +
    +
  • + +Fixed an issue when using granular [Path-Based +Permissions](/api-management/policies#secure-your-apis-by-method-and-path) in access policies and keys that led to authorization +incorrectly being granted to endpoints if an invalid regular expression was configured in the key/policy. Also fixed an issue +where path-based parameters were not correctly handled by Path-Based Permissions. Now Tyk's authorization check correctly +handles both of these scenarios granting access only to the expected resources. + +
    • +
  • + +Fixed an issue where a parameterized endpoint URL (e.g. `/user/{id}`) would be invoked if a request is made that omits +the parameter. For example, a request to `/user/` will now be interpreted as a request to `/user` and not to +`/user/{id}`. + +
    • +
+ +--- + +### 5.3.4 Release Notes + +#### Release Date August 26th 2024 + +#### Release Highlights + +Gateway 5.3.4 was version bumped only, to align with Dashboard 5.3.4. Subsequently, no changes were encountered in +release 5.3.4. For further information please see the release notes for Dashboard +[v5.3.4](/developer-support/release-notes/dashboard#530-release-notes) + +#### Breaking Changes + +**Attention**: Please read this section carefully. + +There are no breaking changes in this release, however if moving from an version of Tyk older than 5.3.0 please read the +explanation provided with [5.3.0 release](#TykOAS-v5.3.0). + +#### Deprecations + +There are no deprecations in this release. + +#### Upgrade Instructions + +When upgrading to 5.3.4 please follow the [detailed upgrade instructions](#upgrading-tyk). + +#### Dependencies + +{/* Required. Use this section to announce the following types of dependencies compatible with the release: +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +3rd party dependencies and tools */} + +##### Compatibility Matrix For Tyk Components + +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :--------------- | :------------------------------------------------------------------ | :----------------------- | +| 5.3.4 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v1.4.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------- | :--------------------- | :--------------------- | :------------------------------------------------------------------------------------------ | +| [Go](https://go.dev/dl/) | 1.19 (GQL), 1.21 (GW) | 1.19 (GQL), 1.21 (GW) | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the +ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.3.4) + - ```bash + docker pull tykio/tyk-gateway:v5.3.4 + ``` +- Helm charts + - [tyk-charts v1.4](/developer-support/release-notes/helm-chart#140-release-notes) +- [Source code tarball of Tyk Gateway v5.3.4](https://github.com/TykTechnologies/tyk/releases/tag/v5.3.4) + +

Changelog

+Since this release was version bumped only to align with Dashboard v5.3.4, no changes were encountered in this release. + +--- + +### 5.3.3 Release Notes + +#### Release Date August 2nd 2024 + +#### Breaking Changes + +**Attention**: Please read this section carefully. + +There are no breaking changes in this release, however if moving from an version of Tyk older than 5.3.0 please read the +explanation provided with [5.3.0 release](#TykOAS-v5.3.0). + +#### Deprecations + +There are no deprecations in this release. + +#### Upgrade Instructions + +When upgrading to 5.3.3 please follow the [detailed upgrade instructions](#upgrading-tyk). + +#### Release Highlights + +##### Bug Fixes + +This release primarily focuses on bug fixes. For a comprehensive list of changes, please refer to the detailed +[changelog](#Changelog-v5.3.3) below. + +##### FIPS Compliance + +Tyk Gateway now offers [FIPS 140-2](https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.140-2.pdf) compliance. For further +details please consult [Tyk API Management +FIPS support](/developer-support/release-types/fips-release). + +#### Dependencies + +{/* Required. Use this section to announce the following types of dependencies compatible with the release: +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +3rd party dependencies and tools */} + +##### Compatibility Matrix For Tyk Components + +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :--------------- | :------------------------------------------------------------------ | :----------------------- | +| 5.3.3 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v1.4.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------- | :--------------------- | :--------------------- | :------------------------------------------------------------------------------------------ | +| [Go](https://go.dev/dl/) | 1.19 (GQL), 1.21 (GW) | 1.19 (GQL), 1.21 (GW) | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the +ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.3.3) + - ```bash + docker pull tykio/tyk-gateway:v5.3.3 + ``` +- Helm charts + - [tyk-charts v1.4](/developer-support/release-notes/helm-chart#140-release-notes) +- [Source code tarball of Tyk Gateway v5.3.3](https://github.com/TykTechnologies/tyk/releases/tag/v5.3.3) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Added + +{/* This section should be a bullet point list of new features. Explain: + +- The purpose of the new feature +- How does the new feature benefit users? +- Link to documentation of the new feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + +Added [FIPS compliance](/developer-support/release-types/fips-release) for Tyk Gateway. + +
    • + +
  • + +Fixed an issue where nested API endpoints, such as '/test' and '/test/abc', might incorrectly apply middleware from the +parent path to the nested path. The fix ensures that API endpoint definitions are correctly ordered, preventing this +middleware misapplication and ensuring both the HTTP method and URL match accurately. + +
    • +
+ +--- + +##### Fixed + +{/* This section should be a bullet point list that describes the issues fixed in the release. For each fixed issue explain: +- What problem the issue caused +- How was the issue fixed +- Link to (new) documentation created as a result of a fix. For example, a new configuration parameter may have been introduced and documented for the fix +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +Addressed an issue where creating or resetting a key caused an exponential number of Redis DeleteRawKey commands. +Previously, the key creation sequence repeated for every API in the access list, leading to excessive deletion events, +especially problematic for access lists with over 100 entries. Now, the key creation sequence executes only once, and +redundant deletion of non-existent keys in Redis has been eliminated, significantly improving performance and stability +for larger access lists. + + + +Fixed a bug that caused Server Side Event (SSE) streaming responses to be considered for caching, which required +buffering the response and prevented SSE from being correctly proxied. + + + +Resolved an issue where Host and Latency fields (Total and Upstream) were not correctly reported for edge gateways in +MDCB setups. The fix ensures accurate Host values and Latency measurements are now captured and displayed in analytics +data. + + +
    • +
+ +--- + +### 5.3.2 Release Notes + +#### Release Date 5th June 2024 + +#### Breaking Changes + +**Attention**: Please read this section carefully. + +There are no breaking changes in this release, however if moving from an version of Tyk older than 5.3.0 please read the +explanation provided with [5.3.0 release](#TykOAS-v5.3.0). + +#### Deprecations + +There are no deprecations in this release. + +#### Upgrade Instructions + +When upgrading to 5.3.2 please follow the [detailed upgrade instructions](#upgrading-tyk). + +#### Release Highlights + +This release primarily focuses on bug fixes. For a comprehensive list of changes, please refer to the detailed +[changelog](#Changelog-v5.3.2) below. + +#### Dependencies + +{/* Required. Use this section to announce the following types of dependencies compatible with the release: +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +3rd party dependencies and tools */} + +##### Compatibility Matrix For Tyk Components + +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :--------------- | :------------------------------------------------------------------ | :----------------------- | +| 5.3.2 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v1.4.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------- | :--------------------- | :--------------------- | :------------------------------------------------------------------------------------------ | +| [Go](https://go.dev/dl/) | 1.19 (GQL), 1.21 (GW) | 1.19 (GQL), 1.21 (GW) | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the +ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.3.2) + - ```bash + docker pull tykio/tyk-gateway:v5.3.2 + ``` +- Helm charts + - [tyk-charts v1.4](/developer-support/release-notes/helm-chart#140-release-notes) +- [Source code tarball of Tyk Gateway v5.3.2](https://github.com/TykTechnologies/tyk/releases/tag/v5.3.2) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Fixed + +{/* This section should be a bullet point list that describes the issues fixed in the release. For each fixed issue explain: +- What problem the issue caused +- How was the issue fixed +- Link to (new) documentation created as a result of a fix. For example, a new configuration parameter may have been introduced and documented for the fix +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +In Gateway version 5.2+ and 5.3+, we discovered a bug within the OpenTelemetry tracing feature that inadvertently +transmits sensitive information. Specifically, `tyk.api.apikey` and `tyk.api.oauthid` attributes were exposing API keys. +We have fixed the issue to ensure that only the hashed version of the API key is transmitted in traces. + + + +Addressed an issue where an API with a custom domain might not be invoked if another API with the same listen path but +no custom domain was also deployed on the Gateway. Now APIs with custom domain names are loaded first, so requests will +be checked against these first before falling back to APIs without custom domains. + + + +Addressed an issue in service discovery where an IP:port returned by Consul wasn't parsed correctly on the Gateway side, +leading to errors when proxying requests to the service. The issue primarily occurred with IP:port responses, while +valid domain names were unaffected. + + + +Fixed an issue with nested field mapping in UDG when used with GraphQL (GQL) operations for a field's data source. +Previously, querying only the mentioned field resulted in an error, but querying alongside another 'normal' field from +the same level worked without issue. + + + +Addressed a potential issue when working with Tyk OAS APIs where request context variables are automatically made +available to relevant Tyk and custom middleware. We have introduced a control in the Tyk OAS API definition to disable +this access if required. + + +
    • +
+ +--- + +### 5.3.1 Release Notes + +#### Release Date 24 April 2024 + +#### Breaking Changes + +**Attention**: Please read this section carefully. + +There are no breaking changes in this release, however if moving from an version of Tyk older than 5.3.0 please read the +explanation provided with [5.3.0 release](#TykOAS-v5.3.0). + +#### Deprecations + +There are no deprecations in this release. + +#### Upgrade Instructions + +When upgrading to 5.3.1 please follow the [detailed upgrade instructions](#upgrading-tyk). + +#### Release Highlights + +This release primarily focuses on bug fixes. For a comprehensive list of changes, please refer to the detailed +[changelog](#Changelog-v5.3.1) below. + +#### Dependencies + +{/* Required. Use this section to announce the following types of dependencies compatible with the release: + +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. + +3rd party dependencies and tools */} + +##### Compatibility Matrix For Tyk Components + +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :--------------- | :------------------------------------------------------------------ | :----------------------- | +| 5.3.1 | MDCB v2.5.1 | MDCB v2.5.1 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v1.3.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------- | :--------------------- | :--------------------- | :------------------------------------------------------------------------------------------ | +| [Go](https://go.dev/dl/) | 1.19 (GQL), 1.21 (GW) | 1.19 (GQL), 1.21 (GW) | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the +ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.3.1) + - ```bash + docker pull tykio/tyk-gateway:v5.3.1 + ``` +- Helm charts + - [tyk-charts v1.3](/developer-support/release-notes/helm-chart#130-release-notes) +- [Source code tarball of Tyk Gateway v5.3.1](https://github.com/TykTechnologies/tyk/releases/tag/v5.3.1) + +

Changelog

+##### Fixed + +
    +
  • + + +Issues were addressed where Tyk failed to properly reject custom plugin bundles with signature verification failures, +allowing APIs to load without necessary plugins, potentially exposing upstream services. With the fix, if the plugin +bundle fails to load (for example, due to failed signature verification) the API will not be loaded and an error will be +logged in the Gateway. + + + +Fixed a panic scenario that occurred when a custom JavaScript plugin that requests access to the session metadata +(`require_session:true`) is assigned to the same endpoint as the Ignore Authentication middleware. While the custom +plugin expects access to a valid session, the configuration flag doesn't guarantee its presence, only that it's passed +if available. As such, the custom plugin should be coded to verify that the session metadata is present before +attempting to use it. + + + +Fixed a bug where the Gateway could crash when using custom Python plugins that access the Redis storage. The Tyk Python +API methods `store_data` and `get_data` could fail due to connection issues with the Redis. With this fix, the Redis +connection will be created if required, avoiding the crash. + + + +In some instances users were noticing gateway panics when using the **Persist GQL** middleware without arguments +defined. This issue has been fixed and the gateway will not throw panics in these cases anymore. + + + +In cases where `detailed_tracing` was set to `false` and the client was sending a malformed request to a GraphQL API, +the traces were missing GraphQL attributes (operation name, type and document). This has been corrected and debugging +GraphQL with OTel will be easier for users. + + + +GQL Open Telemetry semantic conventions attribute names were missing `graphql` prefix and therefore were not in line +with the community standard. This has been fixed and all attributes have the correct prefix. + + + +Fixed two bugs in the handling of usage quotas by the URL rewrite middleware when it was configured to rewrite to itself +(e.g. to `tyk://self`). Quota limits were not observed and the quota related response headers always contained `0`. + + + +Resolved an issue in distributed deployments where the MDCB data plane gateway counter was inaccurately incremented when +a Gateway was stopped and restarted. + + + +Addressed a bug where clearing the API cache from the Tyk Dashboard failed to invalidate the cache in distributed data +plane gateways. This fix requires MDCB 2.5.1. + + + +Fixed a bug where custom Go plugins compiled in RHEL8 environments were unable to load into Tyk Gateway due to a +discrepancy in base images between the Gateway and Plugin Compiler environments. This fix aligns the plugin compiler +base image with the gateway build environment, enabling seamless plugin functionality on RHEL8 environments. + + + +Removed several unused packages from the plugin compiler image. The packages include: docker, buildkit, ruc, sqlite, curl, wget, and other build tooling. The removal was done in order to address invalid CVE reporting, none of the removed dependencies are used to provide plugin compiler functionality. + + +
    • +
+ +--- + +### 5.3.0 Release Notes + +#### Release Date 5 April 2024 + +#### Breaking Changes + +{/* Required. Use the following statement if there are no breaking changes, or explain if there are */} + +**Attention: Please read this section carefully** + +
Tyk OAS APIs Compatibility Caveats - Tyk OSS
+This upgrade transitions Tyk OAS APIs out of [Early Access](/developer-support/release-types/early-access-feature). + +For licensed deployments (Tyk Cloud, Self Managed including MDCB), please refer to the [release notes of Tyk Dashboard 5.3.0](/developer-support/release-notes/dashboard#530-release-notes). + +- **Out of Early Access** + - This means that from now on, all Tyk OAS APIs will be backwards compatible and in case of a downgrade from v5.3.X to + v5.3.0, the Tyk OAS API definitions will always work. +- **Not Backwards Compatible** + - Tyk OAS APIs in Tyk Gateway v5.3.0 are not [backwards compatible](https://tinyurl.com/3xy966xn). This means that the + new Tyk OAS API format created by Tyk Gateway v5.3.X does not work with older versions of Tyk Gateway, i.e. you + cannot export these API definitions from a v5.3.X Tyk Gateway and import them to an earlier version. + - The upgrade is **not reversible**, i.e. you cannot use version 5.3.X Tyk OAS API definitions with an older version + of Tyk Dashboard. + - This means that if you wish to downgrade or revert to your previous version of Tyk, you will need to restore these + API definitions from a backup. Please go to the [backup](#upgrade-instructions) section for detailed + instructions on backup before upgrading to v5.3.0. + - If you are not using Tyk OAS APIs, Tyk will maintain backward compatibility standards. +- **Not Forward Compatible** + - Tyk OAS API Definitions prior to v5.3.0 are not [forward compatible](https://tinyurl.com/t3zz88ep) with Tyk Gateway + v5.3.X. + - This means that any Tyk OAS APIs created in any previous release (4.1.0-5.2.x) cannot work with the new Tyk Gateway + v5.3.X without being migrated to its latest format. +- **After upgrade (the good news)** + - Tyk OAS API definitions that are part of the file system **are not automatically converted** to the new + format. Subsequently, users will have to manually update their + OAS API Definitions to the new format. + - If users upgrade to 5.3.0, create new Tyk OAS APIs and then decide to rollback then the upgrade is non-reversible. + Reverting to your previous version requires restoring from a backup. + +**Important:** Please go to the [backup](#upgrade-instructions) section for detailed instructions on +backup before upgrading to v5.3.0 + +##### Python plugin support + +Starting from Tyk Gateway version v5.3.0, Python is no longer bundled with the official Tyk Gateway Docker image to +reduce exposure to security vulnerabilities in the Python libraries. + +Whilst the Gateway still supports Python plugins, you must [extend +the image](/api-management/plugins/rich-plugins#install-the-python-development-packages) +to add the language support. + +{/* The following "Changed error log messages" section is Optional! +Instructions: We should mention in the changelog section ALL changes in our application log messages. In case we made such changes, this section should also be added, to make sure the users don't miss this notice among other changelog lines. */} +{/* ##### Changed error log messages +Important for users who monitor Tyk components using the application logs (i.e. Tyk Gateway log, Tyk Dashboard log etc.). +We try to avoid making changes to our log messages, especially at error and critical levels. However, sometimes it's necessary. Please find the list of changes made to the application log in this release: */} + +{/* The following "|Planned Breaking Changes" section is optional! +Announce future scheduled breaking changes, e.g. Go version updates, DB driver updates etc. */} +{/* ##### Planned Breaking Changes */} + +

Dependencies

+{/* Required. Use this section to announce the following types of dependencies compatible with the release: + +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. + +3rd party dependencies and tools */} + +##### Compatibility Matrix For Tyk Components + +{/* Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. */} + +| Gateway Version | Recommended Releases | Backwards Compatibility | +| :--------------- | :------------------------------------------------------------------ | :----------------------- | +| 5.3.0 | MDCB v2.5 | MDCB v2.4.2 | +| | Operator v0.17 | Operator v0.16 | +| | Sync v1.4.3 | Sync v1.4.3 | +| | Helm Chart (tyk-stack, tyk-oss, tyk-dashboard, tyk-gateway) v1.3.0 | Helm all versions | +| | EDP v1.8.3 | EDP all versions | +| | Pump v1.9.0 | Pump all versions | +| | TIB (if using standalone) v1.5.1 | TIB all versions | + +##### 3rd Party Dependencies & Tools + +{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :------------------------------------------------------------- | :--------------------- | :--------------------- | :------------------------------------------------------------------------------------------ | +| [Go](https://go.dev/dl/) | 1.19 (GQL), 1.21 (GW) | 1.19 (GQL), 1.21 (GW) | [Go plugins](/api-management/plugins/golang#) must be built using Go 1.21 | +| [Redis](https://redis.io/download/) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway | +| [OpenAPI Specification](https://spec.openapis.org/oas/v3.0.3) | v3.0.x | v3.0.x | Supported by [Tyk OAS](/api-management/gateway-config-tyk-oas) | + +Given the potential time difference between your upgrade and the release of this version, we recommend users verify the +ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations + +{/* Required. Use the following statement if there are no deprecations, or explain if there are */} + +In 5.3.0, we have simplified the configuration of response transform middleware. We encourage users to embrace the +`global_headers` mechanism as the `response_processors.header_injector` is now an optional setting and will be removed +in a future release. + +{/* Optional section! +Used to share and notify users about our plan to deprecate features, configs etc. +Once you put an item in this section, we must keep this item listed in all the following releases till the deprecation happens. */} +{/* ###### Future deprecations */} + +

Upgrade instructions

+If you are upgrading to 5.3.0, please follow the detailed [upgrade instructions](#upgrading-tyk). + +**The following steps are essential to follow before upgrading** Tyk Cloud (including Hybrid Gateways) and Self Managed +users - Please refer to the [release notes of Tyk Dashboard 5.3.0](/developer-support/release-notes/dashboard#530-release-notes). + +For OSS deployments - + +1. Backup Your environment using the [usual guidance](/developer-support/upgrading) documented with every release (this includes + backup config file and database). +2. Backup all your API definitions (Tyk OAS API and Classic Definitions) by saving your API and policy files or by + exporting them using the `GET /tyk/apis` and `Get /tyk/policies` +3. Performing the upgrade - follow the instructions in the [upgrade + guide](/developer-support/upgrading) when upgrading Tyk. + +#### Release Highlights + +{/* Required. Use similar ToV to previous release notes. For example for a patch release: +This release primarily focuses on bug fixes. +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-vX.Y.0) below. */} + +We’re thrilled to announce the release of 5.3.0, an update packed with exciting features and significant fixes to +elevate your experience with Tyk Gateway. For a comprehensive list of changes, please refer to the detailed +[changelog](#Changelog-v5.3.0) below. + +##### Tyk OAS Feature Maturity + +Tyk OAS is now out of [Early +Access](/developer-support/release-types/early-access-feature) as we have reached feature maturity. +You are now able to make use of the majority of Tyk Gateway's features from your Tyk OAS APIs, so they are a credible alternative +to the legacy Tyk Classic APIs. + +From Tyk 5.3.0 we support the following features when using Tyk OAS APIs with Tyk Gateway: + +- Security + + - All Tyk-supported client-gateway authentication methods including custom auth plugins + - Automatic configuration of authentication from the OpenAPI description + - Gateway-upstream mTLS + - CORS + +- API-level (global) middleware including: + + - Response caching + - Custom plugins for PreAuth, Auth, PostAuth, Post and Response hooks + - API-level rate limits + - Request transformation - headers + - Response transformation - headers + - Service discovery + - Internal API + +- Endpoint-level (per-path) middleware including: + + - Request validation - headers and body (automatically configurable from the OpenAPI description) + - Request transformation - method, headers and body + - Response transformation - headers and body + - URL rewrite and internal endpoints + - Mock responses (automatically configurable from the OpenAPI description) + - Response caching + - Custom Go Post-Plugin + - Request size limit + - Virtual endpoint + - Allow and block listing + - Do-not-track + - Circuit breakers + - Enforced timeouts + - Ignore authentication + +- Observability + + - Open Telemetry tracing + - Detailed log recording (include payload in the logs) + - Do-not-track endpoint + +- Governance + - API Versioning + +##### Enhanced KV storage of API Definition Fields + +Tyk is able to store configuration data from the API definition in KV systems, such as Vault and Consul, and then +reference these values during configuration of the Tyk Gateway or APIs deployed on the Gateway. Previously this was +limited to the Target URL and Listen Path but from 5.3.0 you are able to store any `string` type field from your API +definition, unlocking the ability to store sensitive information in a centralised location. For full details check out +the [documentation](/tyk-configuration-reference/kv-store) of this powerful feature. + +##### Redis v7.x Compatibility + +We have upgraded Redis driver [go-redis](https://github.com/redis/go-redis) to v9. Subsequently, Tyk 5.3 is compatible +with Redis v7.x. + +##### Gateway and Component Upgrades + +We've raised the bar with significant upgrades to our Gateway and components. Leveraging the power and security of Go 1.21, upgrading [Sarama](https://github.com/Shopify/sarama), a widly used Kafka client in Go, to version 1.41.0 and enhancing the GQL engine with Go version 1.19, we ensure improved +functionality and performance to support your evolving needs seamlessly. + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=&page_size=&ordering=&name=v5.3.0) + - ```bash + docker pull tykio/tyk-gateway:v5.3.0 + ``` +- Helm charts + - [tyk-charts v1.3](/developer-support/release-notes/helm-chart#130-release-notes) +- [Source code tarball of Tyk Gateway v5.3.0](https://github.com/TykTechnologies/tyk/releases/tag/v5.3.0) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. + +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Added + +{/* This section should be a bullet point list of new features. Explain: + +- The purpose of the new feature +- How does the new feature benefit users? +- Link to documentation of the new feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +The following features have been added in 5.3.0 to bring Tyk OAS to feature maturity: + +- Detailed log recording (include payload in the logs) +- Enable Open Telemetry tracing +- Context variables available to middleware chain +- API-level header transforms (request and response) +- Endpoint-level cache +- Circuit breakers +- Track endpoint logs for inclusion in Dashboard aggregated data +- Do-not-track endpoint +- Enforced upstream timeouts +- Configure endpoint as Internal, not available externally +- URL rewrite +- Per-endpoint request size limit +- Request transformation - method, header +- Response transformation - header +- Custom domain certificates + + + +We have implemented support for all `string` type fields in the Tyk OAS and Tyk Classic API Definitions to be stored in +separate KV storage, including Hashicorp Consul and Vault. + + + +Tyk 5.3 refactors Redis connection logic by using +[storage v1.2.2](https://github.com/TykTechnologies/storage/releases/tag/v1.2.2), which integrates with +[go-redis](https://github.com/redis/go-redis) v9. Subsequently, Tyk 5.3 supports Redis v7.0.x. + + + +Some of the error messages generated by the GQL engine were unclear for users, especially relating to variable +validation. The errors have been changed and are now much more clearer and helpful in cases where engine processing +fails. + + + +Upgraded Go version for GraphQL engine to [1.19](https://go.dev/doc/go1.19). + + + +We've added OpenTelemetry semantic conventions for GraphQL spans. Spans will now incorporate ``, +`` and `` tags. + + + +GraphQL APIs can now use the `detailed_tracing` setting in an API definition. With that property set to `true` any call +to a GraphQL API will create a span for each middleware involved in request processing. While it is set to `false`, only +two spans encapsulating the entire request lifecycle will be generated. This setting helps to reduce the size of traces, +which can get large for GraphQL APIs. Furthermore, this gives users an option to customize the level of tracing detail +to suit their monitoring needs. + + + +This release introduces an enhanced trace generation system for Universal Data Graph (UDG). It consolidates all spans +from both Tyk-managed and external data source executions into a single trace when used together. Furthermore, when UDG +solely utilizes Tyk-managed data sources, trace management is simplified and operational visibility is improved. + + + +For GraphQL requests normalization and validation has been disabled in the GraphQL engine. Both of those actions were +performed in the Tyk Gateway and were unnecessary to be done again in the engine. This enhances performance slightly and +makes detailed OTel traces concise and easier to read. + + + +The Tyk Dashboard API endpoint _/api/data-graphs/data-sources/import_ now handles OpenAPI schemas with arrays of +objects. This addition means users can now import more complex OpenAPI documents and transform them into UDG +configurations. + + + +The OAS-to-UDG converter now seamlessly handles OpenAPI descriptions that utilize the _allOf_, _anyOf_ and _oneOf_ +keywords, ensuring accurate and comprehensive conversion to a Tyk API definition. The feature expands the scope of +OpenAPI documents that the converter can handle and allows our users to import REST API data sources defined in OAS in +more complex cases. + + + +The OAS-to-UDG converter can now create GraphQL types even if an object's definition doesn’t have an explicit name. + + + +The OAS-to-UDG converter was unable to handle a document properly if an object within the OpenAPI description had no +properties defined. This limitation resulted in unexpected behavior and errors during the conversion process. The tool +will now handle such cases seamlessly, ensuring a smoother and more predictable conversion process. + + + +Previously OAS-to-UDG converter had limitations in handling enums from OpenAPI descriptions, leading to discrepancies +and incomplete conversions. With the inclusion of enum support, the OAS converter now seamlessly processes enums defined +in your OpenAPI descriptions, ensuring accurate and complete conversion to GraphQL schemas. + + + +OAS-to-UDG converter can now handle HTTP status code ranges that are defined by the OpenAPI Specification. This means +that code ranges defined as 1XX, 2XX, etc will be correctly converted by the tool. + + + +We have added the capability for users to define a [custom rate limit +key](/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/configuring-custom-rate-limit-keys) +within session metadata. This increases flexibility with rate limiting, as the rate limit can be assigned to different entities +identifiable from the session metadata (such as a client app or organization) and is particularly useful for users of Tyk's +Enterprise Developer Portal. + + +
    • +
+ +##### Changed + +{/* This should be a bullet-point list of updated features. Explain: + +- Why was the update necessary? +- How does the update benefit users? +- Link to documentation of the updated feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} +
    +
  • + + +Previously, when operating in a worker configuration (in the data plane), the Tyk Gateway fetched session expiry +information from the control plane the first time an API was accessed for a given organization. This approach led to a +significant issue: if the MDCB connection was lost, the next attempt to consume the API would incur a long response +time. This delay, typically around 30 seconds, was caused by the Gateway waiting for the session-fetching operation to +time out, as it tried to communicate with the now-inaccessible control plane. + +
    Now, the worker gateway fetches the session expiry information up front, while there is an active connection to +MDCB. This ensures that this data is already available locally in the event of an MDCB disconnection. + +
    This change significantly improves the API response time under MDCB disconnection scenarios by removing the need for +the Gateway to wait for a timeout when attempting to fetch session information from the control plane, avoiding the +previous 30-second delay. This optimization enhances the resilience and efficiency of Tyk Gateway in distributed +environments. +     + + +We have made some changes to the Tyk OAS API Definition to provide a stable contract that will now be under +breaking-change control for future patches and releases as Tyk OAS moves out of Early Access. Changes include the +removal of the unnecessary `slug` field and simplification of the custom plugin contract. + + + +We have optimized the allocation behavior of our sliding window log rate limiter implementation ([Redis +Rate Limiter](/api-management/rate-limit#redis-rate-limiter)). Previously the complete +request log would be retrieved from Redis. With this enhancement only the count of the requests in the window is +retrieved, optimizing the interaction with Redis and decreasing the Gateway memory usage. + +     +
    • +
+ +##### Fixed +{/* This section should be a bullet point list that describes the issues fixed in the release. For each fixed issue explain: + +- What problem the issue caused +- How was the issue fixed +- Link to (new) documentation created as a result of a fix. For example, a new configuration parameter may have been + introduced and documented for the fix +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to +expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example +below. */} + +
    +
  • + + +In this release, we fixed automated token trimming in Redis, ensuring efficient management of OAuth tokens by +implementing a new hourly job within the Gateway and providing a manual trigger endpoint. + + + +We fixed a bug in the Tyk OAS Validate Request middleware where we were not correctly validating date-time format +schema, which could lead to invalid date-time values reaching the upstream services. + + + +Fixed an issue when using the Distributed Rate Limiter (DRL) where the Gateway did not apply any rate limit until a DRL +notification was received. Now the rate of requests will be limited at 100% of the configured rate limit until the DRL +notification is received, after which the limit will be reduced to an even share of the total (i.e. 100% divided by the +number of Gateways) per the rate limit algorithm design. + + + +Fixed an issue where the OAS-to-UDG converter was sometimes adding the same field to an object type many times. This +caused issues with the resulting GQL schema and made it non-compliant with GQL specification. + + + +Fixed an issue where the Gateway attempted to execute a query with GQL engine version 1 (which lacks OTel support), +while simultaneously trying to validate the same query with the OpenTelemetry (OTel) supported engine. It caused the API +to fail with an error message "Error socket hang up". Right now with OTel enabled, the gateway will enforce GQL engine +to default to version 2, so that this problem doesn't occur anymore. + + + +The OAS-to-UDG converter now effectively handles array of objects within POST paths. Previously, there were instances +where the converter failed to accurately interpret and represent these structures in the generated UDG configuration. + + + +An issue was identified where the encoding from the GQL upstream cache was causing readability problems in the response body. Specifically, the upstream GQL cache was utilizing [brotli compression](https://www.ietf.org/rfc/rfc7932.txt) and not respecting the Accept-Encoding header. Consequently, larger response bodies became increasingly unreadable for the GQL engine due to compression, leading to usability issues for users accessing affected content. The issue has now been fixed by adding the brotli encoder to the GQL engine. + + + +OAS-to-UDG converter was unable to correctly process Tyk OAS API definitions where "JSON" was used as one of enum +values. This issue is now fixed and whenever "JSON" is used as one of enums in the OpenAPI description, it will get +correctly transformed into a custom scalar in GQL schema. + + + +Fixed an issue where the Gateway could panic while updating a Tyk OAS API with the Virtual Endpoint middleware +configured. + + + +Fixed an issue where reloading a bundle containing JS plugins could cause the Gateway to panic. + + + +Fixed an issue where the _Disable introspection_ setting was not working correctly in cases where field-based +permissions were set (allow or block list). It was not possible to introspect the GQL schema while introspection was +technically allowed but field-based permissions were enabled. Currently, Allow/Block list settings are ignored only for +introspection queries and introspection is only controlled by the _Disable introspection_ setting. + + + +The OAS-to-UDG converter was unable to handle a document properly if an object within the OpenAPI description had no +properties defined. This limitation resulted in unexpected behavior and errors during the conversion process. The tool +will now handle such cases seamlessly, ensuring a smoother and more predictable conversion process + + + +Addressed a memory leak issue in Tyk Gateway linked to a logger mutex change introduced in v5.2.4. Reverting these +changes has improved connection management and enhanced system performance. + + + +Resolved an issue where in certain conditions external clients could access internal endpoints. This was caused by incorrect combination of middleware which could lead to internal endpoints proxying traffic from external sources. This has now been addressed, so that an endpoint with the internal middleware configured will not be reachable from external requests. + + +
    • + +
+ +##### Security Fixes + +{/* This section should be a bullet point list that should be included when any security fixes have been made in the release, e.g. CVEs. For CVE fixes, consideration needs to be made as follows: +1. Dependency-tracked CVEs - External-tracked CVEs should be included on the release note. +2. Internal scanned CVEs - Refer to the relevant engineering and delivery policy. + +For agreed CVE security fixes, provide a link to the corresponding entry on the NIST website. For example: + +- Fixed the following CVEs: + - [CVE-2022-33082](https://nvd.nist.gov/vuln/detail/CVE-2022-33082) */} + +
    +
  • + +Fixed the following high priority CVEs identified in the Tyk Gateway, providing increased protection against security +vulnerabilities: + +- [CVE-2023-39325](https://nvd.nist.gov/vuln/detail/CVE-2023-39325) +- [CVE-2023-45283](https://nvd.nist.gov/vuln/detail/CVE-2023-45283) + +
    • +
+ +{/* Required. use 3 hyphens --- between release notes of every patch (minors will be on a separate page) */} + +{/* Repeat the release notes section above for every patch here */} + +{/* The footer of the release notes page. It contains a further information section with details of how to upgrade Tyk, +links to API documentation and FAQs. You can copy it from the previous release. */} + +## 5.2 Release Notes + +### 5.2.5 Release Notes + +#### Release Date 19 Dec 2023 + +#### Breaking Changes + +**Attention**: Please read carefully this section. We have two topics to report: + +#### Early Access Features: +Please note that the `Tyk OAS APIs` feature, currently marked as *Early Access*, is subject to breaking changes in subsequent releases. Please refer to our [Early Access guide](/developer-support/release-types/early-access-feature) for specific details. Upgrading to a new version may introduce changes that are not backwards-compatible. Downgrading or reverting an upgrade may not be possible resulting in a broken installation. + +Users are strongly advised to follow the recommended upgrade instructions provided by Tyk before applying any updates. + +#### Deprecations +There are no deprecations in this release. + +#### Upgrade Instructions +If you are using a 5.2.x version, we advise you to upgrade ASAP to this latest release. If you are on an older version, you should skip 5.2.0 and upgrade directly to this release. Go to the [Upgrading Tyk](#upgrading-tyk) section for detailed upgrade instructions. + +#### Release Highlights +This release implements a bug fix. +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.2.5) below. + +#### Downloads +- [Docker image to pull](https://hub.docker.com/layers/tykio/tyk-gateway/v5.2.5/images/sha256-c09cb03dd491e18bb84a0d9d4e71177eb1396cd5debef694f1c86962dbee10c6?context=explore) +- [source code](https://github.com/TykTechnologies/tyk/releases/tag/v5.2.5) + +

Changelog

+##### Fixed +
    +
  • + +Fixed an issue where custom keys over 24 characters in length were deleted from Redis in the Data Plane when key update action signalled in distributed (MDCB) setups. + +
    • +
+ +--- + +### 5.2.4 Release Notes + +#### Release Date 7 Dec 2023 + +#### Breaking Changes +**Attention**: Please read carefully this section. We have two topics to report: + +#### Early Access Features: +Please note that the `Tyk OAS APIs` feature, currently marked as *Early Access*, is subject to breaking changes in subsequent releases. Please refer to our [Early Access guide](/developer-support/release-types/early-access-feature) for specific details. Upgrading to a new version may introduce changes that are not backwards-compatible. Downgrading or reverting an upgrade may not be possible resulting in a broken installation. + +Users are strongly advised to follow the recommended upgrade instructions provided by Tyk before applying any updates. + +#### Deprecations +There are no deprecations in this release. + +#### Upgrade Instructions +If you are using a 5.2.x version, we advise you to upgrade ASAP to this latest release. If you are on an older version, you should skip 5.2.0 and upgrade directly to this release. Go to the [Upgrading Tyk](#upgrading-tyk) section for detailed upgrade instructions. + +#### Release Highlights +This release enhances security, stability, and performance. +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.2.4) below. + +#### Downloads +- [Docker image to pull](https://hub.docker.com/layers/tykio/tyk-gateway/v5.2.4/images/sha256-c0d9e91e4397bd09c85adf4df6bc401b530ed90c8774714bdafc55db395c9aa5?context=explore) +- [source code](https://github.com/TykTechnologies/tyk/releases/tag/v5.2.4) + +

Changelog

+##### Fixed +
    +
  • + + +Fixed an issue where the Validate Request middleware provided too much information when reporting a schema validation failure in a request to a Tyk OAS API. + + + +Fixed a bug where the gateway didn't correctly apply Path-Based Permissions from different policies when using the same `sub` claim but different scopes in each policy. Now the session will be correctly configured for the claims provided in the policy used for each API request. + + + +Fixed a bug when using the build_id argument with the Tyk Plugin Compiler that prevents users from hot-reloading different versions of the same plugin compiled with different build_ids. The bug was introduced with the plugin module build change implemented in the upgrade to Go version 1.19 in Tyk 5.1.0. + + + +Fixed a bug that was introduced in the fix applied to the URL Rewrite middleware in Tyk 5.0.5/5.1.2. The previous fix did not correctly handle escaped characters in the query parameters. Now you can safely include escaped characters in your query parameters and Tyk will not modify them in the URL Rewrite middleware. + + +
    • +
+ +--- + +### 5.2.3 Release Notes + +#### Release Date 21 Nov 2023 + +#### Breaking Changes +**Attention**: Please read carefully this section. We have two topics to report: + +#### Early Access Features: +Please note that the `Tyk OAS APIs` feature, currently marked as *Early Access*, is subject to breaking changes in subsequent releases. Please refer to our [Early Access guide](/developer-support/release-types/early-access-feature) for specific details. Upgrading to a new version may introduce changes that are not backwards-compatible. Downgrading or reverting an upgrade may not be possible resulting in a broken installation. + +Users are strongly advised to follow the recommended upgrade instructions provided by Tyk before applying any updates. + +#### Deprecations +There are no deprecations in this release. + +#### Upgrade Instructions +If you are using a 5.2.x version, we advise you to upgrade ASAP to this latest release. If you are on an older version, you should skip 5.2.0 and upgrade directly to this release. Go to the [Upgrading Tyk](#upgrading-tyk) section for detailed upgrade instructions. + +#### Release Highlights +This release enhances security, stability, and performance. +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.2.3) below. + +#### Downloads +- [Docker image to pull](https://hub.docker.com/layers/tykio/tyk-gateway/v5.2.3/images/sha256-8a94658c8c52ddfe30f78c5438dd4308c4d019655d8af7773a33fdffda097992?context=explore) +- [source code](https://github.com/TykTechnologies/tyk/releases/tag/v5.2.3) + +

Changelog

+##### Fixed + +
    +
  • + + +Fixed an issue where Tyk was not auto-detecting the installed Python version if it had multiple digits in the minor version (e.g. Python 3.11). The regular expression was updated to correctly identify Python versions 3.x and 3.xx, improving compatibility and functionality. + + + +Improved the behavior when using JWTs and the MDCB (Multi Data Center Bridge) link is down; the Gateway will no longer be blocked attempting to fetch OAuth client info. We’ve also enhanced the error messages to specify which type of resource (API key, certificate, OAuth client) the data plane Gateway failed to retrieve due to a lost connection with the control plane. + + + +Fixed an issue where the session object generated when creating a Custom Key in a Go Plugin did not inherit parameters correctly from the Security Policy. + + + +Fixed an issue where uploading a public key instead of a certificate into the certificate store, and using that key for mTLS, caused all the Gateways that the APIs are published on to cease negotiating TLS. This fix improves the stability of the gateways and the successful negotiation of TLS. + + +
    • +
+ +##### Added + +
    +
  • + + +This prints the release version, git commit, Go version used, architecture and other build details. + + + +Added new option for Tyk to use the default version of an API if the requested version does not exist. This is referred to as falling back to default and is enabled using a [configuration](/api-management/gateway-config-tyk-oas#versioning) flag in the API definition; for Tyk OAS APIs the flag is `fallbackToDefault`, for Tyk Classic APIs it is `fallback_to_default`. + + + +Added a backoff limit for GraphQL subscription connection retry to prevent excessive error messages when the upstream stops working. The connection retries and linked error messages now occur in progressively longer intervals, improving error handling and user experience. + + +
    • +
+ +##### Community Contributions + +Special thanks to the following member of the Tyk community for their contribution to this release: + +
    +
  • + +Fixed a minor issue with Go Plugin virtual endpoints where a runtime log error was produced from a request, even if the response was successful. Thanks to [uddmorningsun](https://github.com/uddmorningsun) for highlighting the [issue](https://github.com/TykTechnologies/tyk/issues/4197) and proposing a fix. + +
    • +
+ +--- + +### 5.2.2 Release Notes + +#### Release Date 31 Oct 2023 + +#### Breaking Changes +**Attention**: Please read carefully this section. We have two topics to report: + +#### Early Access Features: +Please note that the `Tyk OAS APIs` feature, currently marked as *Early Access*, is subject to breaking changes in subsequent releases. Please refer to our [Early Access guide](/developer-support/release-types/early-access-feature) for specific details. Upgrading to a new version may introduce changes that are not backwards-compatible. Downgrading or reverting an upgrade may not be possible resulting in a broken installation. + +Users are strongly advised to follow the recommended upgrade instructions provided by Tyk before applying any updates. + +#### Deprecations +There are no deprecations in this release. + +#### Upgrade Instructions +If you are using a 5.2.x version, we advise you to upgrade ASAP to this latest release. If you are on an older version, you should skip 5.2.0 and upgrade directly to this release. Go to the [Upgrading Tyk](#upgrading-tyk) section for detailed upgrade instructions. + +#### Release Highlights +This release primarily focuses on bug fixes. +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.2.2) below. + +#### Downloads +- [Docker image to pull](https://hub.docker.com/layers/tykio/tyk-gateway/v5.2.2/images/sha256-84d9e083872c78d854d3b469734ce40b7e77b9963297fe7945e214a0e6ccc614?context=explore) +- [source code](https://github.com/TykTechnologies/tyk/releases/tag/v5.2.2) + +

Changelog

+##### Security + +The following CVEs have been resolved in this release: + +- [CVE-2022-40897](https://nvd.nist.gov/vuln/detail/CVE-2022-40897) +- [CVE-2022-1941](https://nvd.nist.gov/vuln/detail/CVE-2022-1941) +- [CVE-2021-23409](https://nvd.nist.gov/vuln/detail/CVE-2021-23409) +- [CVE-2021-23351](https://nvd.nist.gov/vuln/detail/CVE-2021-23351) +- [CVE-2019-19794](https://nvd.nist.gov/vuln/detail/CVE-2019-19794) +- [CVE-2018-5709](https://nvd.nist.gov/vuln/detail/CVE-2018-5709) +- [CVE-2010-0928](https://nvd.nist.gov/vuln/detail/CVE-2010-0928) +- [CVE-2007-6755](https://nvd.nist.gov/vuln/detail/CVE-2007-6755) + + + +##### Fixed + +
    +
  • + + +Fixed an issue where [enforced timeouts](/planning-for-production/ensure-high-availability/enforced-timeouts) values were incorrect on a per-request basis. Since we enforced timeouts only at the transport level and created the transport only once within the value set by [max_conn_time](/tyk-oss-gateway/configuration#max_conn_time), the timeout in effect was not deterministic. Timeouts larger than 0 seconds are now enforced for each request. + + + +Fixed an issue when using MongoDB and [Tyk Security Policies](/api-management/policies#what-is-a-security-policy) where Tyk could incorrectly grant access to an API after that API had been deleted from the associated policy. This was due to the policy cleaning operation that is triggered when an API is deleted from a policy in a MongoDB installation. With this fix, the policy cleaning operation will not remove the final (deleted) API from the policy; Tyk recognizes that the API record is invalid and denies granting access rights to the key. + + + +The [Logstash](/api-management/logs-metrics#logstash) formatter timestamp is now in [RFC3339Nano](https://www.rfc-editor.org/rfc/rfc3339) format. + + + +Fixed a potential race condition where the *DRL Manager* was not properly protected against concurrent read/write operations in some high-load scenarios. + + + +Fixed a performance issue encountered when Tyk Gateway retrieves a key via MDCB for a JWT API. The token is now validated against [JWKS or the public key](/basic-config-and-security/security/authentication-authorization/json-web-tokens) in the API Definition. + + + +Fixed a performance issue where JWT middleware introduced latency which significantly reduced the overall request/response throughput. + + + +Fixed an issue that prevented *UDG* examples from being displayed in the dashboard when the *Open Policy Agent(OPA)* is enabled. + + + +Fixed an issue where the Tyk Gateway logs would include sensitive information when the incorrect signature is provided in a request to an API protected by HMAC authentication. + + +
    • +
+ +##### Community Contributions + +Special thanks to the following members of the Tyk community for their contributions to this release: + +
    +
  • + +- Implemented *ULID Normalization*, replacing valid ULID identifiers in the URL with a `{ulid}` placeholder for analytics. This matches the existing UUID normalization. Thanks to [Mohammad Abdolirad](https://github.com/atkrad) for the contribution. + +
    • +
  • + +Fixed an issue where a duplicate error message was reported when a custom Go plugin returned an error. Thanks to [@PatrickTaibel](https://github.com/PatrickTaibel) for highlighting the issue and suggesting a fix. + +
    • +
+ + +--- + +### 5.2.1 Release Notes + +#### Release Date 10 Oct 2023 + +#### Breaking Changes +**Attention**: Please read carefully this section. We have two topics to report: + +#### Early Access Features: +Please note that the `Tyk OAS APIs` feature, currently marked as *Early Access*, is subject to breaking changes in subsequent releases. Please refer to our [Early Access guide](/developer-support/release-types/early-access-feature) for specific details. Upgrading to a new version may introduce changes that are not backwards-compatible. Downgrading or reverting an upgrade may not be possible resulting in a broken installation. + +Users are strongly advised to follow the recommended upgrade instructions provided by Tyk before applying any updates. + +#### Deprecations +There are no deprecations in this release. + +#### Upgrade Instructions +If you are on a 5.2.0 we advise you to upgrade ASAP and if you are on an older version skip 5.2.0 and upgrade directly to this release. Go to the [Upgrading Tyk](#upgrading-tyk) section for detailed upgrade instructions. + +#### Release Highlights +This release primarily focuses on bug fixes. +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.2.0) below. + +#### Downloads +- [Docker image to pull](https://hub.docker.com/layers/tykio/tyk-gateway/v5.2.1/images/sha256-47cfffda64ba492f79e8cad013a476f198011f5a97cef32464f1f47e1a9be9a2?context=explore) +- [source code](https://github.com/TykTechnologies/tyk/releases/tag/v5.1.2) + +

Changelog

+##### Changed + +
    +
  • + + +Enhance log message quality by eliminating unnecessary messages + + + +Fixed a bug that occurs during Gateway reload where the Gateway would continue to load new API definitions even if policies failed to load. This led to a risk that an API could be invoked without the associated policies (for example, describing access control or rate limits) having been loaded. Now Tyk offers a configurable retry for resource loading, ensuring that a specified number of attempts will be made to load resources (APIs and policies). If a resource fails to load, an error will be logged and the Gateway reverts to its last working configuration. + +We have introduced two new variables to configure this behavior: +- `resource_sync.retry_attempts` - defines the number of [retries](/tyk-oss-gateway/configuration#resource_syncretry_attempts) that the Gateway should perform during a resource sync (APIs or policies), defaulting to zero which means no retries are attempted +- `resource_sync.interval` - setting the [fixed interval](/tyk-oss-gateway/configuration#resource_syncinterval) between retry attempts (in seconds) + + + +For OpenTelemetry users, we've included much-needed attributes, `http.response.body.size` and `http.request.body.size`, in both Tyk HTTP spans and upstream HTTP spans. This addition enables users to gain better insight into incoming/outgoing request/response sizes within their traces. + + +
    • +
+ +##### Fixed + +
    +
  • + + +Fixed a memory leak issue in Gateway 5.2.0 if [OpenTelemetry](https://opentelemetry.io/) (abbreviated "OTel") is [enabled](/api-management/logs-metrics#opentelemetry). It was caused by multiple `otelhttp` handlers being created. We have updated the code to use a single instance of `otelhttp` handler in 5.2.1 to improve performance under high traffic load. + + + +Fixed a memory leak that occurred when enabling the [strict routes option](/tyk-oss-gateway/configuration#http_server_optionsenable_strict_routes) to change the routing to avoid nearest-neighbor requests on overlapping routes (`TYK_GW_HTTPSERVEROPTIONS_ENABLESTRICTROUTES`) + + + +Fixed a potential performance issue related to high rates of *Tyk Gateway* reloads (when the Gateway is updated due to a change in APIs and/or policies). The gateway uses a timer that ensures there's at least one second between reloads, however in some scenarios this could lead to poor performance (for example overloading Redis). We have introduced a new [configuration option](/tyk-oss-gateway/configuration#reload_interval), `reload_interval` (`TYK_GW_RELOADINTERVAL`), that can be used to adjust the duration between reloads and hence optimize the performance of your Tyk deployment. + + + +Fixed an issue with GraphQL APIs, where [headers](/api-management/graphql#graphql-apis-headers) were not properly forwarded upstream for [GQL/UDG subscriptions](/api-management/graphql#graphql-subscriptions). + + + +Fixed a bug where the Gateway did not correctly close idle upstream connections (sockets) when configured to generate a new connection after a configurable period of time (using the [max_conn_time](/tyk-oss-gateway/configuration#max_conn_time) configuration option). This could lead to the Gateway eventually running out of sockets under heavy load, impacting performance. + + + +Removed the extra chunked transfer encoding that was added unnecessarily to `rawResponse` analytics + + + +Resolved a bug with HTTP GraphQL APIs where, when the [Persist GraphQL middleware](/api-management/graphql#persisting-graphql-queries) was used in combination with [Response Body Transform](/api-management/traffic-transformation/response-body), the response's body transformation was not being executed. +Bug in persistent gql and response body transform + + + +Fixed a bug where, if you created a key which provided access to an inactive or draft API, you would be unable to subsequently modify that key (via the Tyk Dashboard UI, Tyk Dashboard API or Tyk Gateway API) + + +
    • +
+ + +##### Dependencies +- Updated TykTechnologies/gorm to v1.21 in Tyk Gateway + +--- + +### 5.2.0 Release Notes + +#### Release Date 29 Sep 2023 + +#### Breaking Changes +**Attention**: Please read carefully this section. We have two topics to report: + +#### Early Access Features: +Please note that the `Tyk OAS APIs` feature, currently marked as *Early Access*, is subject to breaking changes in subsequent releases. Please refer to our [Early Access guide](/developer-support/release-types/early-access-feature) for specific details. Upgrading to a new version may introduce changes that are not backwards-compatible. Downgrading or reverting an upgrade may not be possible resulting in a broken installation. + +Users are strongly advised to follow the recommended upgrade instructions provided by Tyk before applying any updates. + +#### Deprecations +There are no deprecations in this release. + +#### Release Highlights + +We're thrilled to bring you some exciting enhancements and crucial fixes to improve your experience with Tyk Gateway. For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.2.0) below. + +##### Added Body Transform Middleware to Tyk OAS API Definition + +With this release, we are adding the much requested *Body Transformations* to *Tyk OAS API Definition*. You can now [configure](/api-management/gateway-config-tyk-oas#transformbody) middleware for both [request](/api-management/traffic-transformation/request-body) and [response](/api-management/traffic-transformation/response-body) body transformations and - as a Tyk Dashboard user - you’ll be able to do so from within our simple and elegant API Designer tool. + +##### Reference Tyk OAS API Definition From Within Your Custom Go Plugins + +Reference the *Tyk OAS API definition* from within your custom *Go Plugins*, bringing them up to standard alongside those you might use with a *Tyk Classic API*. + +##### Configure Caching For Each API Endpoint + +We’ve added the ability to [configure](/api-management/response-caching#configuring-the-middleware-in-the-tyk-oas-api-definition) per-endpoint timeouts for Tyk’s response cache, giving you increased flexibility to tailor your APIs to your upstream services. + +##### Added Header Management in Universal Data Graph + +With this release we are adding a concept of [header management](/api-management/data-graph#header-management) in *Universal Data Graph*. With multiple upstream data sources, data graphs need to be sending the right headers upstream, so that our users can effectively track the usage and be able to enforce security rules at each stage. All *Universal Data Graph* headers now have access to *request context* variables like *JWT claims*, *IP address* of the connecting client or *request ID*. This provides extensive configurability of customizable information that can be sent upstream. + +##### Added Further Support For GraphQL WebSocket Protocols + +Support for [WebSocket](/api-management/graphql#graphql-websockets) protocols between client and the *Gateway* has also been expanded. Instead of only supporting the *graphql-ws protocol*, which is becoming deprecated, we now also support [graphql-transport-ws](https://github.com/enisdenjo/graphql-ws/blob/master/PROTOCOL.md) by setting the *Sec-WebSocket-Protocol* header to *graphql-transport-ws*. + +##### Added OpenTelemetry Tracing + +In this version, we're introducing the support for *OpenTelemetry Tracing*, the new [open standard](https://opentelemetry.io/) for exposing observability data. This addition gives you improved visibility into how API requests are processed, with no additional license required. It is designed to help you with monitoring and troubleshooting APIs, identify bottlenecks, latency issues and errors in your API calls. For detailed information and guidance, you can check out our [OpenTelemetry Tracing](/api-management/logs-metrics#opentelemetry) resource. + +*OpenTelemetry* makes it possible to isolate faults within the request lifetime through inspecting API and Gateway meta-data. Additionally, performance bottlenecks can be identified within the request lifetime. API owners and developers can use this feature to understand how their APIs are being used or processed within the Gateway. + +*OpenTelemetry* functionality is also available in [Go Plugins](/api-management/plugins/advance-config#instrumenting-plugins-with-opentelemetry). Developers can write code to add the ability to preview *OpenTelemetry* trace attributes, error status codes etc., for their Go Plugins. + +We offer support for integrating *OpenTelemetry* traces with supported open source tools such [Jaeger](/api-management/logs-metrics#using-docker), [Dynatrace](/api-management/logs-metrics#dynatrace) or [New Relic](/api-management/logs-metrics#new-relic). This allows API owners and developers to gain troubleshooting and performance insights from error logs, response times etc. +You can also find a direct link to our docs in the official [OpenTelemetry Integration page](https://opentelemetry.io/ecosystem/integrations/) + + + +*Tyk Gateway 5.2* now includes *OpenTelemetry Tracing*. Over the next year, we'll be deprecating *OpenTracing*. We recommend migrating to *OpenTelemetry* for better trace insights and more comprehensive support. This change will offer you significant advantages in managing your distributed tracing needs. + + + +#### Downloads + +- [Docker image to pull](https://hub.docker.com/layers/tykio/tyk-gateway/v5.2.0/images/sha256-cf0c57619e8285b1985bd5e4bf86b8feb42abec56cbc241d315cc7f8c0d43025?context=explore) +- [source code](https://github.com/TykTechnologies/tyk/releases/tag/v5.2.0) + +

Changelog

+##### Added: + +
    +
  • + + +Added support for [configuring](/tyk-oss-gateway/configuration#opentelemetry) distributed tracing behavior of *Tyk Gateway*. This includes enabling tracing, configuring exporter types, setting the URL of the tracing backend to which data is to be sent, customizing headers, and specifying enhanced connectivity for *HTTP*, *HTTPS* and *gRPC*. Subsequently, users have precise control over tracing behavior in *Tyk Gateway*. + + + +Added support to configure *OpenTelemetry* [sampling types and rates](/tyk-oss-gateway/configuration#opentelemetrysampling) in the *Tyk Gateway*. This allows users to manage the need for collected detailed tracing information against performance and resource usage requirements. + + + +Added span attributes to simplify identifying Tyk API and request meta-data per request. Example span attributes include: *tyk.api.id*, *tyk.api.name*, *tyk.api.orgid*, *tyk.api.tags*, *tyk.api.path*, *tyk.api.version*, *tyk.api.apikey*, *tyk.api.apikey.alias* and *tyk.api.oauthid*. This allows users to use *OpenTelemetry* [semantic conventions](https://github.com/open-telemetry/opentelemetry-specification/blob/v1.25.0/specification/trace/semantic_conventions/README.md) to filter and create metrics for increased insight and observability. + + + +Added custom resource attributes: *service.name*, *service.instance.id*, *service.version*, *tyk.gw.id*, *tyk.gw.dataplane*, *tyk.gw.group.id*, *tyk.gw.tags* to allow process information to be available in traces. + + + +Added a new feature that allows clients to retrieve the trace ID from response headers. This feature is available when *OpenTelemetry* is [enabled](/tyk-oss-gateway/configuration#opentelemetryenabled) and simplifies debugging API requests, empowering users to seamlessly correlate and analyze data for a specific trace in any *OpenTelemetry* backend like [Jaeger](https://www.jaegertracing.io/). + + + +Added configuration parameter to enable/disable [detailed_tracing](/api-management/logs-metrics#capturing-detailed-logs) for *Tyk Classic API*. + + + +Added *OpenTelemetry* support for GraphQL. This is activated by setting [opentelemetry.enabled](/tyk-oss-gateway/configuration#opentelemetryenabled) to *true*. This integration enhances observability by enabling GQL traces in any OpenTelemetry backend, like [Jaeger](https://www.jaegertracing.io/), granting users comprehensive insights into the execution process, such as request times. + + + +Added a new [timeout option](/api-management/response-caching#configuring-the-middleware-in-the-tyk-oas-api-definition), offering granular control over cache timeout at the endpoint level. + + + +Added support for using [request context variables](/api-management/traffic-transformation/request-context-variables) in *UDG* global or data source headers. This feature enables much more advanced [header management](/api-management/data-graph#header-management) for UDG and allows users to extract header information from an incoming request and pass it to upstream data sources. + + + +Added support for configuration of [global headers](/api-management/data-graph#header-management) for any *UDG*. These headers will be forwarded to all data sources by default, enhancing control over data flow. + + + +Added the ability for Custom GoPlugin developers using *Tyk OAS APIs* to access the *API Definition* from within their plugin. The newly introduced *ctx.getOASDefinition* function provides read-only access to the *OAS API Definition* and enhances the flexibility of plugins. + + + +Added support for the websocket protocol, *graphql-transport-ws protocol*, enhancing communication between the client and *Gateway*. Users [connecting](/api-management/graphql#graphql-websockets) with the header *Sec-WebSocket-Protocol* set to *graphql-transport-ws* can now utilize messages from this [protocol](https://github.com/enisdenjo/graphql-ws/blob/master/PROTOCOL.md) for more versatile interaction. + + + +Added support for API Developers using *Tyk OAS API Definition* to [configure](/api-management/gateway-config-tyk-oas#transformbody) a body transform middleware that operates on API responses. This enhancement ensures streamlined and selective loading of the middleware based on configuration, enabling precise response data customization at the per-endpoint level. + + + +- Added support for enhanced *Gateway* usage reporting. *MDCB v2.4* and *Gateway v5.2* can now report the number of connected gateways and data planes. Features such as data plane gateway visualisation are available in *Tyk Dashboard* for enhanced monitoring of your deployment. + + +
    • +
+ +##### Changed: +
    +
  • + +Updated *Response Body Transform* middleware for *Tyk Classic APIs* to remove unnecessary entries in the *API definition*. The dependency on the *response_processor.response_body_transform* configuration has been removed to streamline middleware usage, simplifying API setup. + +
    • +
+ +##### Fixed: +
    +
  • + + +Fixed an issue with querying a *UDG* API containing a query parameter of array type in a REST data source. The *UDG* was dropping the array type parameter from the final request URL sent upstream. + + + +Fixed an issue with introspecting GraphQL schemas that previously raised an error when dealing with custom root types other than *Query*, *Mutation* or *Subscription*. + + + +Fixed an issue where the [Enforced Timeout](/planning-for-production/ensure-high-availability/enforced-timeouts) configuration parameter of an API endpoint accepted negative values, without displaying validation errors. With this fix, users receive clear feedback and prevent unintended configurations. + + + +Fixed an issue where *allowedIPs* validation failures replaced the reported errors list, causing the loss of other error types. This fix appends IP validation errors to the list, providing users with a comprehensive overview of encountered errors. Subsequently, this enhances the clarity and completeness of validation reporting. + + + +Fixed a critical issue in MDCB v2.3 deployments, relating to *Data Plane* stability. The *Data Plane* Gateway with versions older than v5.1 was found to crash with a panic when creating a Tyk OAS API. The bug has been addressed, ensuring stability and reliability in such deployments. + + +
    • +
+ + +--- + +## 5.1 Release Notes + +### Release Date 23 June 2023 + +### Breaking Changes + +**Attention warning*: Please read carefully this section. + +#### Golang Version upgrade +Our Gateway is using [Golang 1.19](https://tip.golang.org/doc/go1.19) programming language starting with the 5.1 release. This brings improvements to the code base and allows us to benefit from the latest features and security enhancements in Go. Don’t forget that, if you’re using GoPlugins, you'll need to [recompile](/api-management/plugins/golang#upgrading-your-tyk-gateway) these to maintain compatibility with the latest Gateway. + +#### Early Access Features: +Please note that the `Tyk OAS APIs` feature, currently marked as *Early Access*, is subject to breaking changes in subsequent releases. Please refer to our [Early Access guide](/developer-support/release-types/early-access-feature) for specific details. Upgrading to a new version may introduce changes that are not backward-compatible. Downgrading to a previous version after upgrading may result in a broken installation. + +Users are strongly advised to follow the recommended upgrade instructions provided by Tyk before applying any updates. + +### Deprecations +There are no deprecations in this release. + +### Upgrade Instructions +Go to the [Upgrading Tyk](#upgrading-tyk) section for detailed upgrade instructions. + +### Release Highlights + +#### Request Body Size Limits + +We have introduced a new Gateway-level option to limit the size of requests made +to your APIs. You can use this as a first line of defense against overly large +requests that might affect your Tyk Gateways or upstream services. Of course, +being Tyk, we also provide the flexibility to configure API-level and +per-endpoint size limits so you can be as granular as you need to protect and +optimize your services. Check out our improved documentation for full +description of how to use these powerful [features](/api-management/traffic-transformation/request-size-limits). + +#### Changed default RPC pool size for MDCB deployments + +We have reduced the default RPC pool size from 20 to 5. This can reduce the CPU and +memory footprint in high throughput scenarios. Please monitor the CPU and memory +allocation of your environment and adjust accordingly. You can change the pool +size using [slave_options.rpc_pool_size](/tyk-oss-gateway/configuration#slave_optionsrpc_pool_size) + +### Downloads + +- [docker image to pull](https://hub.docker.com/layers/tykio/tyk-gateway/v5.1/images/sha256-3d1e64722be1a983d4bc4be9321ca1cdad10af9bb3662fd6824901d5f22820f1?context=explore) +- [source code](https://github.com/TykTechnologies/tyk/releases/tag/v5.1.0) + + +### Changelog + +#### Added + +- Added `HasOperation`, `Operation` and `Variables` to GraphQL data source API definition for easier nesting +- Added abstractions/interfaces for ExecutionEngineV2 and ExecutionEngine2Executor with respect to graphql-go-tools +- Added support for the `:authority` header when making GRPC requests. If the `:authority` header is not present then some GRPC servers return PROTOCOL_ERROR which prevents custom GRPC plugins from running. Thanks to [vanhtuan0409](https://github.com/vanhtuan0409) from the Tyk Community for his contribution! + +#### Changed + +- Tyk Gateway updated to use Go 1.19 +- Updated [_kin-openapi_](https://github.com/getkin/kin-openapi) dependency to the version [v0.114.0](https://github.com/getkin/kin-openapi/releases/tag/v0.114.0) +- Enhanced the UDG parser to comprehensively extract all necessary information for UDG configuration when users import to Tyk their OpenAPI document as an API definition +- Reduced default CPU and memory footprint by changing the default RPC pool size from 20 to 5 connections. + +#### Fixed + +- Fixed an issue where invalid IP addresses could be added to the IP allow list +- Fixed an issue when using custom authentication with multiple authentication methods, custom authentication could not be selected to provide the base identity +- Fixed an issue where OAuth access keys were physically removed from Redis on expiry. Behavior for OAuth is now the same as for other authorization methods +- Fixed an issue where the `global_size_limit` setting didn't enable request size limit middleware. Thanks to [PatrickTaibel](https://github.com/PatrickTaibel) for the contribution! +- Fixed minor versioning, URL and field mapping issues when importing OpenAPI document as an API definition to UDG +- When the control API is not protected with mTLS we now do not ask for a cert, even if all the APIs registered have mTLS as an authorization mechanism + +### Tyk Classic Portal Changelog + +#### Changed + +- Improved performance when opening the Portal page by optimizing the pre-fetching of required data + + + +## 5.0 Release Notes + +

5.0.15 Release Notes

+#### Release Date 24 October 2024 + +#### Breaking Changes + +There are no breaking changes in this release. + +#### Upgrade Instructions + +Go to the [Upgrading Tyk](/developer-support/release-notes/gateway#upgrading-tyk) +section for detailed upgrade instructions. + +#### Release Highlights + +This patch release for Tyk Gateway addresses critical stability issues for users running Tyk Gateway within the data +plane, connecting to the control plane or Tyk Hybrid. Affected users should upgrade immediately to version 5.0.15 to +avoid service interruptions and ensure reliable operations with the control plane or Tyk Hybrid. + +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v5.0.15) below. + +

Changelog

+##### Fixed + +
    +
  • + +In version 5.0.14, Tyk Gateway could encounter panic when attempting to reconnect to the control plane after it was restarted. This patch version has resolved this issue, ensuring stable connectivity between the gateway and control plane following reconnections and reducing the need for manual intervention. + +
    • +
+ +--- + +

5.0.14 Release Notes

+#### Release Date 18th September 2024 + + + +**Important Update**

Date: 12 October 2024
Topic: Gateway panic when +reconnecting to MDCB control plane or Tyk Cloud
Workaround: Restart Gateway
Affected Product: Tyk +Gateway as an Edge Gateway
Affected versions: v5.6.0, v5.3.6, and v5.0.14
Issue Description:
+ +

We have identified an issue affecting Tyk Gateway deployed as a data plane connecting to the Multi-Data Center Bridge (MDCB) control plane or Tyk Cloud. In the above mentioned Gateway versions a panic may occur when gateway reconnect to the control plane after the control plane is restarted.

+ +

Our engineering team is actively working on a fix, and a patch (versions 5.6.1, 5.3.7, and 5.0.15) will be released soon.

+ +Recommendations:
+
    +
  • For users on versions 5.5.0, 5.3.5, and 5.0.13
    +We advise you to delay upgrading to the affected versions (5.6.0, 5.3.6, or 5.0.14) until the patch is available.
    • + +
  • For users who have already upgraded to 5.6.0, 5.3.6, or 5.0.14 and are experiencing a panic in the gateway:
    +Restarting the gateway process will restore it to a healthy state. If you are operating in a *Kubernetes* environment, Tyk Gateway instance should automatically restart, which ultimately resolves the issue.
    • +
+ +

We appreciate your understanding and patience as we work to resolve this. Please stay tuned for the upcoming patch release, which will address this issue.

+ + + + +#### Breaking Changes + +**Attention:** Please read this section carefully. + +There are no breaking changes in this release. + +#### Upgrade Instructions + +This release is not tightly coupled with Tyk Dashboard v5.0.14, so you do not have to upgrade both together. + +Go to the [Upgrading Tyk](/developer-support/release-notes/gateway#upgrading-tyk) +section for detailed upgrade instructions. + +#### Release Highlights + +This release fixes some issues related to the way that Tyk performs URL path matching, introducing two new Gateway +configuration options to control path matching strictness. + +

Changelog

+##### Added + +
    +
  • + +We have introduced two new options in the `http_server_options` [Gateway +configuration](/tyk-oss-gateway/configuration#http_server_options) that will enforce prefix and/or suffix matching +when Tyk performs checks on whether middleware or other logic should be applied to a request: + +- `enable_path_prefix_matching` ensures that the start of the request path must match the path defined in the API + definition +- `enable_path_suffix_matching` ensures that the end of the request path must match the path defined in the API + definition +- combining `enable_path_prefix_matching` and `enable_path_suffix_matching` will ensure an exact (explicit) match is + performed + +These configuration options provide control to avoid unintended matching of paths from Tyk's default _wildcard_ match. +Use of regex special characters when declaring the endpoint path in the API definition will automatically override these +settings for that endpoint. + +**Tyk recommends that exact matching is employed, but both options default to `false` to avoid introducing a breaking +change for existing users.** + +
    • +
+ +##### Fixed + +
    +
  • + + +Fixed an issue when using granular [Path-Based +Permissions](/api-management/policies#secure-your-apis-by-method-and-path) in access policies and keys that led to authorization +incorrectly being granted to endpoints if an invalid regular expression was configured in the key/policy. Also fixed an issue +where path-based parameters were not correctly handled by Path-Based Permissions. Now Tyk's authorization check correctly +handles both of these scenarios granting access only to the expected resources. + + + +Fixed an issue where a parameterized endpoint URL (e.g. `/user/{id}`) would be invoked if a request is made that omits +the parameter. For example, a request to `/user/` will now be interpreted as a request to `/user` and not to +`/user/{id}`. + + + +We have enhanced the Tyk Gateway's synchronization with MDCB to ensure more reliable loading of policies and APIs. A +synchronous initialization process has been implemented to prevent startup failures and reduce the risk of service +disruptions caused by asynchronous operations. This update ensures smoother and more consistent syncing of policies and +APIs from MDCB. + + +
    • +
+ +--- + +### 5.0.13 Release Notes + +#### Release Date 4 July 2024 + +#### Release Highlights + +Resolved an issue encountered in MDCB environments where changes to custom keys made via the Dashboard were not properly +replicated to data planes. The issue impacted both key data and associated quotas, in the following versions: + +- 5.0.4 to 5.0.12 +- 5.1.1 and 5.1.2 +- 5.2.0 to 5.2.6 +- 5.3.0 to 5.3.2 + +###### Action Required + +Customers should clear their edge Redis instances of any potentially affected keys to maintain data consistency and +ensure proper synchronization across their environments. Please refer to the item in the [fixed](#fixed) section of the +changelog for recommended actions. + +

Changelog

+##### Fixed + +
    +
  • + +Resolved a critical issue affecting MDCB environments, where changes to custom keys made via the dashboard were not +properly replicated to data planes. This affected both the key data and associated quotas. This issue was present in +versions: + +- 5.0.4 to 5.0.12 +- 5.1.1 and 5.1.2 +- 5.2.0 to 5.2.6 +- 5.3.0 to 5.3.2 + +**Action Required** + +Customers are advised to clear their edge Redis instances of any keys that might have been affected by this bug to +ensure data consistency and proper synchronization across their environments. There are several methods available to +address this issue: + +1. **Specific Key Deletion via API**: To remove individual buggy keys, you can use the following API call: + +```bash +curl --location --request DELETE 'http://tyk-gateway:{tyk-hybrid-port}/tyk/keys/my-custom-key' \ --header 'X-Tyk-Authorization: {dashboard-key}' +``` + +Replace `{tyk-hybrid-port}`, `my-custom-key` and `{dashboard-key}` with your specific configuration details. This method +is safe and recommended for targeted removals without affecting other keys. + +2. **Bulk Key Deletion Using Redis CLI**: For environments with numerous affected keys, you might consider using the + Redis CLI to remove keys en masse: + +```bash +redis-cli --scan --pattern 'apikey-*' | xargs -L 1 redis-cli del +redis-cli --scan --pattern 'quota-*' | xargs -L 1 redis-cli del +``` + +This method can temporarily impact the performance of the Redis server, so it should be executed during a maintenance +window or when the impact on production traffic is minimal. + +3. **Complete Redis Database Flush**: If feasible, flushing the entire Redis database offers a clean slate: + +```bash +redis-cli FLUSHALL ASYNC +``` expandable + +**Implications** Regardless of the chosen method, be aware that quotas will be reset and will need to resynchronize +across the system. This may temporarily affect reporting and rate limiting capabilities. + +
    • +
+ +--- + +### 5.0.12 Release Notes + +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.12). + +--- + +### 5.0.11 Release Notes + +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.11). + +--- + +### 5.0.10 Release Notes + +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.10). + +--- + +### 5.0.9 Release Notes + +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.9). + +--- + +### 5.0.8 Release Notes + +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.8). + +--- + +### 5.0.7 Release Notes + +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.7). + +--- + +### 5.0.6 Release Notes + +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.6). + +--- + +### 5.0.5 Release Notes + +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.5). + +--- + +### 5.0.4 Release Notes + +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.4). + +--- + +### 5.0.3 Release Notes + +Please refer to our GitHub [release notes](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.3). + +--- + +### 5.0.2 Release Notes + +#### Release Date 29 May 2023 + +#### Release Highlights + +This release primarily focuses on bug fixes. For a comprehensive list of changes, please refer to the detailed +[changelog](#Changelog-v5.0.2) below. + +#### Downloads + +- [docker image to pull](https://hub.docker.com/layers/tykio/tyk-gateway/v5.0.2/images/sha256-5e126d64571989f9e4b746544cf7a4a53add036a68fe0df4502f1e62f29627a7?context=explore) +- [source code](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.2) + +

Changelog

+##### Updated + +- Internal refactoring to make storage related parts more stable and less affected by potential race issues + +--- + +### 5.0.1 Release Notes + +#### Release Date 25 Apr 2023 + +#### Release Highlights + +This release primarily focuses on bug fixes. For a comprehensive list of changes, please refer to the detailed +[changelog](#Changelog-v5.0.1) below. + +#### Downloads + +- [docker image to pull](https://hub.docker.com/layers/tykio/tyk-gateway/v5.0.1/images/sha256-5fa7aa910d62a7ed2c1cfbc68c69a988b4b0e9420d7a52018f80f9a45cadb083?context=explore +- [source code](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.1) + +

Changelog

+##### Added + +- Added a new `enable_distributed_tracing` option to the NewRelic config to enable support for Distributed Tracer + +##### Fixed + +- Fixed panic when JWK method was used for JWT authentication and the token didn't include kid +- Fixed an issue where failure to load GoPlugin middleware didn’t prevent the API from proxying traffic to the upstream: + now Gateway logs an error when the plugin fails to load (during API creation/update) and responds with HTTP 500 if the + API is called; at the moment this is fixed only for file based plugins +- Fixed MutualTLS issue causing leak of allowed CAs during TLS handshake when there are multiple mTLS APIs +- Fixed a bug during hot reload of Tyk Gateway where APIs with JSVM plugins stored in filesystem were not reloaded +- Fixed a bug where the gateway would remove the trailing `/`at the end of a URL +- Fixed a bug where nested field-mappings in UDG weren't working as intended +- Fixed a bug when using Tyk OAuth 2.0 flow on Tyk Cloud where a request for an Authorization Code would fail with a 404 + error +- Fixed a bug where mTLS negotiation could fail when there are a large number of certificates and CAs; added an option + (`http_server_options.skip_client_ca_announcement`) to use the alternative method for certificate transfer +- Fixed CVE issue with go.uuid package +- Fixed a bug where rate limits were not correctly applied when policies are partitioned to separate access rights and + rate limits into different scopes + +--- + +### 5.0.0 Release Notes + +#### Release Date 28 Mar 2023 + +#### Deprecations + +- Tyk Gateway no longer natively supports **LetsEncrypt** integration. You still can use LetsEncrypt CLI tooling to + generate certificates and use them with Tyk. + +#### Release Highlights + +##### Improved OpenAPI support + +We have added some great features to the Tyk OAS API definition bringing it closer to parity with our Tyk Classic API +and to make it easier to get on board with Tyk using your Open API workflows. + +Tyk’s OSS users can now make use of extensive [custom middleware](/api-management/plugins/overview) options with your OAS +APIs, to transform API requests and responses, exposing your upstream services in the way that suits your users and +internal API governance rules. We’ve enhanced the Request Validation for Tyk OAS APIs to include parameter validation +(path, query, headers, cookie) as well as the body validation that was introduced in Tyk 4.1. + +[Versioning your Tyk OAS APIs](/api-management/api-versioning) is easier than ever, with the +Tyk OSS Gateway now looking after the maintenance of the list of versions associated with the base API for you; we’ve +also added a new endpoint on the Tyk API that will return details of the versions for a given API. + +We’ve improved support for [OAS +Mock Responses](/api-management/traffic-transformation/mock-response), with the Tyk OAS API +definition now allowing you to register multiple Mock Responses in a single API, providing you with increased testing +flexibility. + +Of course, we’ve also addressed some bugs and usability issues as part of our ongoing ambition to make Tyk OAS API the +best way for you to create and manage your APIs. + +Thanks to our community contributors [armujahid](https://github.com/armujahid), +[JordyBottelier](https://github.com/JordyBottelier) and [ls-michal-dabrowski](https://github.com/ls-michal-dabrowski) +for your PRs that further improve the quality of Tyk OSS Gateway! + +#### Downloads + +- [docker image to pull](https://hub.docker.com/layers/tykio/tyk-gateway/v5.0.0/images/sha256-196815adff2805ccc14c267b14032f23913321b24ea86c052b62a7b1568b6725?context=explore) +- [source code](https://github.com/TykTechnologies/tyk/releases/tag/v5.0.0) + +

Changelog

+##### Added + +- Support for request validation (including query params, headers and the rest of OAS rules) with Tyk OAS APIs +- Transform request/response middleware for Tyk OAS APIs +- Custom middleware for Tyk OAS APIs +- Added a new API endpoint to manage versions for Tyk OAS APIs +- Improved Mock API plugin for Tyk OAS APIs +- Universal Data Graph and GraphQL APIs now support using context variables in request headers, allowing passing + information it to your subgraphs +- Now you can control access to introspection on policy and key level + +#### Fixed + +- Fixed potential race condition when using distributed rate limiter + +--- + +## 4.3 Release Notes + +### 4.3.0 Release Notes + +#### Release Highlights + +##### Mock Responses with Tyk OAS API Definitions + +Does your Tyk OAS API Definition define examples or a schema for your path responses? If so, starting with Tyk v4.3, Tyk can use those configurations to mock your API responses, enabling your teams to integrate easily without being immediately dependent on each other. Check it out! [Mock Responses Documentation](/api-management/traffic-transformation/mock-response) + +##### External OAuth - 3rd party OAuth IDP integration + +If you’re using a 3rd party IDP to generate tokens for your OAuth applications, Tyk can now validate the generated tokens by either performing JWT validation or by communicating with the authorization server and executing token introspection. + +This can be achieved by configuring the new External OAuth authentication mechanism. Find out more here [External OAuth Integration](/api-management/client-authentication#integrate-with-external-authorization-server-deprecated) + +##### Updated the Tyk Gateway version of Golang, to 1.16. + +**Our Gateway is using Golang 1.16 version starting with 4.3 release. This version of the Golang release deprecates x509 commonName certificates usage. This will be the last release where it's still possible to use commonName, users need to explicitly re-enable it with an environment variable.** + +The deprecated, legacy behavior of treating the CommonName field on X.509 certificates as a host name when no Subject Alternative Names are present is now disabled by default. It can be temporarily re-enabled by adding the value x509ignoreCN=0 to the GODEBUG environment variable. + +Note that if the CommonName is an invalid host name, it's always ignored, regardless of GODEBUG settings. Invalid names include those with any characters other than letters, digits, hyphens and underscores, and those with empty labels or trailing dots. + +##### Improved GQL security + +4.3 adds two important features that improve security settings for GraphQL APIs in Tyk. + +1. Ability to turn on/off introspection - this feature allows much more control over what consumers are able to do when interacting with a GraphQL API. In cases where introspection is not desirable, API managers can now disallow it. The setting is done on API key level, which means API providers will have very granular control over who can and who cannot introspect the API. +2. Support for allow list in field-based permissions - so far Tyk was offering field-based permissions as a β€œblock list” only. That meant that any new field/query added to a graph was by default accessible for all consumers until API manager explicitly blocked it on key/policy level. Adding support for β€œallow list” gives API managers much more control over changing schemas and reduces the risk of unintentionally exposing part of the graph that are not ready for usage. See [Introspection](/api-management/graphql#introspection) for more details. + + +#### Changelog + +##### Tyk Gateway + +###### Added +- Minor modifications to the Gateway needed for enabling support for Graph Mongo Pump. +- Added header `X-Tyk-Sub-Request-Id` to each request dispatched by federated supergraph and Universal Data Graph, so that those requests can be distinguished from requests directly sent by consumers. +- Added a functionality that allows to block introspection for any GraphQL API, federated supergraph and Universal Data Graph (currently only supported via Gateway, UI support coming in the next release). +- Added an option to use allow list in field-based permissions. Implemented for full types and individual fields. (currently only supported via Gateway, UI support coming in the next release) +- Added new middleware that can be used with HTTP APIs to set up persisted queries for GraphQL upstreams. +- Added support for two additional subscription protocols for GraphQL subscriptions. Default protocol used between the gateway and upstream remains to be `graphql-ws`, two additional protocols are possible to configure and use: `graphql-transport-ws` and `SSE`. + +###### Changed + +Updated the Tyk Gateway version of Golang, to 1.16. + +**SECURITY: The release deprecates x509 commonName certificates usage. This will be the last release where it's still possible to use commonName, users need to explicitly re-enable it with an environment variable.** + +The deprecated, legacy behavior of treating the CommonName field on X.509 certificates as a host name when no Subject Alternative Names are present is now disabled by default. It can be temporarily re-enabled by adding the value x509ignoreCN=0 to the GODEBUG environment variable. + +Note that if the CommonName is an invalid host name, it's always ignored, regardless of GODEBUG settings. Invalid names include those with any characters other than letters, digits, hyphens and underscores, and those with empty labels or trailing dots. + +###### Fixed + +- Fixed an issue where introspection query was returning a wrong response in cases where introspection query had additional objects. +- Fixed an issue where gateway was crashing when a subscription was started while no datasource was connected to it. +- Fixed a problem with missing configuration in the GraphQL config adapter that caused issues with batching requests to subgraphs in GraphQL API federation setting. +- A HTTP OAS API version lifetime respects now the date value of the expiration field from Tyk OAS API Definition. +- Now it is possible to proxy traffic from a HTTP API (using Tyk Classic API Definition) to a HTTP OAS API (using Tyk OAS API Definition) and vice versa. + + +#### Updated Versions + +Tyk Gateway 4.3 ([docker images](https://hub.docker.com/r/tykio/tyk-gateway/tags?page=1&name=4.3.0) + +#### Upgrade process + +Follow the [standard upgrade guide](/developer-support/upgrading), there are no breaking changes in this release. + +If you want switch from MongoDB to SQL, you can [use our migration tool](/planning-for-production/database-settings#migrating-from-an-existing-mongodb-instance), but keep in mind that it does not yet support the migration of your analytics data. + + + +Note: Upgrading the Golang version implies that all the Golang custom plugins that you are using need to be recompiled before migrating to 4.3 version of the Gateway. Check our docs for more details [Golang Plugins](/api-management/plugins/golang). + + + +## 4.2 Release Notes + +### 4.2.0 Release Notes + +#### Release Highlights + +##### GraphQL Federation improvements + +###### Changed GUI in Universal Data Graph configuration section. + +A new GUI introduces enhancements to the user experience and more consistent user journey for UDG. +This change does not yet cover all possible use cases and is released with a feature flag. To enable the new GUI, analytics.conf needs the following setting: + +``` +"ui": { + "dev": true +} +``` expandable + +What’s possible with this change: +- Importing GraphQL schema created outside of Tyk (formats accepted .json, .graphql, .grahqls) +- Creating GraphQL schema in Tyk using schema editor +- Hide/Unhide schema editor to focus on graphical representation of the schema +- Resizing schema editor to adjust workspace look & feel to user preferences +- Improved search in schema editor (search and search & replace available) +- Quick link to UDG documentation from schema editor + +> Note: Full configuration of new Universal Data Graph is not yet possible in the GUI, however any UDGs created earlier will not be broken and will work as previously. + +##### Changes to federation entities +###### Defining the base entity +Entities must be defined with the `@key` directive. The fields argument must reference a field by which the entity can be uniquely identified. Multiple primary keys are possible. For example: + +Subgraph 1 (base entity): +``` +type MyEntity @key(fields: "id") @key(fields: "name") { + id: ID! + name: String! +} +``` + Attempting to extend a non-entity with an extension that includes the @key directive or attempting to extend a base entity with an extension that does not include the @key directive will both result in errors. + +###### Entity stubs + +Entities cannot be shared types (be defined in more than one single subgraph). +If one subgraph references a base entity (an entity defined in another subgraph), that reference must be declared as a stub (stubs look like an extension without any new fields in federation v1). This stub would contain the minimal amount of information to identify the entity (referencing exactly one of the primary keys on the base entity regardless of whether there are multiple primary keys on the base entity). For example, a stub for MyEntity from Subgraph 1 (defined above): + +Subgraph 2 (stub) +``` +extend type MyEntity @key(fields: "id") { + id: ID! @external +} +``` expandable + +###### Supergraph extension orphans +It is now possible to define an extension for a type in a subgraph that does not define the base type. +However, if an extension is unresolved (an extension orphan) after an attempted federation, the federation will fail and produce an error. + +###### Improved Dashboard UI and error messages +GraphQL-related (for example when federating subgraphs into a supergraph) errors in the Dashboard UI will show a lean error message with no irrelevant prefixes or suffixes. + +Changed the look & feel of request logs in Playground tab for GraphQL APIs. New component presents all logs in a clearer way and is easier to read for the user + +###### Shared types +Types of the same name can be defined in more than one subgraph (a shared type). This will no longer produce an error if each definition is identical. +Shared types cannot be extended outside of the current subgraph, and the resolved extension must be identical to the resolved extension of the shared type in all other subgraphs (see subgraph normalization notes). Attempting to extend a shared type will result in an error. +The federated supergraph will include a single definition of a shared type, regardless of how many times it has been identically defined in its subgraphs. + +###### Subgraph normalization before federation +Extensions of types whose base type is defined in the same subgraph will be resolved before an attempt at federation. A valid example involving a shared type: + +Subgraph 1: +``` +enum Example { + A, + B +} + +extend enum Example { + C +} +``` + +Subgraph 2: +``` +enum Example { + A, + B, + C +} +``` expandable + +The enum named β€œExample” defined in Subgraph 1 would resolve to be identical to the same-named enum defined in Subgraph 2 before federation takes place. The resulting supergraph would include a single definition of this enum. + +###### Validation +Union members must be both unique and defined. +Types must have bodies, e.g., enums must contain at least one value; inputs, interfaces, or objects must contain at least one field + +##### OpenAPI +Added support for the Request Body Transform middleware, for new Tyk OAS API Definitions. + +##### Universal Data Graph + +Added support for Kafka as a data source in Universal Data Graph. Configuration allows the user to provide multiple topics and broker addresses. + +#### Changelog + +##### Tyk Gateway +###### Added +- Added support for Kafka as a data source in Universal Data Graph. +- Adding a way to defining the base GraphQL entity via @key directive +- It is now possible to define an extension for a type in a subgraph that does not define the base type. +- Added support for the Request Body Transform middleware, for the new Tyk OAS API Definition +- Session lifetime now can be controlled by Key expiration, e.g. key removed when it is expired. Enabled by setting `session_lifetime_respects_key_expiration` to `true` +###### Changed +- Generate API ID when API ID is not provided while creating API. +- Updated the Go plugin loader to load the most appropriate plugin bundle, honoring the Tyk version, architecture and OS +- When GraphQL query with a @skip directive is sent to the upstream it will no longer return β€œnull” for the skipped field, but remove the field completely from the response +- Added validation to Union members - must be both unique and defined. +###### Fixed +- Fixed an issue where the Gateway would not create the circuit breaker events (BreakerTripped and BreakerReset) for which the Tyk Dashboard offers webhooks. +- Types of the same name can be defined in more than one subgraph (a shared type). This will no longer produce an error if each definition is exactly identical. +- Apply Federation Subgraph normalization do avoid merge errors. Extensions of types whose base type is defined in the same subgraph will be resolved before an attempt at federation. + +#### Updated Versions +Tyk Gateway 4.2 + +#### Upgrade process + +Follow the [standard upgrade guide](/developer-support/upgrading), there are no breaking changes in this release. + +If you want switch from MongoDB to SQL, you can [use our migration tool](/planning-for-production/database-settings#migrating-from-an-existing-mongodb-instance), but keep in mind that it does not yet support the migration of your analytics data. + +## 4.1 Release Notes + +### 4.1.0 Release Notes + +#### Release Highlights + +##### OpenAPI as a native API definition format +Tyk has always had a proprietary specification for defining APIs. From Tyk v4.1 we now support defining APIs using the Open API Specification (OAS) as well, which can offer significant time and complexity savings. [This is an early access capability](/developer-support/release-types/early-access-feature). + +As we extend our OAS support, we would very much like your feedback on how we can extend and update to best meet your needs: . + +This capability is available in both the open source and paid versions of Tyk. See our [Tyk OAS documentation](/api-management/gateway-config-tyk-oas) for more details. + + +##### MDCB Synchroniser + +Tyk Gateway v4.1 enables an improved synchroniser functionality within Multi Data Center Bridge (MDCB) v2.0. Prior to this release, the API keys, certificates and OAuth clients required by worker Gateways were synchronised from the controller Gateway on-demand. With Gateway v4.1 and MDCB v2.0 we introduce proactive synchronisation of these resources to the worker Gateways when they start up. + +This change improves resilience in case the MDCB link or controller Gateway is unavailable, because the worker Gateways can continue to operate independently using the resources stored locally. There is also a performance improvement, with the worker Gateways not having to retrieve resources from the controller Gateway when an API is first called. + +Changes to keys, certificates and OAuth clients are still synchronised to the worker Gateways from the controller when there are changes and following any failure in the MDCB link. + +##### Go Plugin Loader +When upgrading your Tyk Installation you need to re-compile your plugin with the new version. At the moment of loading a plugin, the Gateway will try to find a plugin with the name provided in the API definition. If none is found then it will fallback to search the plugin file with the name: `{plugin-name}_{Gw-version}_{OS}_{arch}.so` + +From v4.1.0 the plugin compiler automatically names plugins with the above naming convention. It enables you to have one directory with different versions of the same plugin. For example: + +- `plugin_v4.1.0_linux_amd64.so` +- `plugin_v4.2.0_linux_amd64.so` + +So, if you upgrade from Tyk v4.1.0 to v4.2.0 you only need to have the plugins compiled for v4.2.0 before performing the upgrade. + +#### Changelog + +##### Tyk Gateway +###### Added +- Added support for new OAS API definition format +- Added support for headers on subgraph level for federated GraphQL APIs +- Added support for interfaces implementing interfaces in GQL schema editor +- Added support for passing authorization header in GQL API Playgrounds for subscription APIs +- Added TYK_GW_OMITCONFIGFILE option for Tyk Gateway to ignore the values in the config file and load its configuration only from environment variables and default values +- Added a way to modify Tyk analytics record via Go plugins [configurable with API definition](/api-management/plugins/plugin-types#analytics-plugins). Can be used to sanitise analytics data. +- Added new policy API REST endpoints +- Added option to configure certificates for Tyk Gateway using [environment variable](/tyk-oss-gateway/configuration#http_server_optionscertificates) +- Added support for Python 3.9 plugins +- Added support for headers on subgraph level for federated GraphQL APIs +- Added support for introspecting schemas with interfaces implementing interfaces for proxy only GQL +- Added support for input coercion in lists for GraphQL +- Added support for repeatable directives for GraphQL +###### Changed +- Generate API ID when API ID is not provided while creating API. +- Updated the Go plugin loader to load the most appropriate plugin bundle, honoring Tyk version, architecture and OS +- When a GraphQL query with a @skip directive is sent to the upstream it will no longer return β€œnull” for the skipped field, but remove the field completely from the response +###### Fixed +- Fixed a bug where the MDCB worker Gateway could become unresponsive when a certificate is added in the Tyk Dashboard +- Fixed an issue with the calculation of TTL for keys in an MDCB deployment such that TTL could be different between worker and controller Gateways +- Fixed a bug when using Open ID where quota was not tracked correctly +- Fixed multiple issues with schema merging in GraphQL federation. Federation subgraphs with the same name shared types like objects, interfaces, inputs, enums, unions and scalars will no longer cause errors when users are merging schemas into a federated supergraph. +- Fixed an issue where schema merging in GraphQL federation could fail depending on the order or resolving subgraph schemas and only first instance of a type and its extension would be valid. Subgraphs are now individually normalized before a merge is attempted and all extensions that are possible in the federated schema are applied. +- Fixed an issue with accessing child properties of an object query variable for GraphQL where query `{{.arguments.arg.foo}}` would return `{ "foo":"123456" }` instead of "123456" + +#### Updated Versions +Tyk Gateway 4.1 +Tyk MDCB 2.0.1 + +#### Upgrade process + +Follow the [standard upgrade guide](/developer-support/upgrading), there are no breaking changes in this release. + +If you want switch from MongoDB to SQL, you can [use our migration tool](/planning-for-production/database-settings#migrating-from-an-existing-mongodb-instance), but keep in mind that it does not yet support the migration of your analytics data. + +## 4.0 Release Notes + +### 4.0.0 Release Notes + +#### Release Highlights + +##### GraphQL federation + +As we know, ease-of-use is an important factor when adopting GraphQL. Modern enterprises have dozens of backend services and need a way to provide a unified interface for querying them. Building a single, monolithic GraphQL server is not the best option. It is hard to maintain and leads to a lot of dependencies and over-complication. + +To remedy this, Tyk 4.0 offers GraphQL federation that allows the division of GraphQL implementation across multiple backend services, while still exposing them all as a single graph for the consumers. Subgraphs represent backend services and define a distinct GraphQL schema. A subgraph can be queried directly, as a separate service or federated in the Tyk Gateway into a larger schema of a supergraph – a composition of several subgraphs that allows execution of a query across multiple services in the backend. + +[Federation docs](/api-management/graphql#overview-1) + +[Subgraphs and Supergraphs docs](/api-management/graphql#subgraphs-and-supergraphs) + +##### GraphQL subscriptions + +Subscriptions are a way to push data from the server to the clients that choose to listen to real-time messages from the server, using the WebSocket protocol. There is no need to enable subscriptions separately; Tyk supports them alongside GraphQL as standard. + +With release 4.0, users can federate GraphQL APIs that support subscriptions. Federating subscriptions means that events pushed to consumers can be enriched with information from other federated graphs. + +[Subscriptions docs](/api-management/graphql#graphql-subscriptions) + +#### Changelog + +- Now it is possible to configure GraphQL upstream authentification, in order for Tyk to work with its schema +- JWT scopes now support array and comma delimiters +- Go plugins can be attached on per-endpoint level, similar to virtual endpoints + +#### Updated Versions + +Tyk Gateway 4.0 +Tyk Pump 1.5 + +#### Upgrade process + +Follow the [standard upgrade guide](/developer-support/upgrading), there are no breaking changes in this release. + +If you want switch from MongoDB to SQL, you can [use our migration tool](/planning-for-production/database-settings#migrating-from-an-existing-mongodb-instance), but keep in mind that it does not yet support the migration of your analytics data. + +## 3.2 Release Notes + +### 3.2.0 Release Notes + +#### Release Highlights + +##### GraphQL and UDG improvements + +We've updated the GraphQL functionality of our [Universal Data Graph](/api-management/data-graph#overview). You’re now able to deeply nest GraphQL & REST APIs and stitch them together in any possible way. + +Queries are now possible via WebSockets and Subscriptions are coming in the next Release (3.3.0). + +You're also able to configure [upstream Headers dynamically](/api-management/data-graph#header-forwarding), that is, you’re able to inject Headers from the client request into UDG upstream requests. For example, it can be used to access protected upstreams. + +We've added an easy to use URL-Builder to make it easier for you to inject object fields into REST API URLs when stitching REST APIs within UDG. + +Query-depth limits can now be configured on a per-field level. + +If you’re using GraphQL upstream services with UDG, you’re now able to forward upstream error objects through UDG so that they can be exposed to the client. + +##### Go response plugins + +With Go response plugins you are now able to modify and create a full request round trip made through the Tyk Gateway. +Find out more about [plugins](/api-management/plugins/overview#) and how to write [Go response plugins](/api-management/plugins/golang#creating-a-custom-response-plugin). + +#### Changelog + +In addition to the above, version 3.2 includes all the fixes that are part of 3.0.5 +https://github.com/TykTechnologies/tyk/releases/tag/v3.0.5 + +#### Updated Versions +Tyk Gateway 3.2 + +#### Upgrade process +If you already have GraphQL or UDG APIs you need to follow this [upgrade guide](/api-management/graphql#migrating-to-32) + +## 3.1 Release Notes + +### 3.1.0 Release Notes + +#### Release Highlights + +##### Identity Management UX and SAML support +You will notice that the experience for creating a new profile in the Identity management section of the dashboard was changed to a β€˜wizard’ approach which reduces the time it takes to get started and configure a profile. +In addition, users are now able to use SAML for the dashboard and portal login, whether you use TIB(Tyk Identity Broker) internally or externally of the dashboard. + +This follows the recent changes that we have made to embed TIB (Tyk Identity Broker)in the dashboard. See 3.0 [release notes](/developer-support/release-notes/dashboard#tyk-identity-broker-now-built-in-to-the-dashboard) for more information regarding this. + +To learn more [see the documentation](/api-management/external-service-integration) + +##### UDG (Universal Data Graph) & GraphQL +###### Schema Validation + +For any GraphQL API that is created via Dashboard or through our API, the GraphQL schema is now validated before saving the definition. Instant feedback is returned in case of error. + +###### Sync / Update schema with upstream API (Proxy Only Mode) + +If you’ve configured just a proxy GraphQL API, you can now keep in sync the upstream schema with the one from the API definition, just by clicking on the `Get latest version` button on the `Schema` tab from API Designer + +Docs [here](/api-management/graphql#syncing-gql-schema) + +###### Debug logs + +You can now see what responses are being returned by the data sources used while configuring a UDG (universal data graph). These can be seen by calling the `/api/debug` API or using the playground tab within API designer. + +The data that will be displayed will show information on the query before and after the request to a data source happens, as follows: + +Before the request is sent: + +Example log message: "Query.countries: preSendHttpHook executed”. Along with this message, the log entry will contain the following set of fields: Typename, Fieldname and Upstream url; + + +After the request is sent: + +Example log message: "Query.countries: postReceiveHttpHook executed”. Along with this message, the log entry will contain the following set of fields: Typename, Filename, response body, status code. + +Example: + +```{"typename": "Query", "fielname": "countries", "response_body": "{\"data\":{}}", "status_code": 200}``` + +Docs [here](/api-management/graphql#graphql-playground) + +##### Portal +###### GraphQL Documentation + +Documentation for the GraphQL APIs that you are exposing to the portal is available now through a GraphQL Playground UI component, same as on the playground tab of API Designer. + +Also to overcome the CORS issues that you might encounter while testing documentation pages on the portal, we have pre-filled the CORS settings section in API Designer with explicit values from the start. All you need to do is to check the β€œEnable CORS” option. + +###### Portal - API key is hidden in email +You now have the option to hide the API key in the email generated after you approve the key request for a developer. + +[Docs here](/tyk-developer-portal/tyk-portal-classic/key-requests) + +#### Changelog +The 3.1 version includes the fixes that are part of 3.0.1. +https://github.com/TykTechnologies/tyk/releases/tag/v3.0.1 + + +#### Updated Versions + +- Tyk Gateway 3.1 + +## 3.0 Release Notes + +### 3.0.0 Release Notes + +#### Release Highlights + +##### Version changes and LTS releases + +We have bumped our major Tyk Gateway version from 2 to 3, a long overdue change as we’ve been on version 2 for 3 years. We have also changed our Tyk Dashboard major version from 1 to 3, and from now on it will always be aligned with the Tyk Gateway for major and minor releases. The Tyk Pump has also now updated to 1.0, so we can better indicate major changes in future. + +Importantly, such a big change in versions does not mean that we going to break backward compatibility. More-over we are restructuring our internal release strategy to guarantee more stability and to allow us to deliver all Tyk products at a faster pace. We aim to bring more clarity to our users on the stability criteria they can expect, based on the version number. +Additionally we are introducing Long Term Releases (also known as LTS). + +Read more about this changes in our blog post: https://tyk.io/blog/introducing-long-term-support-some-changes-to-our-release-process-product-versioning/ + + +##### Universal Data Graph and GraphQL + +Tyk now supports GraphQL **natively**. This means Tyk doesn’t have to use any external services or process for any GraphQL middleware. You can securely expose existing GraphQL APIs using our GraphQL core functionality. + +In addition to this you can also use Tyk’s integrated GraphQL engine to build a Universal Data Graph. The Universal Data Graph (UDG) lets you expose existing services as one single combined GraphQL API. + +All this without even have to build your own GraphQL server. If you have existing REST APIs all you have to do is configure the UDG and Tyk has done the work for you. + +With the Universal Data Graph (UDG), Tyk becomes the central integration point for all your internal and external APIs. +It also benefits from the full set of capabilities included with your Tyk installationβ€”meaning your data graph is secure from the start and can take advantage of a wide range of out-of-the-box middleware to power your graph. + +Read more about the [GraphQL](/api-management/graphql) and [Universal Data Graph](/api-management/data-graph#overview) + +##### Using external secret management services + +Want to reference secrets from a KV store in your API definitions? We now have native Vault & Consul integration. You can even pull from a tyk.conf dictionary or environment variable file. + +[Read more](/tyk-configuration-reference/kv-store) + +##### Co-Process Response Plugins + +We added a new middleware hook allowing middleware to modify the response from the upstream. Using response middleware you can transform, inspect or obfuscate parts of the response body or response headers, or fire an event or webhook based on information received by the upstream service. + +At the moment the Response hook is supported for [Python and gRPC plugins](/api-management/plugins/rich-plugins#coprocess-dispatcher---hooks). + + +##### Enhanced Gateway health check API + +Now the standard Health Check API response include information about health of the dashboard, redis and mdcb connections. +You can configure notifications or load balancer rules, based on new data. For example, you can be notified if your Tyk Gateway can’t connect to the Dashboard (or even if it was working correctly with the last known configuration). + +[Read More](/planning-for-production/ensure-high-availability/health-check) + +##### Enhanced Detailed logging +Detailed logging is used in a lot of the cases for debugging issues. Now as well as enabling detailed logging globally (which can cause a huge overhead with lots of traffic), you can enable it for a single key, or specific APIs. + +New detailed logging changes are available only to our Self-Managed customers currently. + +[Read More](/api-management/troubleshooting-debugging#capturing-detailed-logs) + +##### Better Redis failover + +Now, if Redis is not available, Tyk will be more gracefully handle this scenario, and instead of simply timing out the Redis connection, will dynamically disable functionality which depends on redis, like rate limits or quotas, and will re-enable it back once Redis is available. The Tyk Gateway can even be started without Redis, which makes possible scenarios, such as when the Gateway proxies Redis though itself, like in a Redis Sentinel setup. + +##### Ability to shard analytics to different data-sinks + +In a multi-org deployment, each organization, team, or environment might have their preferred analytics tooling. At present, when sending analytics to the Tyk Pump, we do not discriminate analytics by org - meaning that we have to send all analytics to the same database - e.g. MongoDB. Now the Tyk Pump can be configured to send analytics for different organizations to different places. E.g. Org A can send their analytics to MongoDB + DataDog. But Org B can send their analytics to DataDog + expose the Prometheus metrics endpoint. + +It also becomes possible to put a blocklist in-place, meaning that some data sinks can receive information for all orgs, whereas others will not receive OrgA’s analytics if blocked. + +This change requires updating to new Tyk Pump 1.0 + +[Read More](/api-management/tyk-pump#tyk-pump-configuration) + +##### 404 Error logging - unmatched paths + +Concerned that client’s are getting a 404 response? Could it be that the API definition or URL rewrites have been misconfigured? Telling Tyk to track 404 logs, will cause the Tyk Gateway to produce error logs showing that a particular resource has not been found. + +The feature can be enabled by setting the config `track_404_logs` to `true` in the gateway's config file. + +#### Changelog + +##### Fixes + +- Fixed the bug when tokens created with non empty quota, and quota expiration set to `Never`, were treated as having unlimited quota. Now such tokens will stop working, once initial quota is reached. + +#### Updated Versions + +- Tyk Gateway 3.0 +- Tyk Pump 1.0 + +#### Upgrading From Version 2.9 + +No specific actions required. +If you are upgrading from version 2.8, please [read this guide](/developer-support/release-notes/archived#290-release-notes) + +## Further Information + +### Upgrading Tyk +Please refer to the [upgrading Tyk](/developer-support/upgrading) page for further guidance on the upgrade strategy. + +### API Documentation +{/* Required. Update the link to the Gateway "tyk-gateway-api" or dashboard "tyk-dashboard-api" and the Postman collection + +If there were changes in any of Tyk’s API docs: + +- Have API endpoints been documented in the release note summary and changelog? +- Has a link to the endpoint documentation being included? +- Has the benefit of the new/updated endpoint been explained in the release highlights and changelog? */} +- [OpenAPI Document](/tyk-dashboard-api) +- [Postman Collection](https://www.postman.com/tyk-technologies/workspace/tyk-public-workspace/overview) + +### FAQ + +Please visit our [Developer Support](/developer-support/community) page for further information relating to reporting bugs, upgrading Tyk, technical support and how to contribute. diff --git a/developer-support/release-notes/helm-chart.mdx b/developer-support/release-notes/helm-chart.mdx new file mode 100644 index 00000000..aadf26b8 --- /dev/null +++ b/developer-support/release-notes/helm-chart.mdx @@ -0,0 +1,1857 @@ +--- +title: "Tyk Charts Release Notes" +'og:description': "Release notes documenting updates, enhancements and changes for Tyk Charts." +sidebarTitle: "Tyk Chart" +tags: ['Tyk Charts', 'Release notes', 'changelog'] +--- + +****Open Source** ([Mozilla Public License](https://github.com/TykTechnologies/tyk/blob/master/LICENSE.md))** + + +**This page contains all release notes for Tyk Charts displayed in a reverse chronological order** + +## Support Lifetime +Our minor releases are supported until our next minor comes out. + +--- +## 3.0 Release Notes + +### 3.0.0 Release Notes + +#### Release Date 02 April 2025 + +#### Release Highlights + +Tyk Charts 3.0 significantly improves configurability, reliability, and support for Tyk 5.8. This release enhances monitoring capabilities, expands Helm chart flexibility, and resolves key issues related to service availability and configuration management. + +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v3.0.0) below. + +#### Breaking Changes +Tyk Charts 3.0 introduces a breaking configuration changes for Tyk Dashboard: To provide a default secure configuration, `security.forbid_admin_view_access_token` and `security.forbid_admin_reset_access_token` are set to `true` to restrict admin users from being able to view and reset other users' Dashboard API Access Credentials. + +

Dependencies

+##### 3rd Party Dependencies & Tools +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [Kubernetes](https://kubernetes.io) | 1.26.x, 1.27.x, 1.28.x, 1.29.x, 1.30.x, 1.31.x, 1.32.x | 1.19+ | | +| [Helm](https://helm.sh) | 3.14.x | 3.x | | +| [Redis](https://redis.io) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway and Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard, Pump, and MDCB | +| [PostgreSQL](https://www.postgresql.org/download/) | 13.x - 17.x | 13.x - 17.x | Used by Tyk Dashboard, Pump, and MDCB | + +Given the time difference between your upgrade and the release of this version, we recommend customers verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations +There are no deprecation in this release. + +#### Upgrade instructions +You can use helm upgrade to upgrade your release + +```bash +helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ +helm repo update + +helm upgrade [RELEASE_NAME] tyk-helm/[CHART_NAME] +``` + +#### Downloads +- [Source code](https://github.com/TykTechnologies/tyk-charts/archive/refs/tags/v3.0.0.tar.gz) +- [ArtifactHub - tyk-stack](https://artifacthub.io/packages/helm/tyk-helm/tyk-stack/3.0.0) +- [ArtifactHub - tyk-control-plane](https://artifacthub.io/packages/helm/tyk-helm/tyk-control-plane/3.0.0) +- [ArtifactHub - tyk-data-plane](https://artifacthub.io/packages/helm/tyk-helm/tyk-data-plane/3.0.0) +- [ArtifactHub - tyk-oss](https://artifacthub.io/packages/helm/tyk-helm/tyk-oss/3.0.0) + +

Changelog

+##### Added + +
    + +
  • + + +Added readiness and liveness probes using Tyk Pump's health check service, allowing proactive monitoring to detect failures and improve system reliability. + + + +Added support for a global image registry, making it easier for users to configure private registries and streamline container image management. + + + +Added configurable startup probes for Tyk Gateway, improving readiness checks to prevent premature traffic routing during initialization. + + + +Added support for configuring access logs in Tyk Gateway 5.8, allowing users to track API transaction logs for enhanced monitoring and debugging. + + + +Added support for retrieving a single OpenTelemetry Authorization header from a Kubernetes secret, enhancing security by preventing exposure of sensitive credentials in Helm values or Kubernetes manifests. + + + +Replaced hardcoded values in Helm charts with configurable parameters, providing greater flexibility in deployment customization. + + + +Added support for tolerations, affinity, and node selector configurations, enabling users to fine-tune Kubernetes scheduling for better resource allocation and workload distribution. + + + +Added support to enable or disable test pods, allowing users to optimize resource utilization based on their environment needs. + + +
    • + +
+ +##### Changed + +
    + +
  • + +Improved security by setting security.forbid_admin_view_access_token and security.forbid_admin_reset_access_token to true by default, preventing admin users from viewing or resetting other users’ Dashboard API access credentials unless explicitly allowed. + +
    • + +
  • + +Tyk Charts 3.0 will install the following Tyk component versions by default. + + - Tyk Gateway v5.3.10 + - Tyk Dashboard v5.3.10 + - Tyk Pump v1.12.0 + - Tyk MDCB v2.8.0 + - Tyk Developer Portal v1.13.0 + - Tyk Operator v1.2.0 + +
    • + +
+ +##### Fixed + +
    + +
  • + + +Fixed an issue where service annotations for Tyk Pump were not being applied correctly, which caused misconfigurations in Kubernetes deployments. This has been resolved by ensuring annotations are properly processed. + + + +Fixed an issue where the Operator’s liveness and readiness probes were failing, causing the service to enter a CrashLoopBackOff state. This has been resolved by adjusting configurations to ensure proper startup and health checks. + + + +Fixed an issue where TYK_DB_TYKAPI_HOST and TYK_DB_TYKAPI_PORT environment variables were incorrectly set when the Control API was enabled, leading to connectivity issues. This has been resolved by ensuring the correct values are assigned during configuration. + + +
    • + +
+ +{/* ##### Security Fixes +This section should be a bullet point list that should be included when any security fixes have been made in the release, e.g. CVEs. For CVE fixes, consideration needs to be made as follows: +1. Dependency-tracked CVEs - External-tracked CVEs should be included on the release note. +2. Internal scanned CVEs - Refer to the relevant engineering and delivery policy. + +For agreed CVE security fixes, provide a link to the corresponding entry on the NIST website. For example: + +- Fixed the following CVEs: + - [CVE-2022-33082](https://nvd.nist.gov/vuln/detail/CVE-2022-33082) */} + +{/* Required. use 3 hyphens --- between release notes of every patch (minors will be on a separate page) */} +--- +## 2.2 Release Notes + +### 2.2.0 Release Notes + +#### Release Date 09 December 2024 + +#### Release Highlights +{/* Required. Use similar ToV to previous release notes. For example for a patch release: */} + +The Tyk Helm Charts v2.2.0 release brings exciting new features, improvements, and crucial fixes to enhance deployment flexibility, customization, and reliability. Here are the highlights: +* Sidecar containers support +* Dashboard enhancements: Configurable audit log storage, Open Policy Agent (OPA) settings +* Gateway enhancements: Custom liveness and readiness probes, enhanced logging configuration, customizable HPA behavior +* Operator updates: Custom deployment annotations, + +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v2.2.0) below. + +#### Breaking Changes +{/* Required. Use the following statement if there are no breaking changes, or explain if there are */} +This release has no breaking changes. + +{/* The following "Changed error log messages" section is Optional! +Instructions: We should mention in the changelog section ALL changes in our application log messages. In case we made such changes, this section should also be added, to make sure the users don't miss this notice among other changelog lines. */} +{/* ##### Changed error log messages +Important for users who monitor Tyk components using the application logs (i.e. Tyk Gateway log, Tyk Dashboard log etc.). +We try to avoid making changes to our log messages, especially at error and critical levels. However, sometimes it's necessary. Please find the list of changes made to the application log in this release: */} + +{/* The following "|Planned Breaking Changes" section is optional! +Announce future scheduled breaking changes, e.g. Go version updates, DB driver updates etc. +##### Planned Breaking Changes */} + +{/* ##### Dependencies +Required. Use this section to announce the following types of dependencies compatible with the release: + +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. + +3rd party dependencies and tools */} + +{/* ###### Compatibility Matrix For Tyk Components +Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. +| Gateway Version | Recommended Compatibility | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.3 LTS | Helm v2.2 | Helm vX - vY | +| | MDCB v2.5 | MDCB v1.7 - v2.4 | +| | Operator v1.8 | Operator vX - vY | +| | Sync v2.4.1 | Sync vX - vY | +| | | EDP vX - vY | +| | | Pump vX - vY | +| | | TIB vX - vY | */} + +

Dependencies

+##### 3rd Party Dependencies & Tools +{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [Kubernetes](https://kubernetes.io) | 1.26.x, 1.27.x, 1.28.x, 1.29.x, 1.30.x | 1.19+ | | +| [Helm](https://helm.sh) | 3.14.x | 3.x | | +| [Redis](https://redis.io) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway and Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard, Pump, and MDCB | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x | 12.x - 16.x | Used by Tyk Dashboard, Pump, and MDCB | + +Given the time difference between your upgrade and the release of this version, we recommend customers verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations +{/* Required. Use the following statement if there are no deprecations, or explain if there are */} +There are no deprecation in this release. + +{/* Optional section! +Used to share and notify users about our plan to deprecate features, configs etc. +Once you put an item in this section, we must keep this item listed in all the following releases till the deprecation happens +###### Future deprecations. */} + +#### Upgrade instructions +{/* Required. For patches release (Z>0) use this: +For users currently on v2.1.x, we strongly recommend promptly upgrading to the latest release. +
*/} +{/* Go to the [Upgrading Tyk](#upgrading-tyk) section for detailed upgrade Instructions. */} +You can use helm upgrade to upgrade your release + +```bash +helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ +helm repo update + +helm upgrade [RELEASE_NAME] tyk-helm/[CHART_NAME] +``` + +#### Downloads +- [Source code](https://github.com/TykTechnologies/tyk-charts/archive/refs/tags/v2.2.0.tar.gz) +- [ArtifactHub - tyk-stack](https://artifacthub.io/packages/helm/tyk-helm/tyk-stack/2.2.0) +- [ArtifactHub - tyk-control-plane](https://artifacthub.io/packages/helm/tyk-helm/tyk-control-plane/2.2.0) +- [ArtifactHub - tyk-data-plane](https://artifacthub.io/packages/helm/tyk-helm/tyk-data-plane/2.2.0) +- [ArtifactHub - tyk-oss](https://artifacthub.io/packages/helm/tyk-helm/tyk-oss/2.2.0) + +

Changelog

+##### Added + +
    + +
  • + + +User can enable or disable Tyk Streams feature via `global.streaming.enabled`. This option is enabled by default. + + + +Introduced new configuration options to manage audit logging for the Tyk Dashboard. This enhancement allows users to enable, customize, and specify how audit logs are stored and formatted. + +To configure, see [Tyk Stack](/product-stack/tyk-charts/tyk-stack-chart#audit-log-configurations) documentation. + + + +Introduced new options to enable and manage Open Policy Agent (OPA) support directly from the Helm chart. This feature simplifies the configuration process, guiding users to use the correct settings without relying on extraEnvs. + +To configure, see [Tyk Stack](/product-stack/tyk-charts/tyk-stack-chart#opa-configurations) documentation. + + + +Support for configuring liveness and readiness probes for the Tyk Gateway via Helm charts. + +Users can now define custom configurations for these probes, providing more flexibility and control over health checks in Kubernetes deployments. Defaults are provided if custom configurations are not specified. + +This enhancement improves deployment reliability and ensures better integration with Kubernetes health monitoring systems. + +To configure, see [Tyk Stack](/product-stack/tyk-charts/tyk-stack-chart#gateway-probes) documentation. + + + +Support for configuring the Tyk Gateway logging level and format through new fields under `.Values.gateway.log` in the Helm chart values.yaml. + +This enhancement enables fine-tuned control over logging behavior directly from the Helm chart, simplifying deployment customization. + + + +Users can now define custom HPA behavior settings directly in the Helm values file via a new field a new field: `.Values.gateway.autoscaling.behavior`. + +This enhancement provides more flexibility in configuring HPA scaling behavior, allowing tailored performance tuning for Gateway deployments. + + + +Users can now specify annotations directly in the Helm values field `.Values.annotations`, enabling better integration with external tools and systems that rely on metadata annotations. + + + +Support for adding sidecar containers for Tyk components, enhancing flexibility and integration capabilities. This feature allows for the addition of auxiliary containers through `extraContainers` field to the following components: + +- Tyk Gateway +- Tyk Dashboard +- Tyk MDCB +- Tyk Pump +- Tyk Enterprise Developer Portal + + +
    • + +
+ +##### Changed + +
    + +
  • + +Tyk Charts 2.2 will install the following Tyk component versions by default. + + - Tyk Gateway v5.3.8 + - Tyk Dashboard v5.3.8 + - Tyk Pump v1.11.1 + - Tyk MDCB v2.7.2 + - Tyk Developer Portal v1.12.0 + - Tyk Operator v1.1.0 + +
    • + +
+ +##### Fixed + +
    + +
  • + + +Corrected the template name for the OpenTelemetry caFilePath in the Gateway configuration. +Updated template reference from `otel-CAPath` to `otel-tlsCAPath` to ensure proper functionality. +This fix addresses misconfigurations related to the OpenTelemetry TLS CA path and ensures accurate rendering of Gateway templates. + + + +The `externalTrafficPolicy` field is now correctly set under the spec section instead of selectors. +This fix ensures proper functionality and alignment with Kubernetes service configuration requirements. + + + +Resolved an issue where the Tyk OSS chart did not set the Operator license key in the secret created for the Operator. This fix ensures seamless configuration of the license key when deploying Tyk Operator. + + +
    • + +
+ +{/* ##### Security Fixes +This section should be a bullet point list that should be included when any security fixes have been made in the release, e.g. CVEs. For CVE fixes, consideration needs to be made as follows: +1. Dependency-tracked CVEs - External-tracked CVEs should be included on the release note. +2. Internal scanned CVEs - Refer to the relevant engineering and delivery policy. + +For agreed CVE security fixes, provide a link to the corresponding entry on the NIST website. For example: + +- Fixed the following CVEs: + - [CVE-2022-33082](https://nvd.nist.gov/vuln/detail/CVE-2022-33082) */} + +{/* Required. use 3 hyphens --- between release notes of every patch (minors will be on a separate page) */} +--- + +{/* Repeat the release notes section above for every patch here */} + +{/* The footer of the release notes page. It contains a further information section with details of how to upgrade Tyk, +links to API documentation and FAQs. You can copy it from the previous release. */} + +## 2.1 Release Notes +### 2.1.0 Release Notes + +#### Release Date 10 Oct 2024 + +#### Release Highlights +{/* Required. Use similar ToV to previous release notes. For example for a patch release: */} + +Added the ability to specify a static IP for Kubernetes LoadBalancer services, giving users more control over network configurations for the Tyk Gateway and Dashboard. Added an option to configure the Dashboard container port, addressing issues with restricted port permissions. Updated the default versions of Tyk components. + +For a comprehensive list of changes, please refer to the detailed [changelog](#Changelog-v2.1.0) below. + +#### Breaking Changes +{/* Required. Use the following statement if there are no breaking changes, or explain if there are */} +This release has no breaking changes. + +However, if you are upgrading to [Tyk Operator v1.0](/developer-support/release-notes/operator#100-release-notes) using the Helm Chart, please read the [license requirement](/developer-support/release-notes/operator#breaking-changesv1.0.0) and Tyk Operator [installation and upgrade instructions](/api-management/automations/operator#install-and-configure-tyk-operator) carefully. + +{/* The following "Changed error log messages" section is Optional! +Instructions: We should mention in the changelog section ALL changes in our application log messages. In case we made such changes, this section should also be added, to make sure the users don't miss this notice among other changelog lines. */} +{/* ##### Changed error log messages +Important for users who monitor Tyk components using the application logs (i.e. Tyk Gateway log, Tyk Dashboard log etc.). +We try to avoid making changes to our log messages, especially at error and critical levels. However, sometimes it's necessary. Please find the list of changes made to the application log in this release: */} + +{/* The following "|Planned Breaking Changes" section is optional! +Announce future scheduled breaking changes, e.g. Go version updates, DB driver updates etc. +##### Planned Breaking Changes */} + +{/* ##### Dependencies +Required. Use this section to announce the following types of dependencies compatible with the release: + +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. + +3rd party dependencies and tools */} + +{/* ###### Compatibility Matrix For Tyk Components +Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. +| Gateway Version | Recommended Compatibility | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.3 LTS | Helm v2.2 | Helm vX - vY | +| | MDCB v2.5 | MDCB v1.7 - v2.4 | +| | Operator v1.8 | Operator vX - vY | +| | Sync v2.4.1 | Sync vX - vY | +| | | EDP vX - vY | +| | | Pump vX - vY | +| | | TIB vX - vY | */} + +

Dependencies

+##### 3rd Party Dependencies & Tools +{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [Kubernetes](https://kubernetes.io) | 1.26.x, 1.27.x, 1.28.x, 1.29.x, 1.30.x | 1.19+ | | +| [Helm](https://helm.sh) | 3.14.x | 3.x | | +| [Redis](https://redis.io) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway and Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard, Pump, and MDCB | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x | 12.x - 16.x | Used by Tyk Dashboard, Pump, and MDCB | + +Given the time difference between your upgrade and the release of this version, we recommend customers verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations +{/* Required. Use the following statement if there are no deprecations, or explain if there are */} +There are no deprecation in this release. + +{/* Optional section! +Used to share and notify users about our plan to deprecate features, configs etc. +Once you put an item in this section, we must keep this item listed in all the following releases till the deprecation happens +###### Future deprecations. */} + +#### Upgrade instructions +{/* Required. For patches release (Z>0) use this: */} +For users currently on v2.0.x, we strongly recommend promptly upgrading to the latest release. +
+{/* Go to the [Upgrading Tyk](#upgrading-tyk) section for detailed upgrade Instructions. */} +You can use helm upgrade to upgrade your release + +```bash +helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ +helm repo update + +helm upgrade [RELEASE_NAME] tyk-helm/[CHART_NAME] +``` + +#### Downloads +- [Source code](https://github.com/TykTechnologies/tyk-charts/archive/refs/tags/v2.1.0.tar.gz) +- [ArtifactHub - tyk-stack](https://artifacthub.io/packages/helm/tyk-helm/tyk-stack/2.1.0) +- [ArtifactHub - tyk-control-plane](https://artifacthub.io/packages/helm/tyk-helm/tyk-control-plane/2.1.0) +- [ArtifactHub - tyk-data-plane](https://artifacthub.io/packages/helm/tyk-helm/tyk-data-plane/2.1.0) +- [ArtifactHub - tyk-oss](https://artifacthub.io/packages/helm/tyk-helm/tyk-oss/2.1.0) + +

Changelog

+##### Added + +
    + +
  • + + +Added an optional `loadBalancerIP` parameter in the chart that allows users to set a static IP for Tyk Gateway and Dashboard services when using the `LoadBalancer` service type. This update provides enhanced control over IP configuration, useful for network stability in environments with multiple load balancers. + +Tyk gateway service configuration: +- `tyk-gateway.gateway.service.loadBalancerIP` (default to "") + +Tyk Dashboard service configuration: +- `tyk-dashboard.dashboard.service.loadBalancerIP` (default to "") + + + +Enables specifying an alternate port for the container while using standard ports in the service. This option resolves permission issues associated with restricted ports, such as port 443, within containers. + + + +Starting from Tyk Operator v1.0, a license key is required to use the Tyk Operator. You can provide it while installing Tyk Stack, Tyk Control Plane, or Tyk OSS helm chart by setting `global.license.operator` field. You can also set license key via a Kubernetes secret using `global.secrets.useSecretName` field. The secret should contain a key called `OperatorLicense`. + + +
    • + +
+ +##### Changed + +
    + +
  • + +Tyk Charts 2.1 will install the following Tyk component versions by default. +- Tyk Gateway v5.3.6 +- Tyk Dashboard v5.3.6 +- Tyk Pump v1.11.0 +- Tyk MDCB v2.7.1 +- Tyk Developer Portal v1.10.0 +- Tyk Operator v1.0.0 + +
    • + +
+ +{/* ##### Security Fixes +This section should be a bullet point list that should be included when any security fixes have been made in the release, e.g. CVEs. For CVE fixes, consideration needs to be made as follows: +1. Dependency-tracked CVEs - External-tracked CVEs should be included on the release note. +2. Internal scanned CVEs - Refer to the relevant engineering and delivery policy. + +For agreed CVE security fixes, provide a link to the corresponding entry on the NIST website. For example: + +- Fixed the following CVEs: + - [CVE-2022-33082](https://nvd.nist.gov/vuln/detail/CVE-2022-33082) */} + +{/* Required. use 3 hyphens --- between release notes of every patch (minors will be on a separate page) */} +--- + +{/* Repeat the release notes section above for every patch here */} + +{/* The footer of the release notes page. It contains a further information section with details of how to upgrade Tyk, +links to API documentation and FAQs. You can copy it from the previous release. */} + +## 2.0 Release Notes +### 2.0.0 Release Notes + +#### Release Date 26 September 2024 + +#### Breaking Changes + +##### 1. URL Path Matching Configuration Changes + +Tyk Charts v2.0 introduces a **breaking change** related to URL path matching behavior in the Tyk Gateway. If you are using Tyk Gateway versions 5.0.14 (2023 LTS), 5.3.5 (2024 LTS), or 5.5.1 (latest feature branch) or above, two new configuration options have been added to the Gateway: + +- `http_server_options.enable_path_prefix_matching` +- `http_server_options.enable_path_suffix_matching` + +These options allow more restrictive URL path matching by controlling whether the request path matches the start or end of the specified pattern. If both are set to `true`, Tyk enforces "exact" path matching. By default, these options are set to `false` in the Gateway to avoid breaking existing configurations. + +However, starting with **Tyk Charts v2.0**, these options will be set to `true` by default, enforcing stricter security by requiring precise path matches. This change applies to new installations or upgrades via Tyk Charts v2.0 and above. + +From this version of Tyk Charts we also set the following configuration option to `true` by default as part of the stricter path matching: + +- `http_server_options.enable_strict_routes` + +**Impact on existing users:** + +- The change is **backward-compatible** for users upgrading their Tyk Gateway directly (i.e. not via Helm Chart), because by default, these features will not be active. This ensures that existing configurations are not affected if you update the Gateway manually. +- However, **if you install or upgrade via Tyk Charts v2.0**, these options will be set to `true` by default. This means stricter URL path matching will be enforced automatically, which could impact your existing routes or configurations if you're not prepared for it. Please ensure you understand and test these new configurations before upgrading your production environment. + +**Action required:** + +- Familiarize yourself with URL matching in Tyk [here](/getting-started/key-concepts/url-matching). +- For production setup guidance, see [this guide](/planning-for-production#ensure-you-are-matching-only-the-url-paths-that-you-want-to-match). +- Configure the new options via the Helm chart, and test the changes in a non-production environment before upgrading. + +##### 2. Default Tyk Component Versions + +This release changes the default component versions in Tyk Charts v2.0 to **Long-Term Support (LTS)** versions for greater stability in production environments. The new defaults are: + +|Tyk Component|Default Version|Customization Parameter| +| :--- | :--- | :--- | +|Tyk Gateway|5.3.5 LTS|`--set tyk-gateway.gateway.image.tag=`| +|Tyk Dashboard|5.3.5 LTS|`--set tyk-dashboard.dashboard.image.tag=`| +|Tyk Pump|1.11.0|`--set tyk-pump.pump.image.tag=`| +|Tyk MDCB|2.7.0|`--set tyk-mdcb.mdcb.image.tag=`| +|Tyk Developer Portal|1.10.0|`--set tyk-dev-portal.image.tag=`| +|Tyk Operator|0.18.0|`--set tyk-operator.image.tag=`| + +If you need to use a different version for any component, adjust the Helm chart parameters during installation or upgrade. + +{/* The following "Changed error log messages" section is Optional! +Instructions: We should mention in the changelog section ALL changes in our application log messages. In case we made such changes, this section should also be added, to make sure the users don't miss this notice among other changelog lines. */} +{/* ##### Changed error log messages +Important for users who monitor Tyk components using the application logs (i.e. Tyk Gateway log, Tyk Dashboard log etc.). +We try to avoid making changes to our log messages, especially at error and critical levels. However, sometimes it's necessary. Please find the list of changes made to the application log in this release: */} + +{/* The following "|Planned Breaking Changes" section is optional! +Announce future scheduled breaking changes, e.g. Go version updates, DB driver updates etc. +##### Planned Breaking Changes */} + +{/* ##### Dependencies +Required. Use this section to announce the following types of dependencies compatible with the release: + +Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. + +3rd party dependencies and tools */} + +{/* ###### Compatibility Matrix For Tyk Components +Required. Version compatibility with other components in the Tyk stack. This takes the form of a compatibility matrix and is only required for Gateway and Portal. +An illustrative example is shown below. +| Gateway Version | Recommended Compatibility | Backwards Compatibility | +| :---- | :---- | :---- | +| 5.3 LTS | Helm v2.2 | Helm vX - vY | +| | MDCB v2.5 | MDCB v1.7 - v2.4 | +| | Operator v1.8 | Operator vX - vY | +| | Sync v2.4.1 | Sync vX - vY | +| | | EDP vX - vY | +| | | Pump vX - vY | +| | | TIB vX - vY | */} + +#### 3rd Party Dependencies & Tools +{/* Required. Third-party dependencies encompass tools (GoLang, Helm etc.), databases (PostgreSQL, MongoDB etc.) and external software libraries. This section should be a table that presents the third-party dependencies and tools compatible with the release. Compatible is used in the sense of those versions tested with the releases. Such information assists customers considering upgrading to a specific release. + +Additionally, a disclaimer statement was added below the table, for customers to check that the third-party dependency they decide to install remains in support. + +An example is given below for illustrative purposes only. Tested Versions and Compatible Versions information will require discussion with relevant squads and QA. */} + +| Third Party Dependency | Tested Versions | Compatible Versions | Comments | +| :---------------------------------------------------------- | :---------------------- | :---------------------- | :-------- | +| [Kubernetes](https://kubernetes.io) | 1.26.x, 1.27.x, 1.28.x, 1.29.x, 1.30.x | 1.19+ | | +| [Helm](https://helm.sh) | 3.14.x | 3.x | | +| [Redis](https://redis.io) | 6.2.x, 7.x | 6.2.x, 7.x | Used by Tyk Gateway and Dashboard | +| [MongoDB](https://www.mongodb.com/try/download/community) | 5.0.x, 6.0.x, 7.0.x | 5.0.x, 6.0.x, 7.0.x | Used by Tyk Dashboard, Pump, and MDCB | +| [PostgreSQL](https://www.postgresql.org/download/) | 12.x - 16.x | 12.x - 16.x | Used by Tyk Dashboard, Pump, and MDCB | + +Given the time difference between your upgrade and the release of this version, we recommend customers verify the ongoing support of third-party dependencies they install, as their status may have changed since the release. + +#### Deprecations +{/* Required. Use the following statement if there are no deprecations, or explain if there are */} +There are no deprecation in this release. + +{/* Optional section! +Used to share and notify users about our plan to deprecate features, configs etc. +Once you put an item in this section, we must keep this item listed in all the following releases till the deprecation happens +###### Future deprecations. */} + +#### Upgrade instructions +{/* Required. For patches release (Z>0) use this: */} +For users currently on v1.x.x, we strongly recommend promptly upgrading to the latest release. +
+{/* Go to the [Upgrading Tyk](#upgrading-tyk) section for detailed upgrade Instructions. */} +You can use helm upgrade to upgrade your release + +```bash +helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ +helm repo update + +helm upgrade [RELEASE_NAME] tyk-helm/[CHART_NAME] +``` + +#### Release Highlights +{/* Required. Use similar ToV to previous release notes. For example for a patch release: */} + +##### Support Gateway configuration for URL path matching +The default Gateway configuration in the Helm chart will set Tyk's URL path matching to **exact** mode. This ensures that the request URL must exactly match the listen path and endpoint patterns configured in the API definition. + +##### Updated default Tyk versions +Tyk Charts 2.0 will install the following Tyk component versions by default. +- Tyk Gateway v5.3.5 +- Tyk Dashboard v5.3.5 +- Tyk Pump v1.11.0 +- Tyk MDCB v2.7.0 +- Tyk Developer Portal v1.10.0 +- Tyk Operator v0.18.0 + +#### Downloads +- [Source code](https://github.com/TykTechnologies/tyk-charts/archive/refs/tags/v2.0.0.tar.gz) +- [ArtifactHub - tyk-stack](https://artifacthub.io/packages/helm/tyk-helm/tyk-stack/2.0.0) +- [ArtifactHub - tyk-control-plane](https://artifacthub.io/packages/helm/tyk-helm/tyk-control-plane/2.0.0) +- [ArtifactHub - tyk-data-plane](https://artifacthub.io/packages/helm/tyk-helm/tyk-data-plane/2.0.0) +- [ArtifactHub - tyk-oss](https://artifacthub.io/packages/helm/tyk-helm/tyk-oss/2.0.0) + +

Changelog

+{/* Required. The change log should include the following ordered set of sections below that briefly summarise the features, updates and fixed issues of the release. + +Here it is important to explain the benefit of each changelog item. As mentioned by James in a previous Slack message (https://tyktech.slack.com/archives/C044R3ZTN6L/p1686812207060839?thread_ts=1686762128.651249&cid=C044R3ZTN6L): +"...it is important to document the customer impact for the work delivered, so we can share it with prospects/install base. For example: +"New Chart delivers x and y benefit to a and b customer use cases. The business impact for them will be this and that" */} + +##### Added +{/* This section should be a bullet point list of new features. Explain: + +- The purpose of the new feature +- How does the new feature benefit users? +- Link to documentation of the new feature +- For OSS - Link to the corresponding issue if possible on GitHub to allow the users to see further info. + +Each change log item should be expandable. The first line summarises the changelog entry. It should be then possible to expand this to reveal further details about the changelog item. This is achieved using HTML as shown in the example below. */} + +
    + +
  • + +Tyk Charts v2.0 introduces support for the newly added Tyk Gateway configuration options: `enable_path_prefix_matching` and `enable_path_suffix_matching`. These settings allow more secure and explicit URL matching by restricting path pattern matching to the start or end of the request path. This enhancement benefits customers who need more precise route matching to ensure that only intended paths are matched in production environments, reducing the risk of unintentional routing. + +URL path matching mode is configurable using these `tyk-gateway` chart parameters: + +- `gateway.enablePathPrefixMatching` (default to `true`) +- `gateway.enablePathSuffixMatching` (default to `true`) +- `gateway.enableStrictRoutes` (default to `true`) + +Learn more about the settings in the [URL Path Matching](/getting-started/key-concepts/url-matching) documentation. + +
    • + +
  • + +This release adds support for `extraVolumes` and `extraVolumeMounts` parameters in the `tyk-bootstrap` charts, enabling users to mount additional volumes. This is especially useful for users with custom storage or configuration needs in their deployments, offering more flexibility in managing their Tyk installation. + +The options are configurable using these `tyk-bootstrap` chart's parameters: + +- `bootstrap.extraVolumes` (default to empty list) +- `bootstrap.extraVolumeMounts` (default to empty list) + +
    • + +
+ +{/* ###### Changed + + + + +Object + + +Node1 + + +Object ++ +HookType hook_type ++ +string hook_name ++ +map< string, string + > metadata ++ +map< string, string + > spec +Β  + + + + + + + + + +Node2 + + +MiniRequestObject ++ +map< string, string + > headers ++ +map< string, string + > set_headers ++ +repeated string delete +_headers ++ +string body ++ +string url ++ +map< string, string + > params ++ +map< string, string + > add_params ++ +map< string, string + > extended_params ++ +repeated string delete +_params ++ +string method ++ +string request_uri ++ +string scheme ++ +bytes raw_body +Β  + + + + + + + + + +Node2->Node1 + + + + + + +request + + + +Node3 + + +ReturnOverrides ++ +int32 response_code ++ +string response_error ++ +map< string, string + > headers ++ +bool override_error ++ +string response_body +Β  + + + + + + + + + +Node3->Node2 + + + + + + +return_overrides + + + +Node4 + + +SessionState ++ +int64 last_check ++ +double allowance ++ +double rate ++ +double per ++ +int64 expires ++ +int64 quota_max ++ +int64 quota_renews ++ +int64 quota_remaining ++ +int64 quota_renewal_rate ++ +map< string, AccessDefinition + > access_rights +and 18 more... +Β  + + + + + + + + + +Node4->Node1 + + + + + + +session + + + +Node5 + + +BasicAuthData ++ +string password ++ +string hash +Β  + + + + + + + + + +Node5->Node4 + + + + + + +basic_auth_data + + + +Node6 + + +JWTData ++ +string secret +Β  + + + + + + + + + +Node6->Node4 + + + + + + +jwt_data + + + +Node7 + + +Monitor ++ +repeated double trigger +_limits +Β  + + + + + + + + + +Node7->Node4 + + + + + + +monitor + + + +Node8 + + +ResponseObject ++ +int32 status_code ++ +bytes raw_body ++ +string body ++ +map< string, string + > headers +Β  + + + + + + + + + +Node8->Node1 + + + + + + +response + + + +Node9 + + +Header ++ +string key ++ +repeated string values +Β  + + + + + + + + + +Node9->Node8 + + + + + + +multivalue_headers + + + diff --git a/img/heroku-logo.png b/img/heroku-logo.png new file mode 100644 index 00000000..a9db646a Binary files /dev/null and b/img/heroku-logo.png differ diff --git a/img/home-page-review-icon.svg b/img/home-page-review-icon.svg new file mode 100644 index 00000000..a71d70fd --- /dev/null +++ b/img/home-page-review-icon.svg @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/img/home_page_free_trial_icon.svg b/img/home_page_free_trial_icon.svg new file mode 100644 index 00000000..1c762e62 --- /dev/null +++ b/img/home_page_free_trial_icon.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/img/hybrid-gateway/tyk-cloud-hybrid-configuration-home.png b/img/hybrid-gateway/tyk-cloud-hybrid-configuration-home.png new file mode 100644 index 00000000..7578ecd5 Binary files /dev/null and b/img/hybrid-gateway/tyk-cloud-hybrid-configuration-home.png differ diff --git a/img/hybrid-gateway/tyk-cloud-hybrid-confirm-config-removal.png b/img/hybrid-gateway/tyk-cloud-hybrid-confirm-config-removal.png new file mode 100644 index 00000000..1f2aa51a Binary files /dev/null and b/img/hybrid-gateway/tyk-cloud-hybrid-confirm-config-removal.png differ diff --git a/img/hybrid-gateway/tyk-cloud-hybrid-masked-details.png b/img/hybrid-gateway/tyk-cloud-hybrid-masked-details.png new file mode 100644 index 00000000..0785c66e Binary files /dev/null and b/img/hybrid-gateway/tyk-cloud-hybrid-masked-details.png differ diff --git a/img/hybrid-gateway/tyk-cloud-hybrid-open-details.png b/img/hybrid-gateway/tyk-cloud-hybrid-open-details.png new file mode 100644 index 00000000..553276ce Binary files /dev/null and b/img/hybrid-gateway/tyk-cloud-hybrid-open-details.png differ diff --git a/img/hybrid-gateway/tyk-cloud-hybrid-remove-configs.png b/img/hybrid-gateway/tyk-cloud-hybrid-remove-configs.png new file mode 100644 index 00000000..00e2de57 Binary files /dev/null and b/img/hybrid-gateway/tyk-cloud-hybrid-remove-configs.png differ diff --git a/img/hybrid-gateway/tyk-cloud-hybrid-revealed-instructions.png b/img/hybrid-gateway/tyk-cloud-hybrid-revealed-instructions.png new file mode 100644 index 00000000..43c7696c Binary files /dev/null and b/img/hybrid-gateway/tyk-cloud-hybrid-revealed-instructions.png differ diff --git a/img/hybrid-gateway/tyk-cloud-save-hybrid-configuration.png b/img/hybrid-gateway/tyk-cloud-save-hybrid-configuration.png new file mode 100644 index 00000000..332211bb Binary files /dev/null and b/img/hybrid-gateway/tyk-cloud-save-hybrid-configuration.png differ diff --git a/img/k8s.png b/img/k8s.png new file mode 100644 index 00000000..437c0d00 Binary files /dev/null and b/img/k8s.png differ diff --git a/img/keycloak-jwt/client-secret.png b/img/keycloak-jwt/client-secret.png new file mode 100644 index 00000000..10cdb8aa Binary files /dev/null and b/img/keycloak-jwt/client-secret.png differ diff --git a/img/keycloak-jwt/client.png b/img/keycloak-jwt/client.png new file mode 100644 index 00000000..ed88cf44 Binary files /dev/null and b/img/keycloak-jwt/client.png differ diff --git a/img/keycloak-jwt/create-client-step-1.png b/img/keycloak-jwt/create-client-step-1.png new file mode 100644 index 00000000..93bfc62b Binary files /dev/null and b/img/keycloak-jwt/create-client-step-1.png differ diff --git a/img/keycloak-jwt/create-client-step-2.png b/img/keycloak-jwt/create-client-step-2.png new file mode 100644 index 00000000..af8075b3 Binary files /dev/null and b/img/keycloak-jwt/create-client-step-2.png differ diff --git a/img/keycloak-jwt/create-client-step-3.png b/img/keycloak-jwt/create-client-step-3.png new file mode 100644 index 00000000..304f7bdd Binary files /dev/null and b/img/keycloak-jwt/create-client-step-3.png differ diff --git a/img/keycloak-jwt/create-client.png b/img/keycloak-jwt/create-client.png new file mode 100644 index 00000000..dc61d699 Binary files /dev/null and b/img/keycloak-jwt/create-client.png differ diff --git a/img/keycloak-jwt/create-jwt-realm.png b/img/keycloak-jwt/create-jwt-realm.png new file mode 100644 index 00000000..5d226612 Binary files /dev/null and b/img/keycloak-jwt/create-jwt-realm.png differ diff --git a/img/keycloak-jwt/navigate-to-admin-console.png b/img/keycloak-jwt/navigate-to-admin-console.png new file mode 100644 index 00000000..b67453c0 Binary files /dev/null and b/img/keycloak-jwt/navigate-to-admin-console.png differ diff --git a/img/keycloak-sso/add-redirectUrl-to-client.png b/img/keycloak-sso/add-redirectUrl-to-client.png new file mode 100644 index 00000000..3e27474f Binary files /dev/null and b/img/keycloak-sso/add-redirectUrl-to-client.png differ diff --git a/img/keycloak-sso/copy-callback-url.png b/img/keycloak-sso/copy-callback-url.png new file mode 100644 index 00000000..0cf05209 Binary files /dev/null and b/img/keycloak-sso/copy-callback-url.png differ diff --git a/img/keycloak-sso/create-client.png b/img/keycloak-sso/create-client.png new file mode 100644 index 00000000..fbed8f08 Binary files /dev/null and b/img/keycloak-sso/create-client.png differ diff --git a/img/keycloak-sso/create-client2.png b/img/keycloak-sso/create-client2.png new file mode 100644 index 00000000..649d5886 Binary files /dev/null and b/img/keycloak-sso/create-client2.png differ diff --git a/img/keycloak-sso/create-profile.png b/img/keycloak-sso/create-profile.png new file mode 100644 index 00000000..27c9c72d Binary files /dev/null and b/img/keycloak-sso/create-profile.png differ diff --git a/img/keycloak-sso/enable-client-auth.png b/img/keycloak-sso/enable-client-auth.png new file mode 100644 index 00000000..90ae725e Binary files /dev/null and b/img/keycloak-sso/enable-client-auth.png differ diff --git a/img/keycloak-sso/identity-management.png b/img/keycloak-sso/identity-management.png new file mode 100644 index 00000000..349d2474 Binary files /dev/null and b/img/keycloak-sso/identity-management.png differ diff --git a/img/keycloak-sso/keycloak-login.png b/img/keycloak-sso/keycloak-login.png new file mode 100644 index 00000000..06488854 Binary files /dev/null and b/img/keycloak-sso/keycloak-login.png differ diff --git a/img/keycloak-sso/logged-in.png b/img/keycloak-sso/logged-in.png new file mode 100644 index 00000000..1eb1bc73 Binary files /dev/null and b/img/keycloak-sso/logged-in.png differ diff --git a/img/keycloak-sso/login-url.png b/img/keycloak-sso/login-url.png new file mode 100644 index 00000000..d90d1173 Binary files /dev/null and b/img/keycloak-sso/login-url.png differ diff --git a/img/keycloak-sso/realm-discovery-endpoint.png b/img/keycloak-sso/realm-discovery-endpoint.png new file mode 100644 index 00000000..7d1895aa Binary files /dev/null and b/img/keycloak-sso/realm-discovery-endpoint.png differ diff --git a/img/keycloak-sso/retrieve-client-secret.png b/img/keycloak-sso/retrieve-client-secret.png new file mode 100644 index 00000000..e97710d7 Binary files /dev/null and b/img/keycloak-sso/retrieve-client-secret.png differ diff --git a/img/keycloak-sso/set-provider-type.png b/img/keycloak-sso/set-provider-type.png new file mode 100644 index 00000000..727bef85 Binary files /dev/null and b/img/keycloak-sso/set-provider-type.png differ diff --git a/img/logos/Postman-logo-vertical-orange-2021.svg b/img/logos/Postman-logo-vertical-orange-2021.svg new file mode 100644 index 00000000..251b9e4d --- /dev/null +++ b/img/logos/Postman-logo-vertical-orange-2021.svg @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/img/logos/tyk-docs-logo-dark.svg b/img/logos/tyk-docs-logo-dark.svg new file mode 100644 index 00000000..c255dc18 --- /dev/null +++ b/img/logos/tyk-docs-logo-dark.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/img/logos/tyk-logo-opensource.svg b/img/logos/tyk-logo-opensource.svg new file mode 100644 index 00000000..9531901b --- /dev/null +++ b/img/logos/tyk-logo-opensource.svg @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/img/logos/tyk-logo-selfmanaged.svg b/img/logos/tyk-logo-selfmanaged.svg new file mode 100644 index 00000000..7ec4b896 --- /dev/null +++ b/img/logos/tyk-logo-selfmanaged.svg @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/img/mdcb/mdcb-acme-global-bank1.png b/img/mdcb/mdcb-acme-global-bank1.png new file mode 100644 index 00000000..28fdb90a Binary files /dev/null and b/img/mdcb/mdcb-acme-global-bank1.png differ diff --git a/img/mdcb/mdcb-acme-global-bank2.png b/img/mdcb/mdcb-acme-global-bank2.png new file mode 100644 index 00000000..0aecd13a Binary files /dev/null and b/img/mdcb/mdcb-acme-global-bank2.png differ diff --git a/img/mdcb/mdcb-acme-telecoms1.png b/img/mdcb/mdcb-acme-telecoms1.png new file mode 100644 index 00000000..46655627 Binary files /dev/null and b/img/mdcb/mdcb-acme-telecoms1.png differ diff --git a/img/mdcb/mdcb-acme-telecoms2.png b/img/mdcb/mdcb-acme-telecoms2.png new file mode 100644 index 00000000..d43ff761 Binary files /dev/null and b/img/mdcb/mdcb-acme-telecoms2.png differ diff --git a/img/mdcb/mdcb-components.png b/img/mdcb/mdcb-components.png new file mode 100644 index 00000000..dfa5c6f0 Binary files /dev/null and b/img/mdcb/mdcb-components.png differ diff --git a/img/mdcb/mdcb-control-plane.png b/img/mdcb/mdcb-control-plane.png new file mode 100644 index 00000000..27d34fba Binary files /dev/null and b/img/mdcb/mdcb-control-plane.png differ diff --git a/img/mdcb/mdcb-data-plane.png b/img/mdcb/mdcb-data-plane.png new file mode 100644 index 00000000..bfcb8921 Binary files /dev/null and b/img/mdcb/mdcb-data-plane.png differ diff --git a/img/mdcb/mdcb-intro1.png b/img/mdcb/mdcb-intro1.png new file mode 100644 index 00000000..b83399a6 Binary files /dev/null and b/img/mdcb/mdcb-intro1.png differ diff --git a/img/mdcb/mdcb-intro2.png b/img/mdcb/mdcb-intro2.png new file mode 100644 index 00000000..9dfe012a Binary files /dev/null and b/img/mdcb/mdcb-intro2.png differ diff --git a/img/mdcb/mdcb-intro3.png b/img/mdcb/mdcb-intro3.png new file mode 100644 index 00000000..0d224ee6 Binary files /dev/null and b/img/mdcb/mdcb-intro3.png differ diff --git a/img/mdcb/mdcb-intro4.png b/img/mdcb/mdcb-intro4.png new file mode 100644 index 00000000..77f6c4c4 Binary files /dev/null and b/img/mdcb/mdcb-intro4.png differ diff --git a/img/mdcb/mdcb-poc1-screenshot1.png b/img/mdcb/mdcb-poc1-screenshot1.png new file mode 100644 index 00000000..a95b7fe7 Binary files /dev/null and b/img/mdcb/mdcb-poc1-screenshot1.png differ diff --git a/img/mdcb/mdcb-poc1-screenshot2.png b/img/mdcb/mdcb-poc1-screenshot2.png new file mode 100644 index 00000000..f3f22da9 Binary files /dev/null and b/img/mdcb/mdcb-poc1-screenshot2.png differ diff --git a/img/mdcb/mdcb-poc1-screenshot3.png b/img/mdcb/mdcb-poc1-screenshot3.png new file mode 100644 index 00000000..46e700de Binary files /dev/null and b/img/mdcb/mdcb-poc1-screenshot3.png differ diff --git a/img/mdcb/mdcb-poc1-screenshot4.png b/img/mdcb/mdcb-poc1-screenshot4.png new file mode 100644 index 00000000..f6d4f087 Binary files /dev/null and b/img/mdcb/mdcb-poc1-screenshot4.png differ diff --git a/img/mdcb/mdcb-poc1-screenshot5.png b/img/mdcb/mdcb-poc1-screenshot5.png new file mode 100644 index 00000000..d22219a3 Binary files /dev/null and b/img/mdcb/mdcb-poc1-screenshot5.png differ diff --git a/img/mdcb/mdcb-poc1-screenshot6.png b/img/mdcb/mdcb-poc1-screenshot6.png new file mode 100644 index 00000000..0582da5b Binary files /dev/null and b/img/mdcb/mdcb-poc1-screenshot6.png differ diff --git a/img/mdcb/mdcb-poc1-screenshot7.png b/img/mdcb/mdcb-poc1-screenshot7.png new file mode 100644 index 00000000..5b0a2443 Binary files /dev/null and b/img/mdcb/mdcb-poc1-screenshot7.png differ diff --git a/img/mdcb/mdcb-poc1-screenshot8.png b/img/mdcb/mdcb-poc1-screenshot8.png new file mode 100644 index 00000000..97a31e1c Binary files /dev/null and b/img/mdcb/mdcb-poc1-screenshot8.png differ diff --git a/img/mdcb/mdcb_poc1_overview.png b/img/mdcb/mdcb_poc1_overview.png new file mode 100644 index 00000000..d30f7be2 Binary files /dev/null and b/img/mdcb/mdcb_poc1_overview.png differ diff --git a/img/mdcb/synchroniser-after.gif b/img/mdcb/synchroniser-after.gif new file mode 100644 index 00000000..1e5b569d Binary files /dev/null and b/img/mdcb/synchroniser-after.gif differ diff --git a/img/mdcb/synchroniser-before.gif b/img/mdcb/synchroniser-before.gif new file mode 100644 index 00000000..54001389 Binary files /dev/null and b/img/mdcb/synchroniser-before.gif differ diff --git a/img/oas/add-new-api.png b/img/oas/add-new-api.png new file mode 100644 index 00000000..10e9a1f9 Binary files /dev/null and b/img/oas/add-new-api.png differ diff --git a/img/oas/api-actions-menu.png b/img/oas/api-actions-menu.png new file mode 100644 index 00000000..f9e62b32 Binary files /dev/null and b/img/oas/api-actions-menu.png differ diff --git a/img/oas/api-create-deploy-targets.png b/img/oas/api-create-deploy-targets.png new file mode 100644 index 00000000..a8e6be71 Binary files /dev/null and b/img/oas/api-create-deploy-targets.png differ diff --git a/img/oas/api-create-endpoint-list.png b/img/oas/api-create-endpoint-list.png new file mode 100644 index 00000000..ed1829ee Binary files /dev/null and b/img/oas/api-create-endpoint-list.png differ diff --git a/img/oas/api-create-endpoint.png b/img/oas/api-create-endpoint.png new file mode 100644 index 00000000..acea4dd8 Binary files /dev/null and b/img/oas/api-create-endpoint.png differ diff --git a/img/oas/api-create-import-auto-options.png b/img/oas/api-create-import-auto-options.png new file mode 100644 index 00000000..9ef4e3c7 Binary files /dev/null and b/img/oas/api-create-import-auto-options.png differ diff --git a/img/oas/api-create-import-location.png b/img/oas/api-create-import-location.png new file mode 100644 index 00000000..f0d1f5fc Binary files /dev/null and b/img/oas/api-create-import-location.png differ diff --git a/img/oas/api-create-import-manual-options.png b/img/oas/api-create-import-manual-options.png new file mode 100644 index 00000000..1e291dc5 Binary files /dev/null and b/img/oas/api-create-import-manual-options.png differ diff --git a/img/oas/api-create-import-template.png b/img/oas/api-create-import-template.png new file mode 100644 index 00000000..cd0a8efb Binary files /dev/null and b/img/oas/api-create-import-template.png differ diff --git a/img/oas/api-create-import-tykapi.png b/img/oas/api-create-import-tykapi.png new file mode 100644 index 00000000..021d5236 Binary files /dev/null and b/img/oas/api-create-import-tykapi.png differ diff --git a/img/oas/api-create-import.png b/img/oas/api-create-import.png new file mode 100644 index 00000000..31139d1a Binary files /dev/null and b/img/oas/api-create-import.png differ diff --git a/img/oas/api-create-no-endpoints.png b/img/oas/api-create-no-endpoints.png new file mode 100644 index 00000000..35afff65 Binary files /dev/null and b/img/oas/api-create-no-endpoints.png differ diff --git a/img/oas/api-create-set-status.png b/img/oas/api-create-set-status.png new file mode 100644 index 00000000..2c95cfc2 Binary files /dev/null and b/img/oas/api-create-set-status.png differ diff --git a/img/oas/api-create-stepper.png b/img/oas/api-create-stepper.png new file mode 100644 index 00000000..d202f60c Binary files /dev/null and b/img/oas/api-create-stepper.png differ diff --git a/img/oas/api-export-options.png b/img/oas/api-export-options.png new file mode 100644 index 00000000..359d29c6 Binary files /dev/null and b/img/oas/api-export-options.png differ diff --git a/img/oas/api-menu.png b/img/oas/api-menu.png new file mode 100644 index 00000000..1ca153a8 Binary files /dev/null and b/img/oas/api-menu.png differ diff --git a/img/oas/api-update-options.png b/img/oas/api-update-options.png new file mode 100644 index 00000000..3ee3298b Binary files /dev/null and b/img/oas/api-update-options.png differ diff --git a/img/oas/diagram_oas-flow-2.png b/img/oas/diagram_oas-flow-2.png new file mode 100644 index 00000000..8e6b3898 Binary files /dev/null and b/img/oas/diagram_oas-flow-2.png differ diff --git a/img/oas/import-api-button.png b/img/oas/import-api-button.png new file mode 100644 index 00000000..a9e8d4c6 Binary files /dev/null and b/img/oas/import-api-button.png differ diff --git a/img/oas/import-api-version-config.png b/img/oas/import-api-version-config.png new file mode 100644 index 00000000..43f4d920 Binary files /dev/null and b/img/oas/import-api-version-config.png differ diff --git a/img/oas/import-api-version.png b/img/oas/import-api-version.png new file mode 100644 index 00000000..c4029c60 Binary files /dev/null and b/img/oas/import-api-version.png differ diff --git a/img/oas/oas-2-code.png b/img/oas/oas-2-code.png new file mode 100644 index 00000000..cf96e39e Binary files /dev/null and b/img/oas/oas-2-code.png differ diff --git a/img/oas/open-api-format.png b/img/oas/open-api-format.png new file mode 100644 index 00000000..0462bc18 Binary files /dev/null and b/img/oas/open-api-format.png differ diff --git a/img/oas/upstream-url.png b/img/oas/upstream-url.png new file mode 100644 index 00000000..9fc44aff Binary files /dev/null and b/img/oas/upstream-url.png differ diff --git a/img/oas/version-endpoints.png b/img/oas/version-endpoints.png new file mode 100644 index 00000000..d8c8144a Binary files /dev/null and b/img/oas/version-endpoints.png differ diff --git a/img/okta-sso/Okta-create-app.png b/img/okta-sso/Okta-create-app.png new file mode 100644 index 00000000..4a970ce4 Binary files /dev/null and b/img/okta-sso/Okta-create-app.png differ diff --git a/img/okta-sso/okta-bad-request-wrong-callback.png b/img/okta-sso/okta-bad-request-wrong-callback.png new file mode 100644 index 00000000..f7de3937 Binary files /dev/null and b/img/okta-sso/okta-bad-request-wrong-callback.png differ diff --git a/img/okta-sso/okta-mfa-download-google-authenticator-2.png b/img/okta-sso/okta-mfa-download-google-authenticator-2.png new file mode 100644 index 00000000..d371795a Binary files /dev/null and b/img/okta-sso/okta-mfa-download-google-authenticator-2.png differ diff --git a/img/okta-sso/okta-mfa-google-auth-approved-3.png b/img/okta-sso/okta-mfa-google-auth-approved-3.png new file mode 100644 index 00000000..ac32777d Binary files /dev/null and b/img/okta-sso/okta-mfa-google-auth-approved-3.png differ diff --git a/img/okta-sso/okta-mfa-setup-1.png b/img/okta-sso/okta-mfa-setup-1.png new file mode 100644 index 00000000..cd9f54f8 Binary files /dev/null and b/img/okta-sso/okta-mfa-setup-1.png differ diff --git a/img/operator/tyk-api-ownership.svg b/img/operator/tyk-api-ownership.svg new file mode 100644 index 00000000..0b1d4d21 --- /dev/null +++ b/img/operator/tyk-api-ownership.svg @@ -0,0 +1,423 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/img/operator/tyk-operator-context.svg b/img/operator/tyk-operator-context.svg new file mode 100644 index 00000000..dc6e2885 --- /dev/null +++ b/img/operator/tyk-operator-context.svg @@ -0,0 +1,432 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/img/operator/tyk-operator.svg b/img/operator/tyk-operator.svg new file mode 100644 index 00000000..2b889a1c --- /dev/null +++ b/img/operator/tyk-operator.svg @@ -0,0 +1,133 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/img/operator/tyk-organisations.svg b/img/operator/tyk-organisations.svg new file mode 100644 index 00000000..36e7963f --- /dev/null +++ b/img/operator/tyk-organisations.svg @@ -0,0 +1,286 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/img/plugins/apis_menu.png b/img/plugins/apis_menu.png new file mode 100644 index 00000000..5d6c17f5 Binary files /dev/null and b/img/plugins/apis_menu.png differ diff --git a/img/plugins/control_plane_dashboard_link.png b/img/plugins/control_plane_dashboard_link.png new file mode 100644 index 00000000..ddcc0c11 Binary files /dev/null and b/img/plugins/control_plane_dashboard_link.png differ diff --git a/img/plugins/fileserver_settings.png b/img/plugins/fileserver_settings.png new file mode 100644 index 00000000..414847cf Binary files /dev/null and b/img/plugins/fileserver_settings.png differ diff --git a/img/plugins/go-plugin-different-tyk-versions.png b/img/plugins/go-plugin-different-tyk-versions.png new file mode 100644 index 00000000..ebb88f2e Binary files /dev/null and b/img/plugins/go-plugin-different-tyk-versions.png differ diff --git a/img/plugins/multiple_spans.png b/img/plugins/multiple_spans.png new file mode 100644 index 00000000..d3820fe1 Binary files /dev/null and b/img/plugins/multiple_spans.png differ diff --git a/img/plugins/plugin-bundles-overview.png b/img/plugins/plugin-bundles-overview.png new file mode 100644 index 00000000..17ed5ddc Binary files /dev/null and b/img/plugins/plugin-bundles-overview.png differ diff --git a/img/plugins/plugins_classic_api_bundles_config.png b/img/plugins/plugins_classic_api_bundles_config.png new file mode 100644 index 00000000..214d0b3c Binary files /dev/null and b/img/plugins/plugins_classic_api_bundles_config.png differ diff --git a/img/plugins/plugins_classic_api_definition_editor.png b/img/plugins/plugins_classic_api_definition_editor.png new file mode 100644 index 00000000..4c0bd370 Binary files /dev/null and b/img/plugins/plugins_classic_api_definition_editor.png differ diff --git a/img/plugins/plugins_classic_api_source_config.png b/img/plugins/plugins_classic_api_source_config.png new file mode 100644 index 00000000..a05bd725 Binary files /dev/null and b/img/plugins/plugins_classic_api_source_config.png differ diff --git a/img/plugins/plugins_enable.png b/img/plugins/plugins_enable.png new file mode 100644 index 00000000..a0e10539 Binary files /dev/null and b/img/plugins/plugins_enable.png differ diff --git a/img/plugins/plugins_oas_api_bundles_config.png b/img/plugins/plugins_oas_api_bundles_config.png new file mode 100644 index 00000000..c73e4480 Binary files /dev/null and b/img/plugins/plugins_oas_api_bundles_config.png differ diff --git a/img/plugins/plugins_oas_api_driver_options.png b/img/plugins/plugins_oas_api_driver_options.png new file mode 100644 index 00000000..795a5501 Binary files /dev/null and b/img/plugins/plugins_oas_api_driver_options.png differ diff --git a/img/plugins/plugins_oas_api_source_config.png b/img/plugins/plugins_oas_api_source_config.png new file mode 100644 index 00000000..00c5ef14 Binary files /dev/null and b/img/plugins/plugins_oas_api_source_config.png differ diff --git a/img/plugins/plugins_overview.svg b/img/plugins/plugins_overview.svg new file mode 100644 index 00000000..7785e23f --- /dev/null +++ b/img/plugins/plugins_overview.svg @@ -0,0 +1,96 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/img/plugins/postman_all_get_requests.png b/img/plugins/postman_all_get_requests.png new file mode 100644 index 00000000..81008fa3 Binary files /dev/null and b/img/plugins/postman_all_get_requests.png differ diff --git a/img/plugins/postman_success.png b/img/plugins/postman_success.png new file mode 100644 index 00000000..6ccee846 Binary files /dev/null and b/img/plugins/postman_success.png differ diff --git a/img/plugins/span_attributes.png b/img/plugins/span_attributes.png new file mode 100644 index 00000000..350ff4ac Binary files /dev/null and b/img/plugins/span_attributes.png differ diff --git a/img/plugins/span_error_handling.png b/img/plugins/span_error_handling.png new file mode 100644 index 00000000..857c658f Binary files /dev/null and b/img/plugins/span_error_handling.png differ diff --git a/img/plugins/span_from_context.png b/img/plugins/span_from_context.png new file mode 100644 index 00000000..fc9a1bae Binary files /dev/null and b/img/plugins/span_from_context.png differ diff --git a/img/plugins/span_name_and_status.png b/img/plugins/span_name_and_status.png new file mode 100644 index 00000000..eafe4f1c Binary files /dev/null and b/img/plugins/span_name_and_status.png differ diff --git a/img/poc.svg b/img/poc.svg new file mode 100644 index 00000000..0c1497ba --- /dev/null +++ b/img/poc.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/img/portal/portal-graphql-cors.png b/img/portal/portal-graphql-cors.png new file mode 100644 index 00000000..59f87b1b Binary files /dev/null and b/img/portal/portal-graphql-cors.png differ diff --git a/img/portal/portal-graphql-header-injection.png b/img/portal/portal-graphql-header-injection.png new file mode 100644 index 00000000..adbeff58 Binary files /dev/null and b/img/portal/portal-graphql-header-injection.png differ diff --git a/img/portal/portal-graphql-playground-viewdocs.png b/img/portal/portal-graphql-playground-viewdocs.png new file mode 100644 index 00000000..1667ad5c Binary files /dev/null and b/img/portal/portal-graphql-playground-viewdocs.png differ diff --git a/img/portal/portal-graphql-setup.png b/img/portal/portal-graphql-setup.png new file mode 100644 index 00000000..802f13f6 Binary files /dev/null and b/img/portal/portal-graphql-setup.png differ diff --git a/img/portal/portal-graphql.png b/img/portal/portal-graphql.png new file mode 100644 index 00000000..1f85192c Binary files /dev/null and b/img/portal/portal-graphql.png differ diff --git a/img/pump/datadog-tyk-analytics-dashboard.jpeg b/img/pump/datadog-tyk-analytics-dashboard.jpeg new file mode 100644 index 00000000..01979b60 Binary files /dev/null and b/img/pump/datadog-tyk-analytics-dashboard.jpeg differ diff --git a/img/pump/moesif_step5.png b/img/pump/moesif_step5.png new file mode 100644 index 00000000..2b4dce94 Binary files /dev/null and b/img/pump/moesif_step5.png differ diff --git a/img/pump/prometheus/tyk_config_prometheus_target.png b/img/pump/prometheus/tyk_config_prometheus_target.png new file mode 100644 index 00000000..6c49f53c Binary files /dev/null and b/img/pump/prometheus/tyk_config_prometheus_target.png differ diff --git a/img/pump/prometheus/tyk_grafana_configuration.png b/img/pump/prometheus/tyk_grafana_configuration.png new file mode 100644 index 00000000..32989503 Binary files /dev/null and b/img/pump/prometheus/tyk_grafana_configuration.png differ diff --git a/img/pump/prometheus/tyk_prometheus_upstream_time.png b/img/pump/prometheus/tyk_prometheus_upstream_time.png new file mode 100644 index 00000000..8a6103b8 Binary files /dev/null and b/img/pump/prometheus/tyk_prometheus_upstream_time.png differ diff --git a/img/pump/splunk_step1.png b/img/pump/splunk_step1.png new file mode 100644 index 00000000..ef3aba2f Binary files /dev/null and b/img/pump/splunk_step1.png differ diff --git a/img/pump/splunk_step2.png b/img/pump/splunk_step2.png new file mode 100644 index 00000000..ea442f57 Binary files /dev/null and b/img/pump/splunk_step2.png differ diff --git a/img/pump/splunk_step2b.png b/img/pump/splunk_step2b.png new file mode 100644 index 00000000..9c391427 Binary files /dev/null and b/img/pump/splunk_step2b.png differ diff --git a/img/pump/splunk_step3.png b/img/pump/splunk_step3.png new file mode 100644 index 00000000..1826ed21 Binary files /dev/null and b/img/pump/splunk_step3.png differ diff --git a/img/pump/splunk_step4.png b/img/pump/splunk_step4.png new file mode 100644 index 00000000..b09e7b49 Binary files /dev/null and b/img/pump/splunk_step4.png differ diff --git a/img/redhat-logo2.png b/img/redhat-logo2.png new file mode 100644 index 00000000..5ad51b76 Binary files /dev/null and b/img/redhat-logo2.png differ diff --git a/img/release-notes/apply_policy.png b/img/release-notes/apply_policy.png new file mode 100644 index 00000000..d25cc859 Binary files /dev/null and b/img/release-notes/apply_policy.png differ diff --git a/img/release-notes/blacklist_option.png b/img/release-notes/blacklist_option.png new file mode 100644 index 00000000..73b6290d Binary files /dev/null and b/img/release-notes/blacklist_option.png differ diff --git a/img/release-notes/certificate_pinning.png b/img/release-notes/certificate_pinning.png new file mode 100644 index 00000000..97cc74d8 Binary files /dev/null and b/img/release-notes/certificate_pinning.png differ diff --git a/img/release-notes/rate_limits.png b/img/release-notes/rate_limits.png new file mode 100644 index 00000000..e1de9c75 Binary files /dev/null and b/img/release-notes/rate_limits.png differ diff --git a/img/release-notes/tag_headers.png b/img/release-notes/tag_headers.png new file mode 100644 index 00000000..7b4d71d0 Binary files /dev/null and b/img/release-notes/tag_headers.png differ diff --git a/img/sso-auth0/auth0-accept.png b/img/sso-auth0/auth0-accept.png new file mode 100644 index 00000000..d4edf0fa Binary files /dev/null and b/img/sso-auth0/auth0-accept.png differ diff --git a/img/sso-auth0/auth0-app-basic-info.png b/img/sso-auth0/auth0-app-basic-info.png new file mode 100644 index 00000000..a6d7c559 Binary files /dev/null and b/img/sso-auth0/auth0-app-basic-info.png differ diff --git a/img/sso-auth0/auth0-app-type.png b/img/sso-auth0/auth0-app-type.png new file mode 100644 index 00000000..892bb82d Binary files /dev/null and b/img/sso-auth0/auth0-app-type.png differ diff --git a/img/sso-auth0/auth0-create-app.png b/img/sso-auth0/auth0-create-app.png new file mode 100644 index 00000000..820b0b3d Binary files /dev/null and b/img/sso-auth0/auth0-create-app.png differ diff --git a/img/sso-auth0/auth0-create-user.png b/img/sso-auth0/auth0-create-user.png new file mode 100644 index 00000000..d2b618a8 Binary files /dev/null and b/img/sso-auth0/auth0-create-user.png differ diff --git a/img/sso-auth0/auth0-login.png b/img/sso-auth0/auth0-login.png new file mode 100644 index 00000000..e3e14240 Binary files /dev/null and b/img/sso-auth0/auth0-login.png differ diff --git a/img/sso-auth0/auth0-tyk-callback-url.png b/img/sso-auth0/auth0-tyk-callback-url.png new file mode 100644 index 00000000..116e34ae Binary files /dev/null and b/img/sso-auth0/auth0-tyk-callback-url.png differ diff --git a/img/sso-auth0/auth0-user-details.png b/img/sso-auth0/auth0-user-details.png new file mode 100644 index 00000000..48965e4b Binary files /dev/null and b/img/sso-auth0/auth0-user-details.png differ diff --git a/img/sso-auth0/tyk-create-profile.png b/img/sso-auth0/tyk-create-profile.png new file mode 100644 index 00000000..345802bc Binary files /dev/null and b/img/sso-auth0/tyk-create-profile.png differ diff --git a/img/sso-auth0/tyk-dash-success.png b/img/sso-auth0/tyk-dash-success.png new file mode 100644 index 00000000..03a4dec0 Binary files /dev/null and b/img/sso-auth0/tyk-dash-success.png differ diff --git a/img/sso-auth0/tyk-id-profile-provider-config.png b/img/sso-auth0/tyk-id-profile-provider-config.png new file mode 100644 index 00000000..60a3f62d Binary files /dev/null and b/img/sso-auth0/tyk-id-profile-provider-config.png differ diff --git a/img/sso-auth0/tyk-new-profile-config.png b/img/sso-auth0/tyk-new-profile-config.png new file mode 100644 index 00000000..8f0c9c9d Binary files /dev/null and b/img/sso-auth0/tyk-new-profile-config.png differ diff --git a/img/sso-auth0/tyk-new-profile.png b/img/sso-auth0/tyk-new-profile.png new file mode 100644 index 00000000..b4d3a77a Binary files /dev/null and b/img/sso-auth0/tyk-new-profile.png differ diff --git a/img/sso-auth0/tyk-openid.png b/img/sso-auth0/tyk-openid.png new file mode 100644 index 00000000..8e671a85 Binary files /dev/null and b/img/sso-auth0/tyk-openid.png differ diff --git a/img/sso-auth0/tyk-profile-list.png b/img/sso-auth0/tyk-profile-list.png new file mode 100644 index 00000000..f2bdd8b9 Binary files /dev/null and b/img/sso-auth0/tyk-profile-list.png differ diff --git a/img/streams/enable-portal-webhooks.png b/img/streams/enable-portal-webhooks.png new file mode 100644 index 00000000..5ad5bad2 Binary files /dev/null and b/img/streams/enable-portal-webhooks.png differ diff --git a/img/streams/selection.png b/img/streams/selection.png new file mode 100644 index 00000000..f08e54ce Binary files /dev/null and b/img/streams/selection.png differ diff --git a/img/streams/streams-option.png b/img/streams/streams-option.png new file mode 100644 index 00000000..2080c00b Binary files /dev/null and b/img/streams/streams-option.png differ diff --git a/img/streams/subscribe-webhooks.png b/img/streams/subscribe-webhooks.png new file mode 100644 index 00000000..80dbea80 Binary files /dev/null and b/img/streams/subscribe-webhooks.png differ diff --git a/img/streams/tyk-streams-overview.png b/img/streams/tyk-streams-overview.png new file mode 100644 index 00000000..0fa05e10 Binary files /dev/null and b/img/streams/tyk-streams-overview.png differ diff --git a/img/support_icon.png b/img/support_icon.png new file mode 100644 index 00000000..d70c7313 Binary files /dev/null and b/img/support_icon.png differ diff --git a/img/tib/profiles/tib-profile-creation.gif b/img/tib/profiles/tib-profile-creation.gif new file mode 100644 index 00000000..d81e4c8d Binary files /dev/null and b/img/tib/profiles/tib-profile-creation.gif differ diff --git a/img/tyk-charts/tyk-cloud-deployment.png b/img/tyk-charts/tyk-cloud-deployment.png new file mode 100644 index 00000000..fd5f6c25 Binary files /dev/null and b/img/tyk-charts/tyk-cloud-deployment.png differ diff --git a/img/tyk-cloud.svg b/img/tyk-cloud.svg new file mode 100644 index 00000000..cb1f3530 --- /dev/null +++ b/img/tyk-cloud.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/img/tyk-sm.svg b/img/tyk-sm.svg new file mode 100644 index 00000000..79932b5d --- /dev/null +++ b/img/tyk-sm.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/img/upgrade-guides/check_packages.png b/img/upgrade-guides/check_packages.png new file mode 100644 index 00000000..37c206c2 Binary files /dev/null and b/img/upgrade-guides/check_packages.png differ diff --git a/img/upgrade-guides/deb_packages.png b/img/upgrade-guides/deb_packages.png new file mode 100644 index 00000000..c98869a9 Binary files /dev/null and b/img/upgrade-guides/deb_packages.png differ diff --git a/img/upgrade-guides/install_deb.png b/img/upgrade-guides/install_deb.png new file mode 100644 index 00000000..d4b2e314 Binary files /dev/null and b/img/upgrade-guides/install_deb.png differ diff --git a/img/upgrade-guides/list_package.png b/img/upgrade-guides/list_package.png new file mode 100644 index 00000000..a790d21c Binary files /dev/null and b/img/upgrade-guides/list_package.png differ diff --git a/img/upgrade-guides/list_package_2.png b/img/upgrade-guides/list_package_2.png new file mode 100644 index 00000000..54cfa90b Binary files /dev/null and b/img/upgrade-guides/list_package_2.png differ diff --git a/img/upgrade-guides/list_packages.png b/img/upgrade-guides/list_packages.png new file mode 100644 index 00000000..739fb71f Binary files /dev/null and b/img/upgrade-guides/list_packages.png differ diff --git a/img/upgrade-guides/list_rpm.png b/img/upgrade-guides/list_rpm.png new file mode 100644 index 00000000..5d13c54c Binary files /dev/null and b/img/upgrade-guides/list_rpm.png differ diff --git a/img/upgrade-guides/mongo_dump.png b/img/upgrade-guides/mongo_dump.png new file mode 100644 index 00000000..29977061 Binary files /dev/null and b/img/upgrade-guides/mongo_dump.png differ diff --git a/img/upgrade-guides/recompile_plugin.png b/img/upgrade-guides/recompile_plugin.png new file mode 100644 index 00000000..b787a7e9 Binary files /dev/null and b/img/upgrade-guides/recompile_plugin.png differ diff --git a/img/upgrade-guides/redis_config.png b/img/upgrade-guides/redis_config.png new file mode 100644 index 00000000..92d0bfe3 Binary files /dev/null and b/img/upgrade-guides/redis_config.png differ diff --git a/img/upgrade-guides/redis_save.png b/img/upgrade-guides/redis_save.png new file mode 100644 index 00000000..f30687b0 Binary files /dev/null and b/img/upgrade-guides/redis_save.png differ diff --git a/img/upgrade-guides/rpm_packages.png b/img/upgrade-guides/rpm_packages.png new file mode 100644 index 00000000..405d1c65 Binary files /dev/null and b/img/upgrade-guides/rpm_packages.png differ diff --git a/img/upgrade-guides/yum_history.png b/img/upgrade-guides/yum_history.png new file mode 100644 index 00000000..f1b33408 Binary files /dev/null and b/img/upgrade-guides/yum_history.png differ diff --git a/img/upgrade-guides/yum_update.png b/img/upgrade-guides/yum_update.png new file mode 100644 index 00000000..ff10aa1e Binary files /dev/null and b/img/upgrade-guides/yum_update.png differ diff --git a/index.mdx b/index.mdx new file mode 100644 index 00000000..8bc5a45f --- /dev/null +++ b/index.mdx @@ -0,0 +1,243 @@ +--- +mode: "custom" +title: "Tyk API Gateway Documentation" +--- +export const mostViewed = [ + { + title: "Tyk Platforms", + items: [ + { title: "Deployment Options", path: "apim/" }, + { title: "Self-managed", path: "tyk-on-premises/" }, + { title: "Cloud", path: "tyk-cloud/" }, + { title: "Open Source Gateway", path: "apim/open-source/" }, + ], + }, + { + title: "API management", + items: [ + { title: "Create API", path: "getting-started/create-api/" }, + { title: "Access an API", path: "getting-started/create-api-key/" }, + { title: "Create UDG schema", path: "universal-data-graph/udg-getting-started/creating-schema/" }, + { title: "Using OAS API definition", path: "getting-started/using-oas-definitions/" }, + ], + }, + { + title: "Product stack", + items: [ + { title: "OSS Tyk Gateway", path: "tyk-oss-gateway/" }, + { title: "OSS Tyk gateway configuration", path: "tyk-oss-gateway/configuration/" }, + { title: "Tyk Gateway API", path: "tyk-gateway-api/" }, + { title: "Tyk Dashboard", path: "tyk-dashboard/" }, + ], + }, + { + title: "Developer support", + items: [ + { title: "FAQs", path: "frequently-asked-questions/" }, + { title: "Latest Release", path: "product-stack/tyk-gateway/release-notes/overview/" }, + { title: "How to upgrade", path: "upgrading-tyk/" }, + { title: "Contribute to docs", path: "contribute/" }, + ], + }, +]; + +export const tykStacksCols = [ + [ + { title: "Tyk Gateway (Open source)", desc: "Open source Enterprise API Gateway.", path: "tyk-oss-gateway/" }, + { title: "Tyk Dashboard", desc: "The GUI and analytics platform for Tyk.", path: "tyk-dashboard/" }, + { title: "Universal Data Graph", desc: "Combine multiple APIs into one universal interface.", path: "universal-data-graph/" }, + { title: "Tyk Enterprise Developer Portal", desc: "Expose APIs for third-party developers to register and use.", path: "tyk-developer-portal/tyk-enterprise-developer-portal/" }, + { title: "Tyk Multi Data Center Bridge", desc: "Control plane that performs management and synchronisation.", path: "tyk-multi-data-centre/" }, + ], + [ + { title: "Tyk Pump (Open source)", desc: "Responsible for moving analytics into a persistent data store.", path: "tyk-pump/" }, + { title: "Tyk Operator", desc: "Manage your APIs on Tyk Gateway.", path: "tyk-operator/" }, + { title: "Tyk Sync", desc: "Command-line tool to manage and synchronise Tyk installation.", path: "tyk-sync/" }, + { title: "Tyk Identity Broker (Open source)", desc: "A bridge between various Identity Management Systems.", path: "tyk-identity-broker/" }, + { title: "Tyk Streams", desc: "Expose, manage & monetise real-time event streams, brokers and async APIs.", path: "product-stack/tyk-streaming/overview/" }, + ], +]; + +
+ +
+
+ + {/* headline */} +

+ Tyk Documentation +

+ + {/* body copy */} +

+ The hub for Tyk API management. Whether you’re new or experienced, get + started with Tyk, explore our product stack and core concepts, access + in-depth guides, and actively contribute to our ever-evolving products. +

+ +
+
+
+ {/* container-xl with 1 % inline padding β‰₯ 992 px */} +
+ + {/* section heading */} +
+
+

Try it out

+

Get started on any of our platforms

+
+
+ + {/* 2-column Card grid */} +
+ {/* ───────── Card 1 ───────── */} +
+
+
+
+ Guided evaluation +
+

Try Tyk Cloud

+

+ Get 48 hours of free, immediate access to our fully managed SaaS solution. No installation required. +

+

+ Ideal for quick, hassle-free evaluation +

+
+ + +
+
+ + {/* ───────── Card 2 ───────── */} +
+
+
+
+ 48-hour free trial +
+

Try Tyk Self Managed

+

Deploy Tyk on your own infrastructure with a free 14-day trial. Full access, full control.

+

+ Ideal for on premise environments or custom setups +

+
+ + +
+
+ + {/* ───────── Card 3 ───────── */} +
+
+
+
+ Guided evaluation +
+

Proof of Concept

+

+ For complex or regulated environments, work with our team on a tailored proof of concept or request an extended, in-depth trial of Tyk. +

+

+ We’ll map your requirements to Tyk’s architecture. +

+
+ + +
+
+ +
+
+
+ +
+
+ + {/* heading */} +
+

Most viewed

+

+ Quick links to pages our users visit most +

+
+ + {/* responsive grid */} +
+ {mostViewed.map((col) => ( +
+ {col.items.map((item) => ( + + ))} +
+ ))} +
+
+
+ +
+
+ + {/* heading */} +
+

Explore by Tyk Stack

+

+ Explore everything Tyk has to offer +

+
+ + {/* flex row β†’ two vertical lists on md+, one list on mobile */} +
+ {tykStacksCols.map((col, idx) => ( + + ))} +
+
+
+ + + +
diff --git a/key-concepts/grpc-proxy.mdx b/key-concepts/grpc-proxy.mdx new file mode 100644 index 00000000..e817701f --- /dev/null +++ b/key-concepts/grpc-proxy.mdx @@ -0,0 +1,129 @@ +--- +title: "gRPC Proxy" +'og:description': "How to use gRPC in Tyk" +tags: ['gRPC', 'Other Protocol'] +sidebarTitle: "gRPC Proxy" +--- + +## Using Tyk as a gRPC Proxy + +Tyk supports gRPC passthrough proxying when using HTTP/2 as a transport (the most common way to deploy gRPC services). + +The gRPC over HTTP2 specification defines the rules on how the gRPC protocol maps to a HTTP request, for more information [see](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md). In the context of the API Gateway, we are interested in the following: + +- You can target specific methods of the gRPC service using the format: `/{Service-Name}/{method name}`, for example: `/google.pubsub.v2.PublisherService/CreateTopic`. You can use this feature to apply standard ACL rules via Keys and Policies, or use URL rewrite plugins in our [Endpoint Desiger](/transform-traffic/url-rewriting#url-rewrite-middleware). +- HTTP method is always `POST`. +gRPC custom request metadata is added as HTTP headers, where metadata key is directly mapped to the HTTP header with the same name. + +You can also perform [gRPC load balancing](#grpc-load-balancing). + +**Prerequisites** +- Enable HTTP/2 support on the Gateway side, for both incoming and upstream connections, by setting `http_server_options.enable_http2` and `proxy_enable_http2` to true in your `tyk.conf` Gateway config file. +- The `listen path` of the Api should be the same as the gRPC service name, so tyk can route it correctly. +- Ensure that `strip_listen_path` is set to false in your API, so the route of the gRPC service method is build correctly following the standard: `{service_name}/{service_method}`. + +### Secure gRPC Proxy +Tyk Supports secure gRPC proxy connections, in order to do so you only need to attach a certificate to the API that you want to expose just as you do for regular APIs, after that you can consume the service via HTTPS. + +### Insecure gRPC Proxy (H2C) +For scenarios where you want to connect two services calling each other or just need an insecure connection you can use h2c (that is the non-TLS version of HTTP/2). Tyk supports h2c, this can be enabled at api level by setting `h2c` as protocol in the address of the gRPC server (`target_url`) e.g: `h2c://my-grpc-server.com`. + +### gRPC streaming +Tyk supports all kinds of gRPC streaming (client streaming, server streaming and bidirectional streaming). It requires you to set a low value for `flush_interval`, this is required in order to forward data to the downstream target as soon as the upstream target replies. A high flush interval will delay this communication. We recommend the lowest possible value: 1 (1 millisecond). You set this value in your `tyk.conf` file in the `http_server_options.flush_interval` option. + +## Mutual Authentication +Tyk supports Mutual Authentication in gRPC. See [Mutual TLS](/basic-config-and-security/security/mutual-tls/client-mtls#why-use-mutual-tls) to configure Mutual Authentication in Tyk. + +## Basic Authentication +Tyk supports Basic Authentication in gRPC. See [Basic Authentication](/api-management/authentication/basic-authentication) to configure Basic Authentication in Tyk. + +After setting your Tyk configuration, all you need to do is to send credentials with the correct base64 format in an `Authorization` header from your gRPC client. + +`Basic base64Encode(username:password)` + +## Token Based Authentication +Tyk supports Token Based Authentication in gRPC. See [Bearer Tokens](/api-management/authentication/bearer-token) to configure Token Based Authentication in Tyk. + +After setting your Tyk configuration, all you need to do is to send a token in an `Authorization` header from your gRPC client. + +## gRPC load balancing + +Tyk is able to perform load balancing on gRPC traffic using an approach similar to our native [Load Balancing](/planning-for-production/ensure-high-availability/load-balancing) functionality. + +For both secure and insecure gRPC scenarios, the steps above serve as a starting point. + +**In a secure gRPC scenario** + +For configuring multiple upstream targets in a secure gRPC scenario, follow these additional steps: + +* Check the "Enable round-robin load balancing" flag in the "Core Settings" section of your API. +* Define each target as `https://grpc.test.example.com:10000`, `https://grpc.test.example.com:10001` and so on. + +**In an insecure scenario (H2C)** + +Use the same approach but use the H2C scheme instead: `h2c://grpc.test.example.com:10000`, `h2c://grpc.test.example.com:10001`, etc. + +## Example of gRPC proxy using H2C +This is the simplest way to have a working gRPC proxy setup. You will need: + +* A gRPC server. For this example you can use [this server](https://github.com/grpc/grpc-go/tree/master/examples/helloworld) +* A gRPC client. You can use [grpcurl](https://github.com/fullstorydev/grpcurl) which is basically curl but for gRPC +* An instance of the Tyk Gateway and Dashboard + +**Steps for Configuration:** + +* Execute the gRPC server (for this example you can expose it at port `:50051`) +* Create the API via the Tyk dashboard with the following configuration: + * Set HTTP as protocol + * Uncheck `strip listen path` in the api + * Set listen path: `/helloworld.Greeter` + * Now set the `target_url`. In order for Tyk to detect that you will use `h2c` for this API we will need to write the URL with the prefix `h2c://`. For this example the URL can be `h2c://localhost:50051` + * Hit save, and once the Gateway finishes reloading you can test the solution +* From the command line you can consume the service via the Tyk Gateway. To test it, enter `grpcurl -plaintext -proto helloworld.proto -d '{"name": "joe"}' tyk-gateway:8080 helloworld.Greeter/SayHello` and you should get as a response: `{"message": "Hello joe"}` which means that everything is working as it should. + +## Example of gRPC proxy using HTTPS + +**Prerequisites:** + +* a gRPC server. For this example you can use [this server](https://github.com/grpc/grpc-go/tree/master/examples/route_guide) as this example already supports TLS. +* A gRPC client. You can use [grpcurl](https://github.com/fullstorydev/grpcurl) which is basically curl but for gRPC (or you can use the client application) +* A certificate to expose the API via HTTPS +* An instance of Tyk Gateway and Dashboard + +**Steps for Configuration:** + +1. Execute the gRPC server (for this example we can expose it at port `:10000`), the `route_guide` application receives a flag to use TLS (`go run server.go -tls=true`). It's exposed in `grpc.test.example.com:10000` +2. Create the API via dashboard with the next configuration: + * Set HTTPS as protocol + * Uncheck `Strip listen path` in the api + * Add the certificate + * Set a custom domain, for this example use `tyk` + * Set as listen path: `/routeguide.RouteGuide` + * Now in the target URL set the location of the service: `https://grpc.test.example.com:10000` + * Click Save +3. At this point you're ready to test the solution, so, from the command line type: `grpcurl -proto route_guide.proto -d '{"latitude": 1, "longitude":2}' tyk:8080 routeguide.RouteGuide/GetFeature` and you should get a successful response. Note that you are not sending the flag `-plaintext` as the desire is to connect via HTTPS. + +## Example of bidirectional data streaming using a gRPC service exposed via HTTPS but communicating Tyk to service via H2C + +In this example you will expose a gRPC service via HTTPS using Tyk, but Tyk will consume the upstream via h2c. This situation is very common when using Kubernetes, where the internet traffic going through Tyk are TLS encrypted, but traffic in the inner cluster are in plain HTTP (h2c). + +**Prerequisites** +* a gRPC server. For this example you can use [this server](https://github.com/grpc/grpc-go/tree/master/examples/route_guide). +* a client application. You can use [this client](https://github.com/grpc/grpc-go/tree/master/examples/route_guide/client). +* A certificate to expose the API via HTTPS +* An instance of the Tyk Gateway and Dashboard. + +**Steps for Configuration:** + +* Execute the gRPC server (for this example expose it at port `:10000`), the `route_guide` application receives a flag to enable/disable TLS (`go run server.go -tls=false`). It's available in `locahost:10000` +* In the Tyk Gateway config file set `"http_server_options.flush_interval": 1` and run the Gateway (for this example will be running it in port 8000). +* Create the API via the Tyk Dashboard with the following configuration: + * Set HTTPS as the protocol + * Add the certificate + * Set a custom domain, for this example `tyk` + * Set as listen path: `/routeguide.RouteGuide`. For testing purposes we will use the `RouteChat` service as this is a bidirectional service. + * Now in the target URL set the location of the service: `h2c://localhost:10000`. This way Tyk will communicate with the upstream using h2c + * Click Save +* Ensure that the client application has the server address pointing to Tyk, for this example: `https://tyk.com:8000`. +* Now you are ready to test the solution. Run the client application and it should send and receive data simultaneously. + diff --git a/key-concepts/tcp-proxy.mdx b/key-concepts/tcp-proxy.mdx new file mode 100644 index 00000000..a40db4cc --- /dev/null +++ b/key-concepts/tcp-proxy.mdx @@ -0,0 +1,148 @@ +--- +title: "TCP Proxy" +'og:description': "Describe how you can use Tyk as a simple TCP Proxy" +tags: ['TCP Proxy', 'TLS'] +sidebarTitle: "TCP Proxy" +--- + +## Using Tyk as a TCP Proxy + +Tyk can be used as a reverse proxy for your TCP services. It means that you can put Tyk not only on top of your APIs but on top of **any** network application, like databases, services using custom protocols etc. + +### Set via your API + +To enable TCP proxying, set the `protocol` field either to `tcp` or `tls`. In the case of TLS, you can also specify a `certificate` field with a certificate ID or path to it. + +Similar to the above, the proxy target scheme should be set to `tcp://` or `tls://`, depending if your upstream is secured by TLS or not. + +The simplest TCP API definition looks like this: + +```yaml expandable +{ + "listen_port": 30001, + "protocol": "tls", + "certificate": [""], + "proxy": { + "target_url": "tls://upstream:9191" + } +} +``` + +### Set via your Dashboard + +From the **API Designer > Core Settings** tab, select the appropriate protocol from the Protocol field drop-down list. + +Enter the network port number in the **Port** field. + +If using TLS you can also add a PEM formatted SSL certificate in the **Upstream Certificates** section from the **Advanced Options** tab. + +Protocol + +Tyk supports multiplexing based on certificate SNI information, which means that you can have multiple TCP services on the **same port**, served on different domains. Additionally, all services on the same port should share the same protocol: either `tcp`, `tls`, `http` or `https`. + +If Tyk sits behind another proxy, which has the PROXY protocol enabled, you can set `enable_proxy_protocol` to `true`. + +As for features such as load balancing, service discovery, Mutual TLS (both authorization and communication with upstream), certificate pinning, etc. All work exactly the same way as for your HTTP APIs. + +## Allowing specific ports + +By default, you will not be able to run a service on a custom port, until you allow the required ports. +Since TCP services can be configured via the Dashboard, you should be careful who can create such services, and which ports they can use. Below is an example of allowing ports in `tyk.conf`: + +```yaml expandable +{ + ... + "ports_whitelist": { + "https": { + "ranges": [ + { + "from": 8000, + "to": 9000 + } + ] + }, + "tls": { + "ports": [ + 6000, + 6015 + ] + } + } + ... +} +``` + +As you can see, you can use either `ranges` or `ports` directives (or combine them). + +You can also disable this behavior and allow any TCP port by setting `disable_ports_whitelist` to `true`. + + +## Health checks + +TCP health checks are configured the same way as HTTP ones. +The main difference is that instead of specifying HTTP requests, you should specify a list of commands, which send data or expect some data in response. + +A simple health check which verifies only connectivity (e.g. if a port is open), can be: + +```yaml expandable +{ +... + "uptime_tests": { + "check_list": [ + { "url": "127.0.0.1:6379" }, + "commands": [] + ] + } +... +} +``` + +### Complex example + +Here is quite a complex example of using health checks, which shows a Redis Sentinel setup. In this configuration, we put a TCP proxy, e.g. Tyk, on top of two or more Redis nodes, and the role of the proxy will always direct the user to Redis master. To do that we will need to perform health checks against each Redis node, to detect if it is a master or not. In other words, Redis clients who communicate with Redis through the proxy will be always directed to the master, even in case of failover. + +```yaml expandable +{ + "name": "Redis Sentinel", + "listen_port": 6379, + "protocol": "tcp", + "enable_load_balancing": true, + "proxy": { + "target_list": ["192.168.10.1:6379", "192.168.10.2:6379"] + }, + "check_host_against_uptime_tests": true, + "uptime_tests": { + "check_list": [ + { + "url": "192.168.10.1:6379", + "commands": [ + { "name": "send", "message": "PING\r\n" }, + { "name": "expect", "message": "+PONG" }, + { "name": "send", "message": "info replication\r\n" }, + { "name": "expect", "message": "role:master" }, + { "name": "send", "message": "QUIT\r\n" }, + { "name": "send", "message": "+OK" } + ] + }, + { + "url": "192.168.10.2:6379", + "commands": [ + { "name": "send", "message": "PING\r\n" }, + { "name": "expect", "message": "+PONG" }, + { "name": "send", "message": "info replication\r\n" }, + { "name": "expect", "message": "role:master" }, + { "name": "send", "message": "QUIT\r\n" }, + { "name": "send", "message": "+OK" } + ] + } + ] + } +} +``` + +At the moment Tyk supports only 2 commands: + - `send` send string to server +- `expect` expect string from the server + + +[1]: /img/dashboard/system-management/api-protocol.png diff --git a/logo/dark.svg b/logo/dark.svg new file mode 100644 index 00000000..d983d447 --- /dev/null +++ b/logo/dark.svg @@ -0,0 +1 @@ + diff --git a/logo/light.svg b/logo/light.svg new file mode 100644 index 00000000..fbfc62fb --- /dev/null +++ b/logo/light.svg @@ -0,0 +1 @@ + diff --git a/plan-your-api-integration.mdx b/plan-your-api-integration.mdx new file mode 100644 index 00000000..fe3f7cc1 --- /dev/null +++ b/plan-your-api-integration.mdx @@ -0,0 +1,112 @@ +--- +title: "Plan Your API Integration" +'og:description': "Plan your first API integration" +tags: ['Tyk API Management', 'Getting Started', 'Tutorials'] +order: 5 +sidebarTitle: "Plan Your API Integration" +--- + +## Planning Your API Project: A Step-by-Step Guide with Tyk + +Creating and managing APIs is essential for modern applications, but if you’re new to APIs, this process might feel complex. This guide walks you through what APIs are, why they matter, and how to plan, design, deploy, and manage themβ€”step by stepβ€”with support from Tyk’s API management platform. + +--- + +### 1. What is an API and Why Does It Matter? + +An **API (Application Programming Interface)** allows different software systems to talk to each other. Think of it as a bridge that lets applications, websites, or devices interact. For example, when you book a flight or make a payment online, you’re likely interacting with multiple APIs in the background. APIs help create a connected experience, enabling everything from mobile app development to system integrations and data sharing. + +**Why use an API?** +- **Efficiency**: APIs allow applications to reuse functionality (e.g., logins, data storage) instead of building it from scratch. +- **Flexibility**: They let you connect applications across platforms, such as web, mobile, and IoT. +- **Scalability**: APIs enable modular application development, making it easier to add or update features over time. + +--- + +### 2. Steps in Creating, Using, and Managing APIs + +Let’s break down the steps to design, develop, and manage APIs. By planning each stage, you can create APIs that are reliable, secure, and ready for growth. + +#### **Step 1: Define Your API’s Purpose and Scope** + +Begin by clarifying why you need an API and what you hope to achieve: +- **Who will use this API?** Identify whether the API is for internal teams, partners, or external developers. +- **What problem does it solve?** Define the API’s core functions and the kind of data it will handle. +- **What resources will be shared?** List out the specific data or services (like user profiles, inventory, or orders) that the API will expose. + +**Example:** Imagine you’re creating an API to provide real-time inventory data to an e-commerce app. You’ll need to determine which product details are exposed, who can access them, and any restrictions on data sharing. + +#### **Step 2: Design the API Structure** + +With your goals in mind, you’ll now design the API: +- **Choose the API Style**: Decide between REST (easy to implement, flexible) or GraphQL (lets clients query specific data they need). Tyk supports both, so choose based on the complexity and data needs. +- **Define Endpoints and Methods**: Identify the actions your API should allow (e.g., GET for retrieving data, POST for adding data). +- **Specify Data Models**: Define the format of data exchanged. For instance, will product data include details like price, description, and availability? + +**Security Consideration:** Plan how users will [authenticate](/api-management/security-best-practices#authentication). Will they use [tokens](/api-management/authentication/bearer-token) or [OAuth](/api-management/authentication/oauth-2) (for user-specific access)? Tyk offers tools to implement any of these methods effectively. + +#### **Step 3: Document the API** + +Documentation is critical for your API’s usability: +- **Explain each endpoint**: Describe what each endpoint does, its parameters, and any expected responses. +- **Provide examples**: Include sample requests and responses to clarify expected use. +- **Detail errors and edge cases**: Help users understand what happens if they make incorrect requests. + +With [Tyk’s developer portal](/tyk-developer-portal/tyk-portal-classic/customise/custom-developer-portal#why-build-a-custom-developer-portal), you can automatically publish your API documentation, making it easier for developers to get started with your API. + +#### **Step 4: Develop and Test the API** + +Now you’re ready to build and test: +- **Develop the Backend**: Write the code to implement your [API’s endpoints](/api-management/dashboard-configuration#exploring-api-endpoint-designer) and integrate with your [database](/api-management/dashboard-configuration#supported-database) or service layer. +- **Functional Testing**: Check each endpoint to ensure it behaves as expected and handles common errors. +- **Load Testing**: Simulate traffic to see how your API performs under different loads. Tyk offers tools to scale your API seamlessly and track performance metrics. + +#### **Step 5: Deploy and Secure the API** + +Once your API is tested, deploy it for use: +- **Deploying to Production**: Decide where your API will be hosted (cloud, on-premises, or a hybrid setup). Tyk supports multiple deployment environments, allowing you to easily transition between development, staging, and production. +- **Implement Security**: Protect your API with [authentication methods](/basic-config-and-security), [rate limiting](/api-management/rate-limit#what-is-rate-limiting), and access controls to prevent unauthorized access and misuse. + +Tyk’s platform makes security implementation straightforward, offering features like OAuth 2.0, rate limiting, and custom policies to secure your API. + +#### **Step 6: Monitor and Maintain the API** + +An API isn’t a one-time setup; it needs regular monitoring and updates: +- **Monitor Performance**: Use [Tyk’s real-time analytics](/tyk-pump) to track metrics like [latency](/api-management/tyk-pump#latency), error rates, and [usage](/tyk-cloud#track-usage). This helps identify any bottlenecks or security risks. +- **Version and Update**: As you add new features, use [Tyk’s versioning](/api-management/automations/operator#api-versioning) to avoid breaking existing functionality. +- **Optimize and Scale**: With Tyk, you can adjust your rate limits, caching, and load balancing to handle higher volumes as needed. Optimizing is especially necessary as you [move your workload into production](/planning-for-production). + +--- + +### 3. How Tyk Supports Your API Lifecycle + +Tyk provides a comprehensive platform to help you through each stage of API development, from planning to deployment and beyond. + +#### **Design and Planning Phase** + +Tyk simplifies the early stages of API planning by providing: +- **Multi-API Support**: Tyk supports REST, GraphQL, and gRPC APIs, so you can choose the best style for your project. +- **Advanced Security Options**: Easily configure [authentication methods](/basic-config-and-security) and [permissions](/api-management/user-management#user-permissions) to protect data and control access. +- **Developer Portals**: [Tyk’s developer portals](/tyk-developer-portal/tyk-portal-classic) allow you to publish documentation and manage developer access, making it easier for teams or external partners to use your APIs. + +#### **Deployment and Configuration Phase** + +In this stage, Tyk streamlines deployment, whether on the cloud, on-premises, or hybrid: +- **Kubernetes Integration with Tyk Operator**: If your API runs on Kubernetes, [Tyk Operator](/api-management/automations/operator#what-is-tyk-operator) integrates with Kubernetes to help you manage deployments as Kubernetes resources. +- **Custom Domain Setup**: Configure custom domains to make your API URLs user-friendly and secure with SSL/TLS certificates. +- **Deployment Across Multiple Environments**: [Tyk’s gateways](/tyk-oss-gateway) let you deploy, monitor, and secure multiple APIs from a single platform, so you can [manage environments](/api-management/multiple-environments) (development, staging, production) with ease. + +#### **Operations and Business as Usual (BAU) Phase** + +After deployment, Tyk offers robust tools to ensure smooth API operations and maintenance: +- **Real-Time Monitoring and Analytics**: [Tyk’s dashboard](/tyk-dashboard) provides insights into API traffic, usage patterns, and error rates, enabling quick response to issues. +- **Dynamic Policy Management**: Set up and adjust security policies to control access and usage, such as [IP whitelisting](/api-management/gateway-config-tyk-classic#ip-access-control), [request throttling](/api-management/request-throttling), or [rate limiting](/api-management/rate-limit). +- **Plugin Support for Customization**: Use [Tyk’s plugin system](/tyk-cloud#configure-plugins) to add custom functionality, such as [custom authentication](/tyk-cloud#add-custom-authentication), or [traffic transformations](/api-management/traffic-transformation). + +--- + +### 4. Next Steps with Tyk + +Building and managing APIs is a process of continuous improvement. With Tyk, you have a partner that provides the tools to plan, develop, deploy, and maintain APIs with efficiency and security. As you refine your API strategy, you can rely on Tyk’s capabilities to adapt and scale with you, ensuring that your APIs deliver value to users and meet evolving business needs. + +To get started in your Tyk journey, [get started on Tyk Cloud](/getting-started/create-account) and learn about [Tyk components](/tyk-components) to further understand how Tyk supports your specific goals. \ No newline at end of file diff --git a/planning-for-production.mdx b/planning-for-production.mdx new file mode 100644 index 00000000..157b6b9e --- /dev/null +++ b/planning-for-production.mdx @@ -0,0 +1,237 @@ +--- +title: "Planning for Production" +'og:description': "How to plan for production deployment of Tyk Gateway" +sidebarTitle: "Planning for Production" +--- + +So you want to deploy Tyk to production? + +There's a few things worth noting that can ensure high performance of your Tyk Gateway nodes. Here are some of the basic things we do for load testing to make sure machines don't run out of resources. + +## Performance Expectations + +Our performance testing plan focused on replicating the setup of our customers, and tried not to optimize for "benchmarks": so no supercomputers and no sub-millisecond inner DC latency. Instead, we tested on a super-low performance 2 CPU Linode machine, with 50ms latency between Tyk and upstream. For testing, we used Tyk Gateway in Multi-Cloud mode, with default config. Test runner was using [Locust][2] framework and [Boomer][3] for load generation. + +With the optimizations outlined below, and using our distributed rate limiter, Tyk can easily handle ~3,000 requests per second with analytics, key authentication, and quota checks enabled. + +In the test below, Tyk v2.7 evaluates each request through its access control list, rate limiter, quota evaluator, and analytics recorder across a single test token and still retains a latency firmly under 70 milliseconds: + +Tyk 2.7 performance + +### Performance changes based on use case + +A popular use case for Tyk that we've seen crop up is as an interstitial API Gateway for microservices that are making service-to-service calls. Now, with these APIs, rate limiting and quotas are not usually needed: only authentication and analytics. If we run the same tests again with rate limits disabled, and quotas disabled, then we see a different performance graph: + +Tyk 2.7 performance + +Here we have pushed the test to 3,000 requests per second, and we can see that Tyk copes just fine. With only a few spikes past the 100ms line, we can clearly see solid performance right up to 3,000 requests per second with acceptable latency. + +### Vanilla Tyk + +If you were to just test Tyk as a pass-through auth proxy, we can see that 4k RPS (requests per second) is easily handled: + +Tyk 2.7 performance + +This configuration has analytics recording disabled, but we are still authenticating the inbound request. As you can see, we easily handle the 4k RPS mark and we can go further with more optimization. + +## Change all the shared secrets + +Ensure that these are changed before deploying to production. The main secrets to consider are: + +### `tyk.conf`: + +* `secret` +* `node_secret` + +### `tyk_analytics.conf`: + +* `admin_secret` +* `shared_node_secret` +* `typ_api_config.secret` + +GW `secret` and DB `tyk_api_config.secret` must match + +GW `node_secret` and DB `shared_node_secret` must match + + +### Use the public/private key message security! + +Tyk makes use of public-key message verification for messages that are sent from the Dashboard to the Gateways, these messages can include: + +* Zeroconfig Dashboard auto-discovery details +* Cluster reload messages +* Cluster configuration getters/setters for individual Gateways in a cluster + +These keys are also used for plugin security, so it is important to use them if you are deploying code to your Gateway. The public key that ships with your Gateways is used to verify the manifest and files that come with any plugin bundle that gets downloaded by the bundle downloader. + +## Change your Control Port + +To secure your Tyk installation, you can configure the following settings in your [tyk.conf](/tyk-oss-gateway/configuration): + +`control_api_hostname` - Set the hostname to which you want to bind the REST API. + +`control_api_port` - This allows you to run the Gateway Control API on a separate port, and protect it behind a firewall if needed. + +If you change these values, you need to update the equivalent fields in the dashboard conf file `tyk_analytics.conf`: `tyk_api_config.Host` and `tyk_api_config.Port` + + +## Connecting multiple gateways to a single dashboard + +Please note that, for a Self-Managed installation, the number of gateway nodes you may register with your dashboard concurrently will be subject to the terms of your license. + +Each gateway node must be configured in the same way, with the exception being if you want to shard your gateways. Each gateway node in the cluster will need connectivity to the same Redis server & persistent database. + +## Other Dashboard Security Considerations + +In addition to changing the default secrets (see [Change all the shared secrets](#change-all-the-shared-secrets)) if you change the Control API port (see [Change your Control Port](#change-your-control-port)), you also need to change the connection string settings in your `tyk_analytics.conf` file. + +## Ensure you are matching only the URL paths that you want to match + +We recommend that you configure Tyk Gateway to use [exact URL path matching](/getting-started/key-concepts/url-matching#exact-match) and to enforce [strict route matching](/tyk-oss-gateway/configuration#http_server_optionsenable_strict_routes) to avoid accidentally invoking your unsecured `/health` endpoint when a request is made to `/customer/{customer_id}/account/health`... + +Unless you want to make use of Tyk's flexible *listen path* and *endpoint path* matching modes and understand the need to configure patterns carefully, you should enable `TYK_GW_HTTPSERVEROPTIONS_ENABLESTRICTROUTES`, `TYK_GW_HTTPSERVEROPTIONS_ENABLEPATHPREFIXMATCHING` and `TYK_GW_HTTPSERVEROPTIONS_ENABLEPATHSUFFIXMATCHING`. + +## Health checks are expensive + +To keep real-time health-check data and make it available to the Health-check API, Tyk needs to record information for every request, in a rolling window - this is an expensive operation and can limit throughput - you have two options: switch it off, or get a box with more cores. + +## Selecting the appropriate log level + +Tyk provides multiple [log levels](/api-management/logs-metrics#system-logs): error, warn, info, debug. Setting higher log levels consumes more computing resources and would have an impact on the Tyk component. Tyk installations default to log level info unless modified by config files or environment variables. + +It is recommended to set to debug only for the duration of troubleshooting as it adds heavier resource overheads. In high performance use cases for Tyk Gateway, consider setting a log level lower than info to improve overall throughput. + +## Use the optimization settings + +The below settings will ensure connections are effectively re-used, removes a transaction from the middleware run that enforces org-level rules, enables the new rate limiter (by disabling sentinel rate limiter) and sets Tyk up to use an in-memory cache for session-state data to save a round-trip to Redis for some other transactions. + +Most of the changes below should be already in your `tyk.conf` by default: + +``` expandable +"close_connections": false, +"proxy_close_connections": false, +"enforce_org_quotas": false, +"enforce_org_data_detail_logging": false, +"experimental_process_org_off_thread": true, +"enable_sentinel_rate_limiter": false, +"local_session_cache": { + "disable_cached_session_state": false +}, +"max_idle_connections_per_host": 500 +``` + +In Tyk v2.7 we optimized the connection pool between Tyk and your Upstream. In previous releases `max_idle_connections_per_host` option, was capped at 100. From v2.7 you have been able to set it to any value. + +`max_idle_connections_per_host` limits the number of keep-alive connections between clients and Tyk. If you set this value too low, then Tyk will not re-use connections and you will have to open a lot of new connections to your upstream. + +If you set this value too high, you may encounter issues when slow clients occupy your connection and you may reach OS limits. + +You can calculate the right value using a straightforward formula: + +If the latency between Tyk and your Upstream is around 50ms, then a single connection can handle 1s / 50ms = 20 requests. So if you plan to handle 2000 requests per second using Tyk, the size of your connection pool should be at least 2000 / 20 = 100. For example, on low-latency environments (like 5ms), a connection pool of 100 connections will be enough for 20k RPS. + +## Protect Redis from overgrowing + +Please read carefully through this [doc](/api-management/policies#set-physical-key-expiry-and-deletion) to make an *aware decision* about the expiration of your keys in Redis, after which they will be removed from Redis. If you don't set the lifetime, a zero default means that keys will stay in Redis until you manually delete them, which is no issue if you have a process outside Tyk Gateway to handle it. If you don't - and especially in scenarios that your flow creates many keys or access tokens for every user or even per call - your Redis can quickly get cluttered with obsolete tokens and eventually affect the performance of the Tyk Gateway. + +## Analytics Optimizations + +If using a [Redis cluster](https://redis.io/docs/management/scaling/) under high load it is recommended that analytics are distributed among the Redis shards. This can be configured by setting the [analytics_config.enable_multiple_analytics_keys](/tyk-oss-gateway/configuration#analytics_configenable_multiple_analytics_keys) parameter to true. Furthermore, analytics can also be disabled for an API using the [do_not_track](/api-management/gateway-config-tyk-classic#traffic-logs) configuration parameter. Alternatively, tracking for analytics can be disabled for selected endpoints using the [do not track endpoint plugin](/api-management/traffic-transformation/do-not-track#do-not-track-using-tyk-oas). + +### Protobuf Serialisation + +In Tyk Gateway, using [protobuf](/tyk-oss-gateway/configuration#analytics_configserializer_type) serialisation, instead of [msgpack](https://msgpack.org) can increase performance for sending and processing analytics. +
+**Note:** *protobuf* is not currently supported in *MDCB* deployment. + +If using Tyk Cloud platform under high load, it is also recommended that analytics are stored within a local region. This means that a local Tyk Pump instance can send the analytics to a localised data sink, such as PostgreSQL or MongoDB (no need for the hybrid pump). This can reduce traffic costs since your analytics would not be sent across regions. + + +## Use the right hardware + +Tyk is CPU-bound: you will get exponentially better performance the more cores you throw at Tyk - it's that simple. Tyk will automatically spread itself across all cores to handle traffic, but if expensive operations like health-checks are enabled, then those can cause keyspace contention, so again, while it helps, health-checks do throttle throughput. + +## Resource limits + +Make sure your host operating system has resource limits set to handle an appropriate number of connections. + +You can increase the maximum number of files available to the kernel by modifying `/etc/sysctl.conf`. + +``` +fs.file-max=160000 +``` + +Please note that a single file, with associated inode & dcache consumes approximately 1KB RAM. As such, setting `fs.file-max=160000` will consume a maximum of 160MB ram. + +The changes will apply after a system reboot, but if you do not wish to reboot quite yet, you can apply the change for the current session using `echo 160000 > /proc/sys/fs/file-max`. + +## File Handles / File Descriptors + +Now we need to configure the file handles available to your Tyk services. + +### systemd + +Override your `systemd` unit files for each of the Tyk services using `systemctl edit {service_name}`. + +* Gateway `systemctl edit tyk-gateway.service` +* Dashboard `systemctl edit tyk-dashboard.service` +* Pump `systemctl edit tyk-pump.service` +* Multi Data-Center Bridge `systemctl edit tyk-sink.service` + +You may then add `LimitNOFILE=80000` to the `[Service]` directive as follows: + +``` +[Service] +LimitNOFILE=80000 +``` + +After making these changes, you'll need to restart your service, for example: + +``` +systemctl restart tyk-gateway.service +``` + +### Docker + +You may set *ulimits* in a container using the `--ulimit` option. See *Docker* documentation for detail on [setting *ulimits*]( https://docs.docker.com/engine/reference/commandline/run/#set-ulimits-in-container---ulimit) + +``` +docker run --ulimit nofile=80000:80000 \ + -it --name tyk-gateway \ + tykio/tyk-gateway:latest +``` + + + +If you are not using *systemd* or Docker, please consult your Operating System documentation for controlling the number of File Descriptors available for your process. + + + +## File modification at runtime + +Understanding what files are created or modified by the Dashboard and Gateway during runtime can be important if you are running infrastructure orchestration systems such as Puppet, which may erroneously see such changes as problems that need correcting. + +* Both the Gateway and Dashboard will create a default configuration file if one is not found. +* Dashboard will write the license into the configuration file if you add it via the UI. +* From Tyk v2.3 onwards it has been possible for a Dashboard to remotely change the config of a Gateway. This will cause the Gateway's configuration file to update. + +## Evaluate Performance Benchmarks + +An API Gateway serves as the single point of entry into your ecosystem, introducing an extra hop in the process. Because of this, performance is critical. Tyk was built with speed in mind from day one, which is why it’s written in **Go**, a language known for its efficiency and performance. + +## Resources to Explore Tyk’s Performance: + +1. **[Tyk Performance Benchmarks](https://tyk.io/blog/performance-benchmarks)** + Discover how Tyk Gateway performs across: + - The full Tyk feature set + - Multiple cloud environments + - CPU scalability + - Head-to-head comparisons with competitors + +2. **[Performance Tuning Your Gateway](https://tyk.io/blog/performance-tuning-your-tyk-api-gateway/)** + A step-by-step guide to fine-tuning your Gateway for maximum performance. + +3. **[Manual Performance Testing on AWS](https://tyk.io/a-manual-for-simple-performance-testing-with-tyk-on-aws/)** + Best practices for setting up and running manual performance tests on AWS to ensure your Gateway is optimized. + + diff --git a/planning-for-production/database-settings.mdx b/planning-for-production/database-settings.mdx new file mode 100644 index 00000000..69700620 --- /dev/null +++ b/planning-for-production/database-settings.mdx @@ -0,0 +1,318 @@ +--- +title: "Database Management" +'og:description': "How to configure Tyk's data storage for production" +sidebarTitle: "Database Management" +--- + +import MongodbVersionsInclude from '/snippets/mongodb-versions-include.mdx'; +import SqlVersionsInclude from '/snippets/sql-versions-include.mdx'; +import RedisCalculator from '/snippets/RedisCalculator.mdx'; +import DatabaseCalculator from '/snippets/DatabaseCalculator.mdx'; + +Visit the following sections to see how to configure the Database for Production: +* [Redis](/planning-for-production/database-settings#redis) +* [MongoDB](/planning-for-production/database-settings#mongodb-sizing-guidelines) +* [PostgreSQL](/planning-for-production/database-settings#postgresql) + +Please consult the [data storage configuration](/api-management/dashboard-configuration#data-storage-solutions) guide for further information relating to how to configure Tyk's data storage across different database engines. + +## Redis + +**Supported Versions** +- Tyk 5.3 supports Redis 6.2.x, 7.0.x, and 7.2.x +- Tyk 5.2.x and earlier supports Redis 6.0.x and Redis 6.2.x only. + + +**Split out Your Databases** + +This is a no-brainer, but keep Redis and MongoDB off the system running the Gateway, they both use lots of RAM, and with Redis and the Gateway constantly communicating you will be facing resource contention on the CPU for a marginal decrease in latency. + +So in our setup, we recommend that Redis and MongoDB/PostgreSQL live on their own systems, separate from your Tyk Gateway. If you like, run them together on the same box, that's up to you. + +The network topology we like to use is: + +* Two or more Tyk Gateway nodes (load balanced, each Gateway installed on separate machines). +* A separate MongoDB or PostgreSQL cluster +* A separate Redis server with fail-over or cluster +* One Tyk Dashboard node installed on a separate machine +* One Tyk Pump node installed on a separate machine that handles data transitions + +If you are making use of the Tyk Caching feature, then it is possible to use a secondary Redis server or Redis cluster to store cache data. This can be very useful in high-traffic APIs where latency is at a premium. + + +**Make sure you have enough Redis connections** + +Tyk makes heavy use of Redis in order to provide a fast and reliable service, in order to do so effectively, it keeps a passive connection pool ready. For high-performance setups, this pool needs to be expanded to handle more simultaneous connections, otherwise you may run out of Redis connections. + +Tyk also lets you set a maximum number of open connections so that you don't over-commit connections to the server. + +To set your maximums and minimums, edit your `tyk.conf` and `tyk_analytics.conf` files to include: + +```yaml expandable +"storage": { + ... + "optimisation_max_idle": 2000, + "optimisation_max_active": 4000, + ... +}, +``` + + +Set the `max_idle` value to something large, we usually leave it at around `2000` for HA deployments, and then set your `max_active` to your upper limit (as in, how many additional connections over the idle pool should be used). + +**Protection of Redis data** + +Tyk uses Redis to store API tokens and OAuth clients, so it is advisable to *not* treat Redis instances as ephemeral. The exception to this is when you are using Tyk Multi Data Center Bridge, but you will still need to retain the master Redis instance. + +You must ensure that Redis is persisted, or at least in a configuration where it is easy to restore or failover. So, for example, with Elasticache, making sure there are many read-replicas and regular snapshots can ensure that your data survives a failure. + +**Redis Encryption** + +Redis supports [SSL/TLS encryption](https://redis.io/topics/encryption) from version 6 as an optional feature, enhancing the security of data in transit. To configure TLS or mTLS connections between an application and Redis, consider the following settings in Tyk's configuration files: + +- `storage.use_ssl`: Set this to true to enable TLS encryption for the connection. + +- `storage.ssl_insecure_skip_verify`: A flag that, when set to true, instructs the application not to verify the Redis server's TLS certificate. This is not recommended for production due to the risk of `man-in-the-middle` attacks. + +From **Tyk 5.3**, additional options are available for more granular control: + +- `storage.ca_file`: Path to the Certificate Authority (CA) file for verifying the Redis server's certificate. + +- `storage.cert_file` and `storage.key_file`: Paths to your application's certificate and private key files, necessary for mTLS where both parties verify each other's identity. + +- `storage.max_version` and `storage.min_version`: Define the acceptable range of TLS versions, enhancing security by restricting connections to secure TLS protocols (1.2 or 1.3). + +**Setting up an Insecure TLS Connection** +- **Enable TLS**: By setting `"use_ssl": true`, you encrypt the connection. +- **Skip Certificate Verification**: Setting `"ssl_insecure_skip_verify": true` bypasses the server's certificate verification, suitable only for non-production environments. + +**Setting up a Secure TLS Connection** +- Ensure `use_ssl` is set to `true`. +- Set `ssl_insecure_skip_verify` to `false` to enforce certificate verification against the CA specified in `ca_file`. +- Specify the path to the CA file in `ca_file` for server certificate verification. +- Adjust `min_version` and `max_version` to secure TLS versions, ideally 1.2 and 1.3. + +**Setting up a Mutual TLS (mTLS) Connection** +- Follow the steps for a secure TLS connection. +- Provide paths for `cert_file` and `key_file` for your application's TLS certificate and private key, enabling Redis server to verify your application's identity. + +**Example Gateway Configuration** +```json expandable +"storage": { + "type": "redis", + "host": "server1", + "port": 6379, + "use_ssl": true, + "ssl_insecure_skip_verify": false, + "ca_file": "/path/to/ca.crt", + "cert_file": "/path/to/client.crt", + "key_file": "/path/to/client.key", + "max_version": "1.3", + "min_version": "1.2" +} +``` + +**Capping Analytics** +Tyk Gateways can generate a lot of analytics data. Be sure to read about [capping your Dashboard analytics](/api-management/tyk-pump#tyk-pump-capping-analytics-data-storage) + + +### Redis Sizing Guidelines + +The average single request analytics record (without detailed logging turned on) is around 1KB. + +In terms of Redis, in addition to key storage itself, it should be able to hold the last 10 seconds of analytics data, preferably more, in the case of a Tyk Pump failure. So if you have 100 requests per second, you will need approximately 6MB for storing 60 seconds of data. Be aware that if detailed logging is turned on, this can grow by a magnitude of 10. + + + +MDCB and Multi-Cloud clients - the Gateways write the data to a temporary Redis list and periodically send the analytics directly to the MDCB server, which, similar to Pump, processes them for purging to MongoDB or PostgreSQL. + + + +**Redis RAM Calculator** +You can calculate your Redis RAM requirements by entering your known values in the middle section of the calculator settings below: + + + + +## MongoDB + +### Supported Versions + + + +### Choose a MongoDB driver + +From Tyk 5.0.2, we added an option to use the official MongoDB Go driver to connect to MongoDB. + +We recommend using the *mongo-go* driver if you are using MongoDB 4.4.x+. For MongoDB versions prior to 4.4, please use the *mgo* driver. + +With the mongo-go driver, we support the latest versions of MongoDB (5.0.x, v6.0.x, and v7.0.x) and also features such as the "+srv" connection string and SCRAM-SHA-256. For more details, visit the MongoDB doc: +* [Connection Guide](https://www.mongodb.com/docs/drivers/go/v1.11/fundamentals/connection/) +* [Authentication Mechanisms](https://www.mongodb.com/docs/drivers/go/v1.11/fundamentals/auth/) + +You can configure which driver to use with the MongoDB driver option: +* [Configure Dashboard MongoDB driver](/tyk-dashboard/configuration#mongo_driver) +* [Configure MDCB MongoDB driver](/tyk-multi-data-centre/mdcb-configuration-options#analyticsdriver) +* [Configure Pump MongoDB driver](https://github.com/TykTechnologies/tyk-pump#driver-type) + +**Split out your DB** + +This is a no-brainer, but keep Redis and MongoDB off the system running the Gateway, they both use lots of RAM, and with Redis and the Gateway constantly communicating you will be facing resource contention on the CPU for a marginal decrease in latency. + +So in our setup, we recommend that Redis and MongoDB/PostgreSQL live on their own systems, separate from your Tyk Gateway. If you like, run them together on the same box, that's up to you. + +The network topology we like to use is: + +* Two or more Tyk Gateway nodes (load balanced, each Gateway installed on separate machines). +* A separate MongoDB or PostgreSQL cluster +* A separate Redis server with fail-over or cluster +* One Tyk Dashboard node installed on a separate machine +* One Tyk Pump node installed on a separate machine that handles data transitions + +**Special notes for DocumentDB** + + +If you are using [DocumentDB](https://aws.amazon.com/documentdb/), [capped collections](/api-management/tyk-pump#tyk-pump-capping-analytics-data-storage) are not supported. See [here](https://docs.aws.amazon.com/documentdb/latest/developerguide/mongo-apis.html) for more details. + + + +**Special notes for MongoDB Atlas** +To integrate with [MongoDB Atlas](https://www.mongodb.com/atlas/database), make sure the IP firewall connections are whitelisted on the Atlas side, and then use the following Tyk Dashboard configurations to connect: +``` +- TYK_DB_MONGOURL=mongodb://admin:password@tykdb-shard-00-00.h42pp.mongodb.net:27017,tykdb-shard-00-01.h42pp.mongodb.net:27017,tykdb-shard-00-02.h42pp.mongodb.net:27017/tyk_analytics?authSource=admin - TYK_DB_ENABLECLUSTER=false - TYK_DB_MONGOUSESSL=true +``` + +More information on these configuration variables [here](/tyk-dashboard/configuration). + + +### MongoDB Sizing Guidelines + +The aggregate record size depends on the number of APIs and Keys you have. Each counter size is ~50b, and every aggregated value has its own counter. + +So an hourly aggregate record is computed like this: 50 * active_apis + 50 * api_versions + 50 * active_api_keys + 50 * oauth_keys, etc. + +The average aggregate record size (created hourly) on our cloud is about ~ 40KB (a single record includes all the aggregate stats mentioned above). + +So for 1 million requests per day, it will generate 1KB * 1M request stats (1GB) + 24 * 40KB aggregate stats (~1MB). + +Per month: 30GB request logs + 30MB aggregate logs + +**MongoDB Working Data** + +Working data in terms of MongoDB is the data you query most often. The graphs displayed on the Tyk Dashboard, except for the Log browser, use aggregated data. + +So if you rely only on this kind of analytic data, you will not experience issues with working data and memory issues. It is literally hundreds of MBs. + +Even if you use the Log browser, its usage access is usually quite random, and it is unlikely that you check requests for every request. So it can't be called working data. And it is ok to store it on disk and allow MongoDB to do the disk lookups to fetch the data. + +Note, that in order to do fast queries, even from the disk, MongoDB uses indexes. MongoDB recommends that indexes should fit into memory, and be considered working data, but only the part of the index which is commonly used. For example the last month of data. + +For an aggregate collection, the average index size is 6% of the overall collection. For requests stats, it is around 30%. + + +**MongoDB Sizing Example** +If you serve 1 million requests per day, and require fast access to the last seven days of request logs (usually way less, and the performance of the log viewer is not a concern), with 3 months of aggregated logs, the memory requirements for MongoDB can be as follows: + +Request_logs_index ( 30% * (1GB * 7) ) + aggregated(3month * 30MB) ~= 2.1GB + 90MB = ~ 2.2GB + +In addition to storing working data in memory, MongoDB also requires space for some internal data structures. In general, multiplying the resulting number by 2x should be enough. In the above example, your MongoDB server should have around 4.4GB of available memory. + +**Audit Log storage** + +From Tyk Dashboard v5.7+,the audit log can be configured to be stored in the database. If you choose to store the audit logs in the database, you need to account for additional storage for audit logs in the database setup. The size of this table will depend on the number of operations recorded, with each record averaging 1350 to 1450 bytes. + +**Audit Log Considerations** + +- **Data Generation**: The total size of the audit log table will depend on the number of API operations, administrative actions, and system events that are being logged. +- **Daily Estimate**: For example, logging 100,000 operations per day results in 135MB to 145MB of additional data daily. +- **Storage Growth**: Over time, this can significantly impact your storage requirements, especially in high-traffic environments or systems with comprehensive logging enabled. + +**Recommendations for Housekeeping the Audit Log Table** + +1. **Implement Data Retention Policies:** + Define a clear retention period based on business and regulatory requirements, such as 30, 90, or 180 days. Remove older logs that exceed the retention policy to prevent excessive storage growth. + +2. **Archive Older Logs:** + For long-term storage or compliance purposes, move older logs to external systems such as a data lake, object storage (e.g., S3), or a data warehouse. + +3. **Monitor Growth Trends:** + Use monitoring tools to track the size and growth rate of the audit log table. Adjust retention policies or resources proactively based on observed trends. + +4. **Plan for Resource Scaling:** + Audit log storage can significantly impact overall database size, especially in high-traffic environments. Plan for storage and resource scaling based on daily log growth estimates. + +**Example Calculation:** + +- Daily Logs: 100,000 operations/day +- Average Record Size: 1400 bytes +- Storage Growth: 100,000 Γ— 1400 bytes/day = 140MB/day + +For 90 days: 140MB Γ— 90 = ~12.6GB + +**MongoDB Database Storage Calculator** +You can calculate your MongoDB storage requirements by entering your known values in the middle section of the calculator settings below: + + + + +## PostgreSQL + +How you configure your PostgreSQL installation depends on whether you are installing Tyk from fresh using PostgreSQL, or are migrating from an existing MongoDB instance. + +**Supported Versions** + + + +### Migrating from an existing MongoDB instance + +For v4.0 we have provided a migration command that will help you migrate all data from the main storage layer (APIs, Policies, Users, UserGroups, Webhooks, Certificates, Portal Settings, Portal Catalogs, Portal Pages, Portal CSS, etc.). + + + +The migration tool will not migrate any Logs, Analytics, or Uptime analytics data. + + + +1. Make sure your new SQL platform and the existing MongoDB instance are both running +2. Configure the `main` part of the `storage` section of your `tyk-analytics.conf`: + +```yaml expandable +{ +... + "storage": { + ... + "main": { + "type": "postgres", + "connection_string": "user=root password=admin database=tyk-demo-db host=tyk-db port=5432" + } + } +} +``` +3. Run the following command: + +```console +./tyk-analytics migrate-sql +``` +You will see an output listing the transfer of each database table. For example: `Migrating 'tyk_apis' collection. Records found: 7`. + +4. You can now remove your `mongo_url` (or `TYK_DB_MONGOURL` environment variable) from your `tyk-analytics.conf` +5. Restart your Tyk Dashboard + +### PostgreSQL Sizing Guidelines + +The aggregate record size depends on the number of APIs and Keys you have. Each counter size is ~50b, and every aggregated value has its own counter. + +So an hourly aggregate record is computed like this: 50 * active_apis + 50 * api_versions + 50 * active_api_keys + 50 * oauth_keys, etc. + +The average aggregate record size (created hourly) on our cloud is about ~ 40KB (a single record includes all the aggregate stats mentioned above). + +So for 1 million requests per day, it will generate 1KB * 1M request stats (1GB) + 24 * 40KB aggregate stats (~1MB). + +Per month: 30GB request logs + 30MB aggregate logs + +**PostgreSQL Database Storage Calculator** +You can calculate your PostgreSQL storage requirements by entering your known values in the middle section of the calculator settings below: + + + + diff --git a/planning-for-production/ensure-high-availability/circuit-breakers.mdx b/planning-for-production/ensure-high-availability/circuit-breakers.mdx new file mode 100644 index 00000000..ec589618 --- /dev/null +++ b/planning-for-production/ensure-high-availability/circuit-breakers.mdx @@ -0,0 +1,347 @@ +--- +title: "Circuit Breakers" +'og:description': "How to configure Tyk's circuit breaker middleware to protect your upstream services from repeated failures and overloading." +sidebarTitle: "Circuit Breakers" +--- + +## Introduction + +A circuit breaker is a protective mechanism that helps to maintain system stability by preventing repeated failures and overloading of services that are erroring. When a network or service failure occurs, the circuit breaker prevents further calls to that service, allowing the affected service time to recover while ensuring that the overall system remains functional. It is a critical component in ensuring the resilience and reliability of a distributed system. + +Tyk's circuit breaker can be configured at the endpoint level, where it monitors the rate of failure responses (HTTP 500 or higher) received from the upstream service. If that failure rate exceeds the configured threshold, the circuit breaker will trip and Tyk will block further requests to that endpoint (returning `HTTP 503 Service temporarily unavailable`) until the end of a recovery (cooldown) time period. + +Tyk can trigger events when the circuit breaker trips and when it resets. These events can be used for monitoring, alerting, or automation of recovery processes. + +Circuit breaker example + +## When to use a circuit breaker + +**Protection of critical API endpoints** + +Circuit breakers can be used to safeguard essential API endpoints from overloading, ensuring their availability and performance. By implementing circuit breakers, users can prevent these endpoints from being overwhelmed, maintaining their reliability and responsiveness. + +**Handling temporary issues** + +Circuit breakers can help handle temporary issues in the system, such as temporary outages or performance degradation, by opening and closing the circuit when conditions are unfavorable, allowing the system to recover and resume normal operation. + +**Implementing retry logic** + +Circuit breakers can be used to automatically manage the retry of failed requests after a hold-off period, increasing the chances of successful execution. + +**Implementing fallback mechanisms** + +Circuit breakers can trigger alternative workflows or fallback mechanisms when the primary system fails, ensuring uninterrupted service delivery despite system failures. + +## How the circuit breaker works + +Similarly to the circuit breaker in an electrical installation, Tyk's circuit breaker middleware monitors the flow and trips (breaks the connection) if it detects errors. Whilst the electrical circuit breaker monitors the flow of electricity and trips if it detects overcurrent (e.g. a short-circuit), Tyk's monitors the responses back from the upstream service and trips if it detects too many failures. + +From the perspective of the circuit breaker middleware, a failure is considered any response with HTTP status code `500` or above. + +The circuit breaker is rate-based, meaning that it counts the number of failure responses received in a rolling sample window and trips if the failure rate exceeds the configured threshold. + +The rolling sample window is set to 10 seconds and the circuit breaker is designed to trip only if a user-configurable minimum number of samples (requests) fail within the window period. + +Thus, if the _sample size_ is set to 100 and the _failure rate_ is set to 0.5 (50%) then the circuit breaker will trip only when there have been a minimum of 100 requests made in the past 10 seconds of which at least 50 have failed (returned an `HTTP 500` or higher error). + +Once the breaker has been tripped it will remain _open_, blocking calls to the endpoint until a configurable cooldown (or return-to-service) period has elapsed. While the breaker is _open_, requests to the endpoint will return `HTTP 503 Service temporarily unavailable`. + +### Half-open mode + +In some scenarios the upstream service might recover more quickly than the configured cooldown period. The middleware supports a _half-open_ mode that facilitates an early return-to-service so that API clients do not have to wait until the end of the cooldown before the circuit breaker is reset. + +In the _half-open_ mode, Tyk will periodically issue requests to the upstream service to check whether the path has been restored (while continuing to block client requests). If the Gateway detects that the path has been reconnected, the circuit breaker will be automatically reset (following the electrical circuit analogy, the circuit breaker is _closed_) and requests will be passed to the upstream again. + +### Configuring the circuit breaker + +The circuit breaker is configured using only three parameters: +- sample size +- error rate threshold +- cooldown period + +The threshold is a ratio of the number of failures received in the sample window. For example, if the sample window size is 100 requests and you wish to trip the circuit breaker if there are 15 failures in any 100 requests, the threshold should be set to `15/100 = 0.15`. + +The cooldown period is the time that the circuit breaker will remain _open_ after the error rate threshold has been met and the breaker has been tripped. + +There is also an option to enable or disable the _half-open_ state if this would be damaging to your system. + + + +If you are using the Service Discovery module, every time the breaker trips, Tyk will attempt to refresh the Gateway list. + + + +### Using the circuit breaker with multiple upstream hosts + +The circuit breaker works at the endpoint level independent of the number of upstream hosts are servicing the requests. Thus, if you have multiple upstream targets for an API, the sample and failure counts are accumulated across **all** upstream requests. If the failure rate exceeds the threshold, the circuit breaker will trip even if only some of your upstream hosts are failing. Operating in _half-open_ mode will of course cause the breaker to reset if a responsive upstream receives a request, but the `BreakerTripped` (or `BreakerTriggered`) event should alert you to the fact that at least one host is failing. + +### Using the circuit breaker with multiple Tyk Gateways + +Circuit breakers operate on a single Tyk Gateway, they do not centralise or pool back-end data. This ensures optimum speed of response and resilience to Gateway failure. Subsequently, in a load balanced environment where multiple Tyk Gateways are used, some traffic can spill through even after the circuit breaker has tripped on one Gateway as other Gateways continue to serve traffic to the upstream before their own breakers trip. + +### Circuit breaker events + +The circuit breaker automatically controls the flow of requests to the upstream services quickly and efficiently, but it is equally important to alert you to the fact that there is an issue and to confirm when traffic will recommence once the issue is resolved. Tyk's [Event](/api-management/gateway-events#event-categories) system provides the method by which the circuit breaker can alert you to these occurrences. + +- When the circuit breaker trips (from closed to open), Tyk will generate a `BreakerTripped` event +- When the breaker resets (from open to closed), whether at the end of the cooldown period or if connection is restored while in _half-open_ mode, Tyk will generate a `BreakerReset` event +- In addition to these, whenever the circuit breaker changes state (from closed to open or vice versa), Tyk will generate a `BreakerTriggered` event + +For the generic `BreakerTriggered` event, the state change will be indicated in the `Status` field in the webhook template as follows: +- when a breaker trips `CircuitEvent = 0` +- when a breaker resets `CircuitEvent = 1` + +### API-level circuit breaker + +Tyk does not have an API-level circuit breaker that can be applied across all endpoints. If you are using the Tyk Dashboard, however, then you are able to use an [Open Policy Agent](/api-management/dashboard-configuration#extend-permissions-using-open-policy-agent-opa) to append a circuit breaker to every API/Service using the regex `.*` path. + +
+ +If you're using Tyk OAS APIs, then you can find details and examples of how to configure the circuit breaker middleware [here](#configuring-the-circuit-breaker-in-the-tyk-oas-api-definition). + +If you're using Tyk Classic APIs, then you can find details and examples of how to configure the circuit breaker middleware [here](#configuring-the-circuit-breaker-in-the-tyk-classic-api-definition). + +{/* proposed "summary box" to be shown graphically on each middleware page + ## Circuit Breaker middleware summary + - The Circuit Breaker is an optional stage in Tyk's API Request processing chain, activated when the request is proxied to the upstream service. + - The Circuit Breaker is configured at the per-endpoint level within the API Definition and is supported by the API Designer within the Tyk Dashboard. */} + + +**Using the Circuit Breaker middleware with Tyk OAS APIs** + + +Tyk's circuit breaker middleware is configured at the endpoint level, where it monitors the rate of failure responses (HTTP 500 or higher) received from the upstream service. If that failure rate exceeds the configured threshold, the circuit breaker will trip and Tyk will block further requests to that endpoint (returning `HTTP 503 Service temporarily unavailable`) until the end of a recovery (cooldown) time period. + +When working with Tyk OAS APIs the circuit breaker is configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation). You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the legacy Tyk Classic APIs, then check out the [Tyk Classic](#configuring-the-circuit-breaker-in-the-tyk-classic-api-definition) page. + +## Configuring the Circuit Breaker in the Tyk OAS API Definition + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. Endpoint `paths` entries (and the associated `operationId`) can contain wildcards in the form of any string bracketed by curly braces, for example `/status/{code}`. These wildcards are so they are human-readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +The circuit breaker middleware (`circuitBreaker`) can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + +The `circuitBreaker` object has the following configuration: +- `enabled`: enable the middleware for the endpoint +- `threshold`: the proportion of requests that can error before the breaker is tripped, this must be a value between 0.0 and 1.0 +- `sampleSize`: the minimum number of requests that must be received during the rolling sampling window before the circuit breaker can trip +- `coolDownPeriod`: the period for which the breaker will remain _open_ after being tripped before returning to service (seconds) +- `halfOpenStateEnabled`: if set to `true` then the circuit breaker will operate in [half-open mode](#half-open-mode) once it has been tripped + +```json {hl_lines=["39-45"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-breaker", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/status/200": { + "get": { + "operationId": "status/200get", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-breaker", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-breaker/", + "strip": true + } + }, + "middleware": { + "operations": { + "status/200get": { + "circuitBreaker": { + "enabled": true, + "threshold": 0.5, + "sampleSize": 10, + "coolDownPeriod": 60, + "halfOpenStateEnabled": true + } + } + } + } + } +} +``` + +In this example Tyk OAS API Definition the circuit breaker has been configured to monitor requests to the `GET /status/200` endpoint. + +It will configure the circuit breaker so that if a minimum of 10 requests (`sampleSize`) to this endpoint are received during the [rolling sampling window](#how-the-circuit-breaker-works) then it will calculate the ratio of failed requests (those returning `HTTP 500` or above) within that window. +- if the ratio of failed requests exceeds 50% (`threshold = 0.5`) then the breaker will be tripped +- after it has tripped, the circuit breaker will remain _open_ for 60 seconds (`coolDownPeriod`) +- further requests to `GET /status/200` will return `HTTP 503 Service temporarily unavailable` +- the circuit breaker will operate in _half-open_ mode (`halfOpenStateEnabled = true`) so when the threshold has been reached and the breaker is tripped, Tyk will periodically poll the upstream service to test if it has become available again + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the circuit breaker. + +## Configuring the Circuit Breaker in the API Designer + +Adding the circuit breaker to your API endpoints is easy when using the API Designer in the Tyk Dashboard, simply follow these steps: + +**Step 1: Add an endpoint** + +From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + +Tyk OAS API Designer showing no endpoints created + +Adding an endpoint to an API using the Tyk OAS API Designer + +Tyk OAS API Designer showing no middleware enabled on endpoint + +**Step 2: Select the Circuit Breaker middleware** + +Select **ADD MIDDLEWARE** and choose the **Circuit Breaker** middleware from the *Add Middleware* screen. + +Adding the Circuit Breaker middleware + +**Step 3: Configure the middleware** + +Set the circuit breaker configuration parameters so that Tyk can protect your upstream service if it experiences failure: +- threshold failure rate for the proportion of requests that can error before the breaker is tripped (a value between 0.0 and 1.0) +- the minimum number of requests that must be received during the [rolling sampling window](#how-the-circuit-breaker-works) before the circuit breaker can trip +- the cooldown period for which the breaker will remain _open_ after being tripped before returning to service (in seconds) +- optionally enable [half-open mode](#half-open-mode) for upstream services with variable recovery times + +Configuring the circuit breaker for the endpoint + +Select **ADD MIDDLEWARE** to apply the change to the middleware configuration. + +**Step 4: Save the API** + +Select **SAVE API** to apply the changes to your API. + + +## Using the Circuit Breaker middleware with Tyk Classic APIs + + +Tyk's circuit breaker middleware is configured at the endpoint level, where it monitors the rate of failure responses (HTTP 500 or higher) received from the upstream service. If that failure rate exceeds the configured threshold, the circuit breaker will trip and Tyk will block further requests to that endpoint (returning `HTTP 503 Service temporarily unavailable`) until the end of a recovery (cooldown) time period. + +When working with Tyk Classic APIs the circuit breaker is configured in the Tyk Classic API Definition. You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the newer Tyk OAS APIs, then check out the [Tyk OAS](#configuring-the-circuit-breaker-in-the-tyk-oas-api-definition) page. + +If you're using Tyk Operator then check out the [configuring the Circuit Breaker in Tyk Operator](#configuring-the-circuit-breaker-in-tyk-operator) section below. + +## Configuring the Circuit Breaker in the Tyk Classic API Definition + +To configure the circuit breaker you must add a new `circuit_breakers` object to the `extended_paths` section of your API definition, with the following configuration: +- `path`: the endpoint path +- `method`: the endpoint HTTP method +- `threshold_percent`: the proportion of requests that can error before the breaker is tripped, this must be a value between 0.0 and 1.0 +- `samples`: the minimum number of requests that must be received during the rolling sampling window before the circuit breaker can trip +- `return_to_service_after`: the period for which the breaker will remain _open_ after being tripped before returning to service (seconds) +- `disable_half_open_state`: by default the Tyk circuit breaker will operate in [half-open mode](#half-open-mode) when working with Tyk Classic APIs, set this to `true` if you want Tyk to wait the full cooldown period before closing the circuit + +For example: +```json {linenos=true, linenostart=1} expandable +{ + "circuit_breakers": [ + { + "path": "status/200", + "method": "GET", + "threshold_percent": 0.5, + "samples": 10, + "return_to_service_after": 60, + "disable_half_open_state": false + } + ] +} +``` +In this example the circuit breaker has been configured to monitor HTTP `GET` requests to the `/status/200` endpoint. It will configure a sampling window (`samples`) of 10 requests and calculate the ratio of failed requests (those returning HTTP 500 or above) within that window. If the ratio of failed requests exceeds 50% (`threshold_percent = 0.5`) then the breaker will be tripped. After it has tripped, the circuit breaker will remain _open_ for 60 seconds (`return_to_service_after`). The circuit breaker will operate in _half-open_ mode (`disable_half_open_state = false`) so when _open_, Tyk will periodically poll the upstream service to test if it has become available again. + +When the breaker has tripped, it will return `HTTP 503 Service temporarily unavailable` in response to any calls to `GET /status/200`. + +## Configuring the Circuit Breaker in the API Designer + +You can use the API Designer in the Tyk Dashboard to configure the circuit breaker middleware for your Tyk Classic API by following these steps. + +**Step 1: Add an endpoint for the path and select the Circuit Breaker plugin** + +From the **Endpoint Designer** add an endpoint that matches the path for which you want to deploy the circuit breaker. Select the **Circuit Breaker** plugin. + +Plugin dropdown list + +**Step 2: Configure the circuit breaker** + +You can set up the various configurations options for the breaker in the drawer by clicking on it: + +Circuit breaker configuration form + +- **Trigger threshold percentage**: The percentage of requests that can error before the breaker is tripped, this must be a value between 0.0 and 1.0 +- **Sample size (requests)**: The number of samples to take for a circuit breaker window +- **Return to service in (s)**: The cool-down period of the breaker to return to service (seconds) + +**Step 3: Save the API** + +Use the *save* or *create* buttons to save the changes and activate the middleware. + +**Step 4: Optionally configure webhooks to respond to the circuit breaker events** + +The Dashboard supports the separate `BreakerTripped` and `BreakerReset` events, but not the combined `BreakerTriggered` [event type](/api-management/gateway-events#event-types). You should use **API Designer > Advanced Options** to add a Webhook plugin to your endpoint for each event. + +Webhook events + +## Configuring the Circuit Breaker in Tyk Operator + +The example API Definition below configures an API to listen on path `/httpbin-timeout-breaker` and forwards requests upstream to http://httpbin.org. A hard timeout value of 2 seconds is configured for path `/delay/{delay_seconds}`. This will return a `504 Gateway Timeout` response to the client if the upstream response is not received before expiry of the timer. + +```yaml {linenos=true, linenostart=1, hl_lines=["30-35"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-timeout-breaker +spec: + name: httpbin-timeout-breaker + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin-timeout-breaker + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + hard_timeouts: + - method: GET + path: /delay/{delay_seconds} + timeout: 2 + circuit_breakers: + - method: GET + path: /status/500 + return_to_service_after: 10 + samples: 4 + threshold_percent: "0.5" +``` + +A circuit breaker has been configured to monitor `HTTP GET` requests to the `/status/500` endpoint. It will configure a sampling window (samples) of 4 requests and calculate the ratio of failed requests (those returning HTTP 500 or above) within that window. If the ratio of failed requests exceeds 50% (threshold_percent = 0.5) then the breaker will be tripped. After it has tripped, the circuit breaker will remain open for 10 seconds (return_to_service_after). The circuit breaker will operate using the default half-open mode so when open, Tyk will periodically poll the upstream service to test if it has become available again. + +When the breaker has tripped, it will return HTTP 503 Service temporarily unavailable in response to any calls to GET /status/500. + diff --git a/planning-for-production/ensure-high-availability/enforced-timeouts.mdx b/planning-for-production/ensure-high-availability/enforced-timeouts.mdx new file mode 100644 index 00000000..48920cac --- /dev/null +++ b/planning-for-production/ensure-high-availability/enforced-timeouts.mdx @@ -0,0 +1,279 @@ +--- +title: "Enforced Timeouts" +'og:description': "Learn how to use Tyk's Enforced Timeout middleware to manage upstream service response times, ensuring system stability and optimal performance." +sidebarTitle: "Enforced Timeouts" +--- + +## Introduction + +In any system, a task or operation takes a certain period of time to complete. When a client makes a request to the Tyk Gateway, it will be dependent upon the responsiveness of the upstream service before it can continue. If the upstream service is suffering from resource overload or congestion the response may be returned too late leading to unacceptable experience for the end user or even to instability in the system. + +Tyk's Enforced Timeout middleware can be used to apply a maximum time that the Gateway will wait for a response before it terminates (or times out) the request. If the timeout expires, then Tyk will notify the client with an `HTTP 504 Gateway Timeout` error. + +This feature helps to maintain system stability and prevents unresponsive or long-running tasks from affecting the overall performance of the system. The enforced timeout can be customized and configured to suit specific requirements, providing control over resource allocation and ensuring optimal system functionality. + +## When to use an enforced timeout + +**Resource management** + +The enforced timeout can be implemented to manage system resources efficiently, particularly in high-traffic environments, preventing long-running tasks from monopolising resources, ensuring fair distribution and optimal performance. + +**Task prioritization** + +Prioritizing critical tasks by setting timeouts based on their expected time-to-complete helps to ensure that essential tasks are completed by reducing the impact of non-responsive upstream services. + +**Security measures** + +Limiting task durations can help protect against potential security breaches or malicious activities by setting time constraints on user sessions or API requests. + +**Time-sensitive operations** + +For time-sensitive tasks, enforced timeouts can guarantee timely completion and avoid delays or missed deadlines. + +## How the enforced timeout middleware works + +The enforced timeout middleware is enabled and configured at the endpoint level. + +The configuration is very simple, the only option being the duration of the timeout (which is declared in seconds) after which the upstream request will be terminated and an `HTTP 504 Gateway Timeout` error returned to the client. + + + +If you are using the Service Discovery option, if an enforced timeout is triggered, the service discovery module will refresh the host / host list. + + + +
+ +If you're using Tyk OAS APIs, then you can find details and examples of how to configure the enforced timeout middleware [here](#using-the-enforced-timeout-middleware-with-tyk-oas-apis). + +If you're using Tyk Classic APIs, then you can find details and examples of how to configure the enforced timeout middleware [here](#using-the-enforced-timeout-middleware-with-tyk-classic-apis). + +{/* proposed "summary box" to be shown graphically on each middleware page + ## Enforced Timeout middleware summary + - The Enforced Timeout is an optional stage in Tyk's API Request processing chain, activated when the request is proxied to the upstream service. + - The Enforced Timeout is configured at the per-endpoint level within the API Definition and is supported by the API Designer within the Tyk Dashboard. */} + + +## Using the Enforced Timeout middleware with Tyk OAS APIs + + +Tyk's [enforced timeout](/planning-for-production/ensure-high-availability/circuit-breakers) middleware is configured at the endpoint level, where it sets a limit on the response time from the upstream service. If the upstream takes too long to respond to a request, Tyk will terminate the request and return `504 Gateway Timeout` to the client. + +When working with Tyk OAS APIs the enforced timeout is configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation). You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the legacy Tyk Classic APIs, then check out the [Tyk Classic](#using-the-enforced-timeout-middleware-with-tyk-classic-apis) page. + +**Configuring an enforced timeout in the Tyk OAS API Definition** + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. Endpoint `paths` entries (and the associated `operationId`) can contain wildcards in the form of any string bracketed by curly braces, for example `/status/{code}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +The enforced timeout middleware (`enforceTimeout`) can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + +The `enforceTimeout` object has the following configuration: +- `enabled`: enable the middleware for the endpoint +- `value`: the duration of the upstream request timer + +For example: +```json {hl_lines=["39-41"],linenos=true, linenostart=1} expandable +{ + "components": {}, + "info": { + "title": "example-timeout", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/status/200": { + "get": { + "operationId": "status/200get", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-timeout", + "state": { + "active": true + } + }, + "upstream": { + "url": "http://httpbin.org/" + }, + "server": { + "listenPath": { + "value": "/example-timeout/", + "strip": true + } + }, + "middleware": { + "operations": { + "status/200get": { + "enforceTimeout": { + "enabled": true, + "value": 3 + } + } + } + } + } +} + ``` + +In this example Tyk OAS API definition, the enforced timeout has been configured to monitor requests to the `GET /status/200` endpoint. It will configure a timer that will expire (`timeout`) 3 seconds after the request is proxied to the upstream service. If the upstream response is not received before the expiry of the timer, that request will be terminated and Tyk will return `504 Gateway Timeout` to the client. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the enforced timeout. + +**Configuring an enforced timeout in the API Designer** + +Adding the enforced timeout to your API endpoints is easy when using the API Designer in the Tyk Dashboard, simply follow these steps: + +**Step 1: Add an endpoint** + +From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + +Tyk OAS API Designer showing no endpoints created + +Adding an endpoint to an API using the Tyk OAS API Designer + +Tyk OAS API Designer showing no middleware enabled on endpoint + +**Step 2: Select the Enforce Timeout middleware** + +Select **ADD MIDDLEWARE** and choose the **Enforce Timeout** middleware from the *Add Middleware* screen. + +Adding the Enforce Timeout middleware + +**Step 3: Configure the middleware** + +Set the timeout duration that you wish to enforce for requests to the endpoint. + +Configuring the enforced timeout for the endpoint + +Select **ADD MIDDLEWARE** to apply the change to the middleware configuration. + +**Step 4: Save the API** + +Select **SAVE API** to apply the changes to your API. + + +## Using the Enforced Timeout middleware with Tyk Classic APIs + + +Tyk's [enforced timeout](/planning-for-production/ensure-high-availability/circuit-breakers) middleware is configured at the endpoint level, where it sets a limit on the response time from the upstream service. If the upstream takes too long to respond to a request, Tyk will terminate the request and return `504 Gateway Timeout` to the client. + +When working with Tyk Classic APIs the enforced timeout is configured in the Tyk Classic API Definition. You can do this via the Tyk Dashboard API or in the API Designer. + +If you're using the newer Tyk OAS APIs, then check out the [Tyk OAS](#using-the-enforced-timeout-middleware-with-tyk-oas-apis) page. + +If you're using Tyk Operator then check out the [configuring an enforced timeout in Tyk Operator](#configuring-an-enforced-timeout-in-tyk-operator) section below. + +**Configuring an enforced timeout in the Tyk Classic API Definition** + +To configure an enforced timeout you must add a new `hard_timeouts` object to the `extended_paths` section of your API definition. + +It has the following configuration: +- `path`: the endpoint path +- `method`: the endpoint HTTP method +- `timeout`: the duration of the upstream request timer + +For example: +```json +{ + "hard_timeouts": [ + { + "path": "/status/200", + "method": "GET", + "timeout": 3 + } + ] +} +``` expandable + +In this example the enforced timeout has been configured to monitor requests to the `GET /status/200` endpoint. It will configure a timer that will expire (`timeout`) 3 seconds after the request is proxied to the upstream service. + +If the upstream response is not received before the expiry of the timer, that request will be terminated and Tyk will return `504 Gateway Timeout` to the client. + +**Configuring an enforced timeout in the API Designer** + +You can use the API Designer in the Tyk Dashboard to configure the enforced timeout middleware for your Tyk Classic API by following these steps. + +**Step 1: Add an endpoint for the path and select the Enforced Timeout plugin** + +From the **Endpoint Designer** add an endpoint that matches the path for which you want to deploy the enforced timeout. Select the **Enforced timeout** plugin. + +Plugin dropdown + +**Step 2: Configure the timeout** + +Then enter the timeout to be enforced for the endpoint (in seconds): + +Enforced timeout configuration + +**Step 3: Save the API** + +Use the *save* or *create* buttons to save the changes and activate the middleware. + +## Configuring an enforced timeout in Tyk Operator + +The process for configuring the middleware in Tyk Operator is similar to that explained in [configuring an enforced timeout in the Tyk Classic API Definition](#using-the-enforced-timeout-middleware-with-tyk-classic-apis). It is possible to configure an enforced timeout using the `hard_timeouts` object within the `extended_paths` section of the API Definition. + +The example API Definition below configures an API to listen on path `/httpbin-timeout-breaker` and forwards requests upstream to http://httpbin.org. A hard timeout value of 2 seconds is configured for path `/delay/{delay_seconds}`. This will return a `504 Gateway Timeout` response to the client if the upstream response is not received before expiry of the timer. + +```yaml {linenos=true, linenostart=1, hl_lines=["26-29"]} +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: httpbin-timeout-breaker +spec: + name: httpbin-timeout-breaker + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /httpbin-timeout-breaker + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + hard_timeouts: + - method: GET + path: /delay/{delay_seconds} + timeout: 2 + circuit_breakers: + - method: GET + path: /status/500 + return_to_service_after: 10 + samples: 4 + threshold_percent: "0.5" # Tyk Dashboard API doesn't support strings. +``` + +We can test the example using the curl command as shown below: + +```bash +curl http://localhost:8081/httpbin-timeout/delay/3 -i + HTTP/1.1 504 Gateway Timeout +Content-Type: application/json +X-Generator: tyk.io +Date: Fri, 09 Aug 2024 07:43:48 GMT +Content-Length: 57 + +{ + "error": "Upstream service reached hard timeout." +} +``` + diff --git a/planning-for-production/ensure-high-availability/health-check.mdx b/planning-for-production/ensure-high-availability/health-check.mdx new file mode 100644 index 00000000..9a708b27 --- /dev/null +++ b/planning-for-production/ensure-high-availability/health-check.mdx @@ -0,0 +1,243 @@ +--- +title: "Liveness Health Checks" +'og:description': "How to set up liveness health checks for the Tyk Gateway to ensure high availability and monitor the status of components like Redis, Dashboard, and RPC." +sidebarTitle: "Health Checks" +--- + +## Set Up Liveness Health Checks + +Health checks are extremely important in determining the status of an +application - in this instance, the Tyk Gateway. Without them, it can be hard to +know the actual state of the Gateway. + +Depending on your configuration, the Gateway could be using a few components: + +- The Tyk Dashboard. +- RPC +- Redis (compulsory). + +Any of these components could go down at any given point and it is useful to +know if the Gateway is currently usable or not. A good usage of the health +check endpoint is for the configuration of a load balancer to multiple instances of the Gateway or +as a Kubernetes liveness probe. + +The following component status will not be returned: + +* MongDB or SQL +* Tyk Pump + + + + + Health check is implemented as per the [Health Check Response Format for HTTP APIs](https://inadarei.github.io/rfc-healthcheck/) RFC + + + +An example of the response from this API is as follows: + + +```yaml expandable +{ + "status": "pass", + "version": "v3.1.1", + "description": "Tyk GW", + "details": { + "redis": { + "status": "pass", + "componentType": "datastore", + "time": "2020-05-19T03:42:55+01:00" + }, + "dashboard": { + "status": "pass", + "componentType": "system", + "time": "2020-05-19T03:42:55+01:00" + }, + "rpc": { + "status": "pass", + "componentType": "system", + "time": "2020-05-19T03:42:55+01:00" + } + } +} +``` + +## Status Levels + +The following status levels can be returned in the JSON response. + +- **pass**: Indicates that all components required for the Gateway to work 100% are available, and there is no impact on your traffic. + +- **warn**: Indicates that one of the components is having an outage but your Gateway is able to keep processing traffic. The impact is medium (i.e. no quotas are applied, no analytics, no RPC connection to MDCB). + +- **fail**: Indicates that Redis AND the Tyk Dashboard are unavailable, and can and indicate other failures. The impact is high (i.e. no configuration changes are available for API/policies/keys, no quotas are applied, and no analytics). + +## Configure health check + +By default, the liveness health check runs on the `/hello` path. But +it can be configured to run on any path you want to set. For example: + + +```yaml +health_check_endpoint_name: "status" +``` + +This configures the health check to run on `/status` instead of `/hello`. + +**Refresh Interval** + +The Health check endpoint will refresh every 10 seconds. + +**HTTP error code** +The Health check endpoint will always return a `HTTP 200 OK` response if the polled health check endpoint is available on your Tyk Gateway. If `HTTP 200 OK` is not returned, your Tyk Gateway is in an error state. + + +For MDCB installations the `/hello` endpoint can be polled in either your Management or Worker Gateways. It is recommended to use the `/hello` endpoint behind a load balancer for HA purposes. + +## Health check examples + +The following examples show how the Health check endpoint returns + + +### Pass Status + +The following is returned for a `pass` status level for the Open Source Gateway: + +``` expandable +$ http :8080/hello +HTTP/1.1 200 OK +Content-Length: 156 +Content-Type: application/json +Date: Wed, 14 Apr 2021 17:36:09 GMT + +{ + "description": "Tyk GW", + "details": { + "redis": { + "componentType": "datastore", + "status": "pass", + "time": "2021-04-14T17:36:03Z" + } + }, + "status": "pass", + "version": "v3.1.1" +} +``` + +### Redis outage + +``` expandable +$ http :8080/hello +HTTP/1.1 200 OK +Content-Length: 303 +Content-Type: application/json +Date: Wed, 14 Apr 2021 14:58:06 GMT + +{ + "description": "Tyk GW", + "details": { + "dashboard": { + "componentType": "system", + "status": "pass", + "time": "2021-04-14T14:58:03Z" + }, + "redis": { + "componentType": "datastore", + "output": "storage: Redis is either down or was not configured", + "status": "fail", + "time": "2021-04-14T14:58:03Z" + } + }, + "status": "warn", + "version": "v3.1.2" +} +``` + +### Dashboard outage + +``` expandable +$ http :8080/hello +HTTP/1.1 200 OK +Content-Length: 292 +Content-Type: application/json +Date: Wed, 14 Apr 2021 15:52:47 GMT + +{ + "description": "Tyk GW", + "details": { + "dashboard": { + "componentType": "system", + "output": "dashboard is down? Heartbeat is failing", + "status": "fail", + "time": "2021-04-14T15:52:43Z" + }, + "redis": { + "componentType": "datastore", + "status": "pass", + "time": "2021-04-14T15:52:43Z" + } + }, + "status": "warn", + "version": "v3.1.2" +} +``` +### Dashboard and Redis outage + +``` expandable +$ http :8080/hello +HTTP/1.1 200 OK +Content-Length: 354 +Content-Type: application/json +Date: Wed, 14 Apr 2021 17:53:33 GMT + +{ + "description": "Tyk GW", + "details": { + "dashboard": { + "componentType": "system", + "output": "dashboard is down? Heartbeat is failing", + "status": "fail", + "time": "2021-04-14T17:53:33Z" + }, + "redis": { + "componentType": "datastore", + "output": "storage: Redis is either down or was not configured", + "status": "fail", + "time": "2021-04-14T17:53:33Z" + } + }, + "status": "fail", + "version": "v3.1.2" +} +``` + + +### MDCB Worker Gateway RPC outage + +``` expandable +$ http :8080/hello +HTTP/1.1 200 OK +Content-Length: 333 +Content-Type: application/json +Date: Wed, 14 Apr 2021 17:21:24 GMT + +{ + "description": "Tyk GW", + "details": { + "redis": { + "componentType": "datastore", + "output": "storage: Redis is either down or was not configured", + "status": "fail", + "time": "2021-04-14T17:21:16Z" + }, + "rpc": { + "componentType": "system", + "output": "Could not connect to RPC", + "status": "fail", + "time": "2021-04-14T17:21:16Z" + } + }, + "status": "fail", + "version": "v3.1.2" +} +``` + diff --git a/planning-for-production/ensure-high-availability/load-balancing.mdx b/planning-for-production/ensure-high-availability/load-balancing.mdx new file mode 100644 index 00000000..94bfdb6e --- /dev/null +++ b/planning-for-production/ensure-high-availability/load-balancing.mdx @@ -0,0 +1,81 @@ +--- +title: "Load Balancing" +'og:description': "How to set up round-robin load balancing in Tyk Gateway and Dashboard, including dynamic load balancing with service discovery." +sidebarTitle: "Load Balancing" +--- + +## Introduction + +Tyk supports native round-robin load-balancing in its proxy. This means that Tyk will rotate requests through a list of target hosts as requests come in. This can be very useful in microservice architectures where clusters of specialized services are launched for high availability. + +Setting up load balancing is done on a per API basis, and is defined in the API Definition file/object: + +* `proxy.enable_load_balancing`: Set this value to `true` to have a Tyk node distribute traffic across a list of servers. + +* `proxy.target_list`: A list of upstream targets (can be one or many hosts): + +```yaml expandable +"target_list": [ + "http://10.0.0.1", + "http://10.0.0.2", + "http://10.0.0.3", + "http://10.0.0.4" +] +``` + + +You must fill in the `target_list` section. + + + + +See [Service Discovery](/planning-for-production/ensure-high-availability/service-discovery) to see how you can integrate a service discovery system such as Consul or etcd with Tyk and enable dynamic load balancing support. + +## Configure load balancing and weighting via the Dashboard + +To set up load balancing via the Dashboard, from the **Core Settings** tab in the **API Designer** select **Enable round-robin load balancing** from the **API Settings** options: + +Dashboard load balancing configuration + +You can now add your Load Balancing **Upstream targets** and apply weighting to it. For example, for testing purposes, you can send 10% (set weighting to `1`) of traffic to a beta environment, and 90% (set weighting to `9`)to the production environment. + + + +Weighting is new from v1.10 of the Dashboard + + + +## Configure load balancing via Tyk Operator + +Load balancing is configured within Tyk Operator, using the following configuration parameters within the proxy configuration block: + +- `enable_load_balancing`: Set to `true` to activate load balancing +- `target_list`: Specifies a list of upstream targets + +An example is shown below: + +```yaml {linenos=table,hl_lines=["14-17"],linenostart=1} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: enable-load-balancing-rr +spec: + name: enable-load-balancing-rr + use_keyless: true + protocol: http + active: true + proxy: + target_url: httpbin.org + listen_path: /httpbin + strip_listen_path: true + enable_load_balancing: true + target_list: + - "httpbin.org" + - "httpbingo.org" +``` + +## gRPC load balancing + +You can also perform [gRPC Load balancing](/key-concepts/grpc-proxy#grpc-load-balancing). + + diff --git a/planning-for-production/ensure-high-availability/service-discovery.mdx b/planning-for-production/ensure-high-availability/service-discovery.mdx new file mode 100644 index 00000000..7bdee1b1 --- /dev/null +++ b/planning-for-production/ensure-high-availability/service-discovery.mdx @@ -0,0 +1,285 @@ +--- +title: "Service Discovery" +'og:description': "How to configure service discovery in Tyk" +sidebarTitle: "Service Discovery" +--- + +## Introduction + +Service Discovery is a very useful feature for when you have a dynamically changing upstream service set. + +For example, you have ten Docker containers that are running the same service, and you are load balancing between them, if one or more fail, a new service will spawn but probably on a different IP address. + +Now the Gateway would need to either be manually reconfigured, or, more appropriately, detect the failure and reconfigure itself. + +This is what the service discovery module does. + +We recommend using the SD module in conjunction with the circuit breaker features, as this makes detection and discovery of failures at the gateway level much more dynamic and responsive. + +## Service Discovery: Dashboard + +The Service Discovery settings are located in the Core tab from the API Designer. + +Service discovery + +**Configuring Service Discovery via the Dashboard** + +Select **Enable service discovery** to enable the discovery module. + +Once enabled, you will have all the options to configure your Service Discovery endpoints: + +Service discovery configuration + +The settings are as follows: + +* **Service discovery options**: New Functionality + +* **Query endpoint**: The endpoint to call, this would probably be your Consul, etcd or Eureka K/V store. + +* **Target path**: Use this setting to set a target path to append to the discovered endpoint, since many SD services only provide host and port data, it is important to be able to target a specific resource on that host, setting this value will enable that. + +* **Data path**: The namespace of the data path, so for example if your service responds with: + +``` expandable +{ + "action": "get", + "node": { + "key": "/services/single", + "value": "http://httpbin.org:6000", + "modifiedIndex": 6, + "createdIndex": 6 + } +} +``` + + Then your namespace would be `node.value`. + +* **Are the values nested?**: Sometimes the data you are retrieving is nested in another JSON object, e.g. this is how etcd responds with a JSON object as a value key: + +``` expandable +{ + "action": "get", + "node": { + "key": "/services/single", + "value": "{"hostname": "http://httpbin.org", "port": "80"}", + "modifiedIndex": 6, + "createdIndex": 6 + } +} +``` + +In this case, the data actually lives within this string-encoded JSON object, so in this case, you set the value to `checked`, and use a combination of the **data path** and **parent data path** (below). + +* **Parent data path**: This is the namespace of the where to find the nested value, in the above example, it would be `node.value`. You would then change the **data path** setting to be `hostname`, since this is where the hostname data resides in the JSON string. Tyk automatically assumes that the data_path in this case is in a string-encoded JSON object and will try to deserialise it. + +Tyk will decode the JSON string and then apply the **data path** namespace to that object in order to find the value. + +* **Port data path**: In the above nested example, we can see that there is a separate `port` value for the service in the nested JSON. In this case you can set the **port data path** value and Tyk will treat **data path** as the hostname and zip them together (this assumes that the hostname element does not end in a slash or resource identifier such as `/widgets/`). + +In the above example, the **port data path** would be `port`. + +* **Is port information separate from the hostname?**: New Functionality + +* **Does the endpoint provide a target list?**: If you are using load balancing, set this value to true and Tyk will treat the data path as a list and inject it into the target list of your API Definition. + +* **Cache timeout**: Tyk caches target data from a discovery service, in order to make this dynamic you can set a cache value when the data expires and new data is loaded. Setting it too low will cause Tyk to call the SD service too often, setting it too high could mean that failures are not recovered from quickly enough. + + +## Service Discovery Config: API Definition + +Service discovery is configured on a per-API basis, and is set up in the API Object under the `proxy` section of your API Definition: + +```yaml expandable +enable_load_balancing: true, +service_discovery: { + use_discovery_service: true, + query_endpoint: "http://127.0.0.1:4001/v2/keys/services/multiobj", + use_nested_query: true, + parent_data_path: "node.value", + data_path: "array.hostname", + port_data_path: "array.port", + use_target_list: true, + cache_timeout: 10, + target_path: "/append-this-api-path/" +}, +``` + +Settings are as follows: + +* `service_discovery.use_discovery_service`: Set this to `true` to enable the discovery module. +* `service_discovery.query_endpoint`: The endpoint to call, this would probably be your Consul, etcd or Eureka K/V store. +* `service_discovery.data_path`: The namespace of the data path so, for example, if your service responds with: + +``` expandable +{ + "action": "get", + "node": { + "key": "/services/single", + "value": "http://httpbin.org:6000", + "modifiedIndex": 6, + "createdIndex": 6 + } +} +``` + +Then your namespace would be `node.value`. + +* `service_discovery.use_nested_query`: Sometimes the data you are retrieving is nested in another JSON object, e.g. this is how etcd responds with a JSON object as a value key: + +``` expandable +{ + "action": "get", + "node": { + "key": "/services/single", + "value": "{"hostname": "http://httpbin.org", "port": "80"}", + "modifiedIndex": 6, + "createdIndex": 6 + } +} +``` +In this case, the data actually lives within this string-encoded JSON object, so in this case, you set the `use_nested_query` to `true`, and use a combination of the `data_path` and `parent_data_path` (below) + +* `service_discovery.parent_data_path`: This is the namespace of the where to find the nested value, in the above example, it would be `node.value`. You would then change the `data_path` setting to be `hostname`, since this is where the host name data resides in the JSON string. Tyk automatically assumes that the `data_path` in this case is in a string-encoded JSON object and will try to deserialise it. + +Tyk will decode the JSON string and then apply the `data_path` namespace to that object in order to find the value. + +* `service_discovery.port_data_path`: In the above nested example, we can see that there is a separate `port` value for the service in the nested JSON. In this case you can set the `port_data_path` value and Tyk will treat `data_path` as the hostname and zip them together (this assumes that the hostname element does not end in a slash or resource identifier such as `/widgets/`). + +In the above example, the `port_data_path` would be `port`. + +* `service_discovery.use_target_list`: If you are using load balancing, set this value to `true` and Tyk will treat the data path as a list and inject it into the target list of your API Definition. + +* `service_discovery.cache_timeout`: Tyk caches target data from a discovery service, in order to make this dynamic you can set a cache value when the data expires and new data is loaded. Setting it too low will cause Tyk to call the SD service too often, setting it too high could mean that failures are not recovered from quickly enough. + +* `service_discovery.target_path`: Use this setting to set a target path to append to the discovered endpoint, since many SD services only provide host and port data, it is important to be able to target a specific resource on that host, setting this value will enable that. + +## Service Discovery Examples + +**Mesosphere Example** + +For integrating service discovery with Mesosphere, you can use the following configuration parameters: + +```yaml expandable + isNested = false + isTargetList = true + endpointReturnsList = false + portSeperate = true + dataPath = "host" + parentPath = "tasks" + portPath = "ports" +``` + +**Eureka Example** + +For integrating service discovery with Eureka, you can use the following configuration parameters (this assumes that the endpoint will return JSON and not XML, this is achieved by creating an API Definition that injects the header that requests the data type and using this API Definition as the endpoint): + +```yaml expandable + isNested = false + isTargetList = true + endpointReturnsList = false + portSeperate = true + dataPath = "hostName" + parentPath = "application.instance" + portPath = "port.$" +``` + +**Etcd Example** + +For integrating with etcd, you can use the following configurations: + +```yaml expandable + isNested = false + isTargetList = false + endpointReturnsList = false + portSeperate = false + dataPath = "node.value" + parentPath = "" + portPath = "" +``` + +**Zookeeper Example** + +For this, you need to spin up a REST server that communicates with the Zookeeper instance. +Here is one open source project, ZooREST, that does just that: https://github.com/Difrex/zoorest + +With Zookeeper and ZooREST running, test the query endpoint. Don't forget the `+json`: +```curl expandable +$ curl http://zoorest:8889/v1/get/zk_tyk+json +{ + "data": { + "path": "httpbin.org" + }, + "error": "", + "path": "/zk_tyk", + "state": "OK", + "zkstat": {} +} +``` + +Then, you can use the following Tyk SD configurations: +```yaml expandable + isNested = false + isTargetList = false + endpointReturnsList = false + portSeperate = false + dataPath = "data.path" + parentPath = "" + portPath = "" +``` + +**Consul Example** + +For integrating service discovery with Consul, you can use the following configuration parameters: + +```yaml expandable + isNested = false + isTargetList = true + endpointReturnsList = true + portSeperate = true + dataPath = "Address" + parentPath = "" + portPath = "ServicePort" +``` + +**Linkerd Example** + + + +This configuration is a Tyk Community Contribution. + + + +To integrate Tyk with Linkerd perform the following: + +**Configure Linkerd** + +To integrate with Linkerd, you need to add the following configuration to your `linkerd.yaml` file, located in the `config/` directory: + +```yaml + routers: + - protocol: http + identifier: + kind: io.l5d.header.token + header: Custom-Header +``` + +Then, in your Tyk Dashboard: + +1. Select your API from the **System Management > APIs** section and click **Edit**. + +2. From the **Core Settings** tab, set the **Target URL** to the Linkerd HTTP server `host:port` address. + +3. From the **Endpoint Designer** tab click **Global Version Settings** and enter `Custom-Header` in the **Add this header**: field and the value of the Linkerd `app-id` in the **Header value** field. + +4. Click **Update** to save your changes. + +This is needed since Tyk appends a "Host" header when proxying the request and the "Host" header is also the default header expected by Linkerd. + +**For further Linkerd information, see:** + +[Linkerd - HTTP proxy documentation](https://linkerd.io/2-edge/reference/proxy-configuration/) (Alternatives Section) + +[Linkered - Header Token Identifier documentation](https://linkerd.io/config/0.9.1/linkerd/index.html#header-token-identifier) + +[The original Community Contribution](https://community.tyk.io/t/using-tyk-gateway-with-linkerd/1568) + diff --git a/planning-for-production/ensure-high-availability/uptime-tests.mdx b/planning-for-production/ensure-high-availability/uptime-tests.mdx new file mode 100644 index 00000000..47fd8ed7 --- /dev/null +++ b/planning-for-production/ensure-high-availability/uptime-tests.mdx @@ -0,0 +1,158 @@ +--- +title: "Uptime Tests" +'og:description': "How to configure uptime tests" +sidebarTitle: "Uptime Tests" +--- + +## Introuduction + +As of v1.9 Tyk supports a kind of built-in "uptime awareness" of the underlying services and hosts that it is managing traffic for by actively polling user-defined endpoints at set intervals. + +Tyk uptime awareness is not meant to replace traditional uptime monitoring tools, in fact, it is designed to supplement them by offering a way to bypass unhealthy nodes when they are down as part of Tyk's role as an API Gateway. + +### Compatibility + +Uptime tests are only available for Tyk Self-Managed users. It is not available on Tyk Cloud. + +### How do the uptime tests work? + +When uptime tests are added into a Tyk cluster, a single Gateway will elect itself as primary. Gateways remain as primary using a dead man's switch, by keeping a key active in Redis. Primaries are re-elected or confirmed every few seconds. If one Gateway stops or fails, another can detect the failure and elect itself as the primary. + +The primary Gateway will then run the uptime tests allocated to the cluster (shard group). + +The Gateway running the uptime test will have a worker pool defined so that it can execute tests simultaneously every few seconds determined by a Gateway-configurable interval loop. Depending on how many uptime tests are being run, this worker pool should be increased or decreased as needed. + + +## Initial configuration + +To configure uptime tests, add the relevant section to your `tyk.conf`: + +```yaml expandable +"uptime_tests": { + "disable": false, // disable uptime tests on the node completely + "poller_group":"", + "config": { + "enable_uptime_analytics": true, + "failure_trigger_sample_size": 3, + "time_wait": 300, + "checker_pool_size": 50 + } +} +``` + +* `disable`: When set to `false` this tells Tyk to run uptime tests, if you do not want any uptime tests to run on a Gateway, set it to `true` and they will be disabled on those Gateways (this could be useful if you are running uptime tests in a separate group of Tyk instances). +* `poller_group`: This field is used to define a different group of uptime tests. All the gateways that have the same `poller_group`, will be candidates to be elected as the primary Gateway of its group. This could be useful if you are running uptime tests in a segmented or sharded group of Tyk instances. +* `enable_uptime_analytics`: Tyk supports the recording of the data that is generated by the uptime tests, these can then be tabulated in the dashboard. Set to `true` to enable uptime analytics. However, since uptime tests run constantly, they can generate large amounts of data, for some users who do not wish to manage this data, they can disable it by setting this value to `false`. +* `failure_trigger_sample_size`: Here we tell Tyk to trigger a `HostDown` or `HostUp` event after the test has failed or passed a set number of times; `3` in this example. Setting the number to higher values can protect against false positives, but can increase lead incident time due to the verification. +* `time_wait`: The number of seconds between running tests. In this example, it is set to `300` seconds. +* `checker_pool_size`: The worker pool for uptime tests. In this example we have configured Tyk to keep `50` idle workers in memory to send tests to, in other words, with this configuration, you are pretty much guaranteed asynchronous testing for up to 50 tests. + +## Configure with the API Definition + +Uptime test checklists sit within API configurations, so in your API Definition add a section for the tests: + +```yaml expandable +uptime_tests: { + check_list: [ + { + "url": "http://google.com/" + }, + { + "url": "http://posttestserver.com/post.php?dir=uptime-checker", + "method": "POST", + "headers": { + "this": "that", + "more": "beans" + }, + "body": "VEhJUyBJUyBBIEJPRFkgT0JKRUNUIFRFWFQNCg0KTW9yZSBzdHVmZiBoZXJl" + } + ] +}, +``` + +Uptime tests are not versioned. + +In the above example, there are two forms for the Uptime test, a "quick" form, which assumes a GET request: + +``` +{ + "url": "http://google.com/" +} +``` + +Or a long form, which allows for a full request to be checked or mocked: + +``` expandable +{ + "url": "http://posttestserver.com/post.php?dir=tyk-checker-target-test&beep=boop", + "method": "POST", + "headers": { + "this": "that", + "more": "beans" + }, + "body": "VEhJUyBJUyBBIEJPRFkgT0JKRUNUIFRFWFQNCg0KTW9yZSBzdHVmZiBoZXJl", + "timeout": 1000 +} +``` + +* `body`: The `body` is Base64 encoded. +* `timeout`: The `timeout` in milli seconds. + + +## Configure with the Dashboard + +To add an uptime test using the dashboard is very simple. Make sure that you have fulfilled the prerequisite configuration in your Gateway to enable the tester. + +**Step 1: Select the Uptime Tests tab** + +From the API Designer select the **Uptime Tests** tab: + +Uptime tests tab location + +**Step 2: Click Add** + +Click **Add** to add a test: + +Add button location + +**Step 3: Enter Path Details** + +From the **Path Details** pop-up, complete the details and click **Add** to add the test: + +Test details form and add button location + +## Events + +**Uptime and downtime events** + +When Tyk detects downtime on a test, it will trigger a system event called `HostDown`, when that host returns to service, it will trigger `HostUp`. These events are triggered in the same event system as other Tyk Events, and therefore can have any kind of action performed: + +* A logger action +* A web hook to a notification URL +* A custom JS script for complex API interactions + +Please see the section on the event system on how to add event handlers to your API Definition. + +Since tests are on a URL-by-URL basis, you could potentially see multiple `HostDown` events for a single host where multiple endpoints are being tested. + +## Load balancing and Service Discovery + +**Downtime detection and service availability** + +If you have configured Tyk to use round-robin load balancing, you can enable an option in the `proxy` section of your API Definition that will check the hostname of the outbound Tyk request (to your service) against the downtime list to see if the server is active, if the host is marked as "down" Tyk will skip to the next host in its list before making the request: + +Note, the fully qualified host, including the port, needs to be exactly the same between the uptime test config and the RRLB entry in order for Tyk to link the two together. + +ie: `www.myapi.com:3000` + +``` expandable +... +"proxy": { + ... + "check_host_against_uptime_tests": true, + ... +} +... +``` + + diff --git a/planning-for-production/monitoring/tyk-components.mdx b/planning-for-production/monitoring/tyk-components.mdx new file mode 100644 index 00000000..cd189619 --- /dev/null +++ b/planning-for-production/monitoring/tyk-components.mdx @@ -0,0 +1,104 @@ +--- +title: "Monitor Tyk Stack" +'og:description': "How to monitor Tyk components in production" +sidebarTitle: "Monitoring" +--- + +A common question that gets asked is how to monitor the Tyk components. + +## Tyk Gateway + +The Gateway & Redis are the only components that will have a high on-demand performance requirement, which needs to scale with your API traffic. + +### Tyk Gateway CPU Utilization + +Tyk Gateway is CPU bound. It will have better performance the more cores you throw at Tyk. Tyk will automatically spread itself across all available cores to handle the traffic. Be sure to limit the cores in a Kubernetes deployment otherwise, the Gateway will attempt to consume all cores in a node. + +Performance benchmarks on how Tyk performs across different CPU architectures, environments and sizes [here](https://tyk.io/performance-benchmarks/). + +A healthy and performant Tyk Gateway should have a CPU utilization of under 60%. If the average CPU utilization is above 60%, then we recommend you scale your Tyk Gateway services. A higher figure than 60% introduces risk because if one Gateway fails, the traffic spillover to healthy nodes might be overwhelming and result in a cascading failure. + +### Liveness Health Check + +Health checks are extremely important in determining the status of our Tyk Components. Depending on your setup and configuration, the statuses of the following components will be returned: **Gateway**, **Dashboard**, **Redis** and **RPC**. + +The health check endpoint is an expensive operation and can throttle throughput so it's important to find an optimal plan if you want to take advantage of this endpoint. The endpoint refreshes every 10 seconds and does not return statuses on the primary database (MongoDB or PostgreSQL) and the Tyk Pump component. + + +```yaml expandable +curl -X GET "http://localhost:8080/hello" + +{ + "status": "pass", + "version": "4.2.3", + "description": "Tyk GW", + "details": { + "dashboard": { + "status": "pass", + "componentType": "system", + "time": "2022-11-21T20:03:44Z" + }, + "redis": { + "status": "pass", + "componentType": "datastore", + "time": "2022-11-21T20:03:44Z" + }, + "rpc": { + "status": "pass", + "componentType": "system", + "time": "2022-11-21T20:03:44Z" + } + } +} +``` + +For information about our health check endpoint, please visit our [Liveness Health Check](#liveness-health-check) documentation. + +## Tyk Dashboard & Tyk MDCB + +These other Tyk components won’t see load proportional to your API requests. However, the Dashboard (and MDCB on Global deployments) synchronise the Gateways and need to be given enough resources to manage this process. + +The Tyk Dashboard liveness health check endpoint can be configured [here](/tyk-dashboard/configuration#health_check_endpoint_name). + +The Tyk MDCB liveness health check endpoint can be configured [here](/api-management/mdcb#health-check). + +Currently, Tyk Dashboard and MDCB liveness endpoints only report whether the service is operational. It is important to note that the service may not yet be ready for use if it is unable to establish a connection with its dependent components (such as Redis and Datastore) or if they are offline. + +## Tyk Pump + +The Tyk Pump component offers a built-in Liveness Health Check endpoint [here](/tyk-pump/tyk-pump-configuration/tyk-pump-environment-variables#health-check). + +Currently, the receipt of an HTTP 200 OK response merely indicates that the Pump service is operational. However, it is important to note that the service may not yet be ready for use if it is unable to establish a connection with its dependent components (such as Redis and Datastore) or if they are offline. + +### Database Sizing + +Based on the Tyk recommended approach for setting up your databases, our team has built tools that will help engineers better understand and plan their infrastructure around their use case: + +* [Redis Sizing](/planning-for-production/database-settings#redis-sizing-guidelines) +* [PostgreSQL Sizing](/planning-for-production/database-settings#postgresql-sizing-guidelines) +* [MongoDB Sizing](/planning-for-production/database-settings#mongodb-sizing-guidelines) + + +### Database Monitoring + +Monitoring guides from our partner: + +**Redis** +1. [How to monitor Redis performance metrics](https://www.datadoghq.com/blog/how-to-monitor-redis-performance-metrics/) +2. [How to collect Redis Metrics](https://www.datadoghq.com/blog/how-to-collect-redis-metrics/) +3. [Monitor Redis using Datadog](https://www.datadoghq.com/blog/monitor-redis-using-datadog/) + + +**Postgres** +1. [Key metrics for PostgreSQL monitoring](https://www.datadoghq.com/blog/postgresql-monitoring/) +2. [Collecting metrics with PostgreSQL monitoring tools](https://www.datadoghq.com/blog/postgresql-monitoring-tools/) +3. [How to collect and monitor PostgreSQL data with Datadog](https://www.datadoghq.com/blog/collect-postgresql-data-with-datadog/) + + +**Mongo** + +1. [Monitoring MongoDB performance metrics (WiredTiger)](https://www.datadoghq.com/blog/monitoring-mongodb-performance-metrics-wiredtiger/) +2. [Collecting MongoDB metrics and statistics](https://www.datadoghq.com/blog/collecting-mongodb-metrics-and-statistics/) +3. [How to monitor MongoDB performance with Datadog](https://www.datadoghq.com/blog/monitor-mongodb-performance-with-datadog/) + + diff --git a/portal/api-consumer.mdx b/portal/api-consumer.mdx new file mode 100644 index 00000000..950ff23d --- /dev/null +++ b/portal/api-consumer.mdx @@ -0,0 +1,36 @@ +--- +title: "Developers and API Consumers in the Developer Portal" +'og:description': "How to configure API Consumers in Tyk developer portal" +tags: ['Developer Portal', 'Tyk', 'API Consumer', 'Developer', 'Organization', 'Invite Codes', 'Consumer Registration'] +sidebarTitle: "Overview" +--- + + + +**Tyk Enterprise Developer Portal** + +If you are interested in getting access contact us at [support@tyk.io]() + + + +## Manage API Consumers + +External developers are referred to as API Consumers. In the Tyk Developer portal, you can manage external API Consumers via the admin dashboard. + +### Glossary + +**API Consumers** - Refers to the whole section within the admin dashboard that manages individual external users, teams, and organizations. + +**Organizations** - An organization can represent larger business units of a company. It works as a container for various teams and users. An organization can be used which can include multiple teams. + +**Teams** - Teams are used to bundle multiple users, a team always needs to be part of an organization. + +**Users** - External developers / portal users. A user can belong to multiple teams but can only belong to one organization. + +Portal API Consumers menu + +### How does the API Consumer section work? + +When installing the Tyk Portal, by default the API Consumers section will already have a default organization with a default team added. This means, if your specific use case doesn't require multiple organizations and teams, you can get started straight away and invite a new external user to the developer portal, adding them to the default organization and default team. + +If your use case requires adding a new organization, see [step by step guide](/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/manage-api-consumer-organisations). When an organization is created, a default team tied to that organization is automatically generated too. Teams and organizations can have different visibility when it comes to API Products and plans. This can be managed within the [catalog section](/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/manage-catalogues#create-a-new-catalog). diff --git a/portal/api-consumer/invite-codes.mdx b/portal/api-consumer/invite-codes.mdx new file mode 100644 index 00000000..87b4bef0 --- /dev/null +++ b/portal/api-consumer/invite-codes.mdx @@ -0,0 +1,39 @@ +--- +title: "Register a User using Invite Codes" +'og:description': "How to configure basic authentication in Tyk?" +sidebarTitle: "Invite Codes" +tags: ['Authentication', 'Authorization', 'Tyk Authentication', 'Tyk Authorization', 'Secure APIs', 'Basic Authentication'] +--- + +## Invite Codes + +Here you’ll learn about how to create invite codes to add a new external user to the developer portal. Invite codes can be used to simplify user onboarding. Using invite codes, users will be directly assigned to a team and organization, giving them the same access rights as this team. For example you can use invite codes to: +- Run a promotional campaign like a Hackathon and give access to specific plans to the users. +- Onboard a partner company into the portal by giving them this code for anyone registering. + +## Prerequisites + +- A Tyk Enterprise portal installation +- A portal admin app login + +## Instructions + +1. From the **API Consumers > Invite Codes** menu, click **Add**. + + Invite Codes menu + Invite Codes dialog + +2. Add the form details: + + - Set your desired **Quota**. Quota is the max number of slots available for your invite code. E.g. if set 100, this code can be used by the top 100 users. + - Set an expiry date, this is the date on which the code will expire and no more users can sign up to it. + - Set state to **Active** - this means the code is activated and developers can start using it. + - Specify the team that any new users that register with this invite code will be added to. + + Invite Codes dialog + +3. **Save** the invite code. +4. Share the invite codes. When the saving changes a new Invite code was created and can be viewed in the overview table. To share the invite code, copy it and send to your developer teams/users. + + Share Invite Codes dialog + diff --git a/portal/api-provider.mdx b/portal/api-provider.mdx new file mode 100644 index 00000000..af2ddb1c --- /dev/null +++ b/portal/api-provider.mdx @@ -0,0 +1,27 @@ +--- +title: "API Providers in the Developer Portal" +'og:description': "How to configure API Providers in Tyk developer portal" +tags: ['Developer Portal', 'Tyk', 'Managing Access', 'Catalogs', 'Rate Limit', 'Dynamic Client Registration', 'Documenting APIs'] +sidebarTitle: "Overview" +--- + + + +**Tyk Enterprise Developer Portal** + +If you are interested in getting access contact us at [support@tyk.io]() + + + +## How to Manage API Access? + +The Tyk Enterprise Developer portal provides a special container for access credentials - an application. Applications hold developer credentials that can be used to access APIs published on the portal with a specific plan. A sample relationship between applications, credentials, API Products, and plans is demonstrated in the diagram below. + +Sample application setup + +Credentials are generated by the Tyk Dashboard when an admin user approves a provisioning request or when provisioning requests are configured to be approved automatically. The provisioning flow is demonstrated in the diagram below. + +Portal provisioning flow + +This section describes how admin users can manage applications and configure the settings of provisioning requests. + diff --git a/portal/customization.mdx b/portal/customization.mdx new file mode 100644 index 00000000..a3588a09 --- /dev/null +++ b/portal/customization.mdx @@ -0,0 +1,20 @@ +--- +title: "Developer Portal Customization" +'og:description': "Customization options for enterprise developer portal" +tags: ['Developer Portal', 'Tyk', 'Customization', 'Webhook', 'Email', 'Themes', 'Templates', 'Pages', 'Menus', 'Branding', 'User Model'] +sidebarTitle: "Overview" +--- + + + +**Tyk Enterprise Developer Portal** + +If you are interested in getting access contact us at [support@tyk.io]() + + + +## Portal Customization Overview + +The Tyk Enterprise Developer Portal uses themes for customizing the live portal. We provide an out of the box theme that is using our own branding, it’s called the `default` theme. You are welcome to use it and modify it for your needs, yet if you want to start with a blank page, you can also create a completely new theme. + +This section provides a complete guide to full customization from the developer's point of view. diff --git a/portal/customization/branding.mdx b/portal/customization/branding.mdx new file mode 100644 index 00000000..05089130 --- /dev/null +++ b/portal/customization/branding.mdx @@ -0,0 +1,93 @@ +--- +title: "Customize Branding in Developer Portal" +'og:description': "How to customize branding in developer portal" +sidebarTitle: "Branding" +--- + +In this section we will explain how to apply your branding (styling - CSS) on the portal elements with your own colors and logo within minutes. + +**Prerequisites** + +- A Tyk Self-Managed [installation](/tyk-self-managed/install) +- A login for the portal admin app +- Access to your Tyk portal file system + +## Changing Portal Logo + +1. Access the file directory for the Developer portal +2. The default logo is located in `/themes/default/assets/images/` and is called `dev-portal-logo.svg`. +3. Replace the default image with your own, keeping the same file name and in `.svg` format, ensuring that `xmlns="http://www.w3.org/2000/svg"` is included within your `` tag. + + + + + If you want to use different naming, path reference or extension, the code is `` and is found on line 6 from the `/themes/default/partials/footer.tmpl` template. + + + +## Changing Brand Colors + +Let’s now explain how to manage borders and change the colors of buttons, texts and backgrounds. The file we’ll be looking at is `/themes/default/assets/stylesheets/main.css` which contains some CSS variables that are used throughout the app. Let’s take a closer look. +You can apply some changes in the portal based on your preferences. For example, you can change the navigation background color, the text color and the different button theme colors. Furthermore, you can change table border color and radius. + +If you want to change the navigation background color you need to edit the variable called `--tdp-nav-bg-color` Similarly other variables as you can see where/how each one is used: + + + +`tdp` stands for Tyk Developer Portal + + + +### Background Colors + +Background Colour settings Tyk Enterprise Portal + +- `--tdp-nav-bg-color` navigation background color +- `--tdp-body-bg-color` App background color + +### Text Colors + +Text Colour settings Tyk Enterprise Portal + +- `--tdp-text-color` default text color +- `--tdp-link-color` links (anchor tags) +- `--tdp-nav-link-color` navigation links + +### Borders + +Border Colour settings Tyk Enterprise Portal + +- `--tdp-card-border-radius` Card component +- `--tdp-border-color-on-error` input color if there’s an error +- `--tdp-table-border-color` table +- `--tdp-border-radius` radius +- `--tdp-primary-border form` elements (such as `` and `` and ` + + {{ range $app := .apps }} + + {{ end }} + +``` + +### Application Create + +**Template Path**: `themes/default/views/app_form_create.tmpl` + +This template is used to render the application creation form. + +#### Available Objects + +- `{{ .errors }}`: Map of template errors (Key: category, Value: error message) + +##### Example Usage +```html expandable +{{ if .errors }} +{{ range $key, $errs := .errors }} + +{{ end }} +{{ end }} +``` + +### Application Detail + +**Template Path**: `themes/default/views/app_form_update.tmpl` + +This template is used to render the application detail and update form. + +#### Available Objects + +- `{{ .errors }}`: Map of template errors (Key: category, Value: error message) +- `{{ .app }}`: Selected application object. +- `{{ .appCerts }}`: Map of application MTLS certificates if applicable (Key: access request ID, Value: certificate) +- `{{ .allCerts }}`: Map of all MTLS certificates stored if applicable (Key: cert fingerprint, Value: cert) + +#### MTLS Certificate Attributes (appCerts) + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .ID }}` | Certificate ID | +| `{{ .Name }}` | Certificate name | +| `{{ .Fingerprint }}` | Certificate fingerprint | +| `{{ .SignatureAlgorithm }}` | Signature algorithm | +| `{{ .Issuer }}` | Certificate issuer | +| `{{ .IsValid }}` | Boolean indicating if the certificate is valid | +| `{{ .ValidNotBefore }}` | Start date of validity | +| `{{ .ValidNotAfter }}` | End date of validity | +| `{{ .Subject }}` | Certificate subject | + +#### MTLS Certificate Attributes (allCerts) + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .ID }}` | Certificate ID | +| `{{ .Name }}` | Certificate name | + +#### Client Attributes + +Accessible via `{{ .app }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .ID }}` | Client ID | +| `{{ .Name }}` | Client name | +| `{{ .Description }}` | Client description | +| `{{ .RedirectURLs }}` | Client redirect URLs | +| `{{ .Credentials }}` | Array of client credentials | +| `{{ .AccessRequests }}` | Array of client access requests | + +#### Client Credentials Attributes + +Accessible via `{{ range $cred := .app.Credentials }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .ID }}` | Credential ID | +| `{{ .Credential }}` | Credential | +| `{{ .CredentialHash }}` | Credential hash | +| `{{ .OAuthClientID }}` | OAuth client ID | +| `{{ .OAuthClientSecret }}` | OAuth client secret | +| `{{ .Expires }}` | Credential expiration | +| `{{ .AccessRequestID }}` | Access request ID associated with the credential | + +#### Client Access Requests Attributes + +Accessible via `{{ range $acreq := .app.AccessRequests }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .ID }}` | Access request ID | +| `{{ .Status }}` | Access request status | +| `{{ .UserID }}` | User ID associated with access request | +| `{{ .AuthType }}` | Access request auth type | +| `{{ .DCREnabled }}` | true if access request DCR enabled | +| `{{ .ProvisionImmediately }}` | true if provisioned immediately is enabled | +| `{{ .CatalogueID }}` | Catalogue ID | + +#### Product Attributes (within Access Request) + +Accessible via `{{ $product := $acreq.Product }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .ID }}` | Product ID | +| `{{ .Name }}` | Product name | +| `{{ .DisplayName }}` | Product display name | +| `{{ .Path }}` | Product path | +| `{{ .Description }}` | Product description | +| `{{ .Content }}` | Product content | +| `{{ .AuthType }}` | Product auth type | +| `{{ .DCREnabled }}` | true if product DCR enabled | + +#### Plan Attributes (within Access Request) + +Accessible via `{{ $acreq.Plan }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .Name }}` | Plan name | +| `{{ .DisplayName }}` | Plan display name | +| `{{ .Description }}` | Plan description | +| `{{ .AuthType }}` | Plan auth type | +| `{{ .Rate }}` | Plan rate | +| `{{ .Per }}` | Plan period | +| `{{ .QuotaMax }}` | Plan quota maximum | +| `{{ .QuotaRenewalRate }}` | Plan quota renewal rate | +| `{{ .AutoApproveAccessRequests }}` | true if auto-approve access requests is enabled | + +##### Example Usage +```html expandable +

{{ .app.Name >}}

+

{{ .app.Description }}

+

Credentials

+{{ range $cred := .app.Credentials }} +
+

ID: {{ $cred.ID }}

+

OAuth Client ID: {{ $cred.OAuthClientID }}

+

Expires: {{ $cred.Expires }}

+
+{{ end }} +

Access Requests

+{{ range $acreq := .app.AccessRequests }} +
+

ID: {{ $acreq.ID }}

+

Status: {{ $acreq.Status }}

+

Product: {{ $acreq.Product.Name }}

+

Plan: {{ $acreq.Plan.Name }}

+
+{{ end }} +``` + +### Blogs + +**Template Path**: `themes/default/views/blog_listing.tmpl` + +This template is used to render the blog listing page. + +#### Available Objects + +- `{{ .posts }}`: List of all published blog posts + +#### Blog Attributes + +Accessible via `{{ range .posts }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .Title }}` | Blog post title | +| `{{ .Lede }}` | Blog post summary | +| `{{ .Content }}` | Blog post content | +| `{{ .MarkdownContent }}` | Markdown content | +| `{{ .MarkdownEnabled }}` | Boolean for Markdown enablement | +| `{{ .Path }}` | Blog post path | +| `{{ .HeaderImage.URL }}` | Header image URL | +| `{{ .BlogSiteID }}` | Blog site ID | +| `{{ .ProductID }}` | Associated product ID | +| `{{ .AuthorID }}` | Author ID | +| `{{ .URL }}` | Full URL of the blog post | + +##### Example Usage +```html expandable +

Blog Posts

+{{ range .posts }} +
+

{{ .Title }}

+ {{ .Title }} +

{{ .Lede }}

+
+{{ end }} +``` + +### Blog Detail + +**Template Path**: `themes/default/views/blog_detail.tmpl` + +This template is used to render the blog detail page. + +#### Available Objects + +- `{{ .post }}`: The selected blog post object. +- `{{ .latest_posts }}`: List of 3 latest blog posts. + +#### Blog Attributes + +Accessible via `{{ .post }}` or `{{ range .latest_posts }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .Title }}` | Blog post title | +| `{{ .Lede }}` | Blog post summary | +| `{{ .Content }}` | Blog post content | +| `{{ .MarkdownContent }}` | Markdown content | +| `{{ .MarkdownEnabled }}` | Boolean for Markdown enablement | +| `{{ .Path }}` | Blog post path | +| `{{ .HeaderImage.URL }}` | Header image URL | +| `{{ .BlogSiteID }}` | Blog site ID | +| `{{ .ProductID }}` | Associated product ID | +| `{{ .AuthorID }}` | Author ID | +| `{{ .URL }}` | Full URL of the blog post | + +##### Example Usage +``` expandable +

{{ .post.Title }}

+{{ .post.Title }} +

{{ .post.Lede }}

+{{ if .post.MarkdownEnabled }} +{{ .post.MarkdownContent | markdownify }} +{{ else }} +{{ .post.Content }} +{{ end }} +

Read more at: {{ .post.URL }}

+

Latest Posts

+{{ range .latest_posts }} +
+

{{ .Title }}

+

{{ .Lede }}

+
+{{ end }} +``` + +### Cart Checkout + +**Template Path**: `themes/default/views/portal_checkout.tmpl` + +This template is used to render the cart checkout page. + +#### Available Objects + +- `{{ .cart }}`: Map with the cart items for the current user +- `{{ .apps }}`: List of applications for the current user +- `{{ .catalogue_count }}`: Cart catalogues count +- `{{ .certs }}`: List of MTLS certificates if applicable +- `{{ .errors }}`: Map of template errors (Key: category, Value: error message) +- `{{ .provisioned }}`: Boolean indicating whether an access request has been provisioned for the cart + +#### Application Attributes + +Accessible via `{{ range .apps }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .Name }}` | Application name | +| `{{ .Description }}` | Application description | +| `{{ .RedirectURLs }}` | Application redirect URLs | + +#### MTLS Certificate Attributes + +Accessible via `{{ range .certs }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .ID }}` | Certificate ID | +| `{{ .Name }}` | Certificate name | + +#### Cart Item Attributes + +Accessible via `{{ range $key, $value := .cart }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ $value.AuthType }}` | Cart item auth type | +| `{{ $value.Catalogue }}` | Cart item catalogue | +| `{{ $value.Products }}` | Cart item array of products | +| `{{ $value.Plan }}` | Cart item plan | +| `{{ $value.DCREnabled }}` | true if cart order consists of DCR products | + +#### Plan Attributes (Within cart item) + +Accessible via `{{ $plan := $value.Plan }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .ID }}` | Plan ID | +| `{{ .PlanName }}` | Plan name | +| `{{ .FormatQuota }}` | Formatted quota information | +| `{{ .FormatRateLimit }}` | Formatted rate limit information | + +#### Catalogue Attributes (Within cart item) + +Accessible via `{{ $catalogue := $value.Catalogue }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .ID }}` | Catalogue ID | + +#### Product Attributes (Within cart item) + +Accessible via `{{ range $product := $value.Products }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .ID }}` | Product ID | +| `{{ .Name }}` | Product name | +| `{{ .DisplayName }}` | Product display name | +| `{{ .Path }}` | Product path | +| `{{ .Description }}` | Product description | +| `{{ .Content }}` | Product content | +| `{{ .AuthType }}` | Product auth type | +| `{{ .DCREnabled }}` | true if product DCR enabled | +| `{{ .AuthTypes }}` | List of product auth types | + +#### Auth Type Attributes (Within product) + +Accessible via `{{ range $auth_type := $product.AuthTypes }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .AuthType }}` | Auth type | + +#### DCR Client Template Attributes (Within product) + +Accessible via `{{ range $template := $product.Templates }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .ID }}` | Template ID | +| `{{ .Name }}` | Template name | +| `{{ .GrantType }}` | Template grant type | +| `{{ .ResponseTypes }}` | Template response types | +| `{{ .TokenEndpointAuthMethod }}` | Template token endpoint auth method | +| `{{ .OktaAppType }}` | Template Okta app type | + + +```html expandable +

Cart Checkout

+{{ range $key, $value := .cart }} +
+

Auth Type: {{ $value.AuthType }}

+

Plan: {{ $value.Plan.Name }}

+ {{ range $product := $value.Products }} +
+

{{ $product.DisplayName }}

+

{{ $product.Description }}

+

Path: {{ $product.Path }}

+
+ {{ end }} +
+{{ end }} +

Your Applications

+{{ range $app := .apps }} +
+

{{ $app.Name }}

+

{{ $app.Description }}

+
+{{ end }} +{{ if .certs }} +

MTLS Certificates

+{{ range $cert := .certs }} +
+

ID: {{ $cert.ID }}

+

Name: {{ $cert.Name }}

+
+{{ end }} +{{ end }} +``` + + + +### Organization User Detail + +**Template Path**: `themes/default/views/user_detail.tmpl` + +This template is used to render the organization user detail page. + +#### Available Objects + +- `{{ .errors }}`: Map of template errors (Key: category, Value: error message) +- `{{ .user }}`: The organization user object. + +#### User Attributes + +Accessible via `{{ .user }}` + +| Attribute/Method | Description | +| :------------------- | :------------- | +| `{{ .ID }}` | User ID | +| `{{ .First }}` | User name | +| `{{ .Last }}` | User surname | +| `{{ .Email }}` | User email | +| `{{ .OrganisationID }}` | User organization ID | +| `{{ .DisplayName }}` | User complete name | +| `{{ .IdentityProvider }}` | User provider (Portal or Tyk Identity Broker) | +| `{{ .GetOrganisationID }}` | User's organization ID | +| `{{ .IsAdmin }}` | true if user is an admin | +| `{{ .IsOrgAdmin }}` | true if user is an organization admin | +| `{{ .DisplayRole }}` | User's role | + + + +```html expandable +

User Details

+{{ if .errors }} +{{ range $key, $errs := .errors }} + +{{ end }} +{{ end }} +

Name: {{ .user.DisplayName }}

+

Email: {{ .user.Email }}

+

Role: {{ .user.DisplayRole }}

+``` + + +### Organization User Edit + +**Template Path**: `themes/default/views/user_edit.tmpl` + +This template is used to render the edit page for organization user. + +#### Available Objects + +- `{{ .errors }}`: Map of template errors (Key: category, Value: error message) +- `{{ .roles }}`: List of possible roles +- `{{ .user }}`: The organization user object. + +#### Role Attributes + +Accessible via `{{ range .roles }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .ID }}` | Role ID | +| `{{ .DisplayName }}` | Role display name | + +#### User Attributes + +Accessible via `{{ .user }}` + +| Attribute/Method | Description | +| :------------------- | :------------- | +| `{{ .ID }}` | User ID | +| `{{ .First }}` | User name | +| `{{ .Last }}` | User surname | +| `{{ .Email }}` | User email | +| `{{ .OrganisationID }}` | User organization ID | +| `{{ .DisplayName }}` | User complete name | +| `{{ .IdentityProvider }}` | User provider (Portal or Tyk Identity Broker) | +| `{{ .GetOrganisationID }}` | User's organization ID | +| `{{ .IsAdmin }}` | true if user is an admin | +| `{{ .IsOrgAdmin }}` | true if user is an organization admin | +| `{{ .DisplayRole }}` | User's role | + + +```html expandable +
+ {{ if .error }} + + {{ end }} +

Developer details

+
+ + +
+
+ + +
+
+ + +
+ {{ if .roles }} +
+ + +
+ {{ end }} +
+ Cancel + +
+
+``` + + +### Organization Users List + +**Template Path**: `themes/default/views/user_list.tmpl` + +This template is used to render the list of organization users. + +#### Available Objects + +- `{{ .roles }}`: Map of available roles (Key: role, Value: role display name) + +##### Example Usage +```html + {{ index $roles $userInvite.Role }} +{{ end }} +``` + +### Product Detail + +**Template Path**: `themes/default/views/portal_product_detail.tmpl` + +This template is used to render the product detail page. + +#### Available Objects + +- `{{ .product }}`: The selected product object +- `{{ .catalogues }}`: List of catalogue objects including the selected product +- `{{ .unique_plans }}`: List of plan objects available for the product +- `{{ .scopes }}`: Product scopes as an array of strings +- `{{ .posts }}`: List of related blog post objects +- `{{ .errors }}`: Map of template errors (Key: category, Value: error message) +- `{{ .added }}`: Boolean indicating if the product is added to the cart + +#### Product Attributes + +Accessible via `{{ .product }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .ID }}` | Product ID | +| `{{ .Name }}` | Product name | +| `{{ .DisplayName }}` | Product display name | +| `{{ .Path }}` | Product path | +| `{{ .ReferenceID }}` | Product reference ID | +| `{{ .Description }}` | Product description | +| `{{ .AuthType }}` | Product auth type | +| `{{ .Logo.URL }}` | Product logo URL | +| `{{ .Feature }}` | true if the product is featured | +| `{{ .DCREnabled }}` | true if DCR is enabled | +| `{{ .ProviderID }}` | Provider ID | + +#### API Details (Within product) + +Accessible via `{{ .product.APIDetails }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .Name }}` | API name | +| `{{ .Description }}` | API description | +| `{{ .APIType }}` | API type | +| `{{ .TargetURL }}` | API target URL | +| `{{ .ListenPath }}` | API listen path | +| `{{ .OASUrl }}` | API OAS URL | +| `{{ .Status }}` | "Active" if API status is active, otherwise "Inactive" | + +#### Documentation (Within product) + +Accessible via `{{ .product.Docs }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .Title }}` | Document title | +| `{{ .ID }}` | Document identifier | +| `{{ .Content }}` | Document content | +| `{{ .MarkdownContent }}` | Markdown content | +| `{{ .MarkdownEnabled }}` | Boolean for Markdown enablement | + +#### Catalogues + +Accessible via `{{ range .catalogues }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .Name }}` | Catalogue name | +| `{{ .VisibilityStatus }}` | Catalogue visibility status | + +#### Plans + +Accessible via `{{ range .unique_plans }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .Name }}` | Plan name | +| `{{ .ID }}` | Plan ID | +| `{{ .DisplayName }}` | Plan display name | +| `{{ .Description }}` | Plan description | +| `{{ .AuthType }}` | Plan authentication type | +| `{{ .Rate }}` | Plan rate | +| `{{ .Per }}` | Plan rate per time unit | +| `{{ .QuotaMax }}` | Plan maximum quota | +| `{{ .QuotaRenewalRate }}` | Plan quota renewal rate | +| `{{ .AutoApproveAccessRequests }}` | Boolean for auto-approval of access requests | + +#### Related Posts + +Accessible via `{{ range .posts }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .Title }}` | Post title | +| `{{ .Lede }}` | Post summary | +| `{{ .Content }}` | Post content | +| `{{ .MarkdownContent }}` | Markdown content | +| `{{ .MarkdownEnabled }}` | Boolean for Markdown enablement | +| `{{ .Path }}` | Post path | +| `{{ .HeaderImage.URL }}` | Header image URL | +| `{{ .BlogSiteID }}` | Blog site ID | +| `{{ .ProductID }}` | Associated product ID | +| `{{ .AuthorID }}` | Author ID | + + +```html expandable +
+

{{ .product.DisplayName }}

+ {{ .product.Name }} logo +

{{ .product.Description }}

+

API Details

+ {{ range .product.APIDetails }} +

{{ .Name }}

+

Status: {{ .Status }}

+

Target URL: {{ .TargetURL }}

+ {{ end }} +

Documentation

+ {{ range .product.Docs }} +

{{ .Title }}

+ {{ if .MarkdownEnabled }} + {{ .MarkdownContent | markdownify }} + {{ else }} + {{ .Content }} + {{ end }} + {{ end }} +

Available in Catalogues

+
    + {{ range .catalogues }} +
  • {{ .Name }} ({{ .VisibilityStatus }})
    • + {{ end }} +
+

Available Plans

+ {{ range .unique_plans }} +
+

{{ .DisplayName }}

+

{{ .Description }}

+

Rate: {{ .Rate }} per {{ .Per }}

+

Quota: {{ .QuotaMax }}

+
+ {{ end }} +

Related Posts

+ {{ range .posts }} +
+

{{ .Title }}

+ {{ .Title }} +

{{ .Lede }}

+
+ {{ end }} +
+``` + + +### Product OAS Documentation + +**Template Paths**: +- `themes/default/views/product_doc_stoplight_spec.tmpl` +- `themes/default/views/product_doc_redoc.tmpl` + +These templates are used to render the OpenAPI Specification (OAS) documentation for a product. The Stoplight Spec and +ReDoc versions are available. + +#### Available Attributes + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .Name }}` | Product name | +| `{{ .Description }}` | Product description | +| `{{ .Path }}` | Product path | +| `{{ .Url }}` | OAS document URL | + + +```html expandable +
+
+
+

+ {{ .Name }} +

+

+ {{ .Description }} +

+
+
+
+ +
+
+``` + + +## Global Helper Functions + +This section provides a detailed overview of the global helper functions available in the Tyk Enterprise Developer +Portal templates. These functions are accessible across the public and private templates and allow you to perform +various operations, retrieve specific data, and create dynamic content within your templates. + +### Available Functions + +- [CanCreateOrganisation](#cancreateorganisation) +- [Clients](#clients) +- [Current User](#currentuser) +- [FeaturedProducts](#featuredproducts) +- [FilterUserInvites](#filteruserinvites) +- [FormatTime](#formattime) +- [GetCart](#getcart) +- [GetCatalogueList](#getcataloguelist) +- [GetCataloguesForProduct](#getcataloguesforproduct) +- [GetClientDescription](#getclientdescription) +- [GetClientName](#getclientname) +- [GetMenus](#getmenus) +- [GetProducts](#getproducts) +- [IsPortalDisabled](#isportaldisabled) +- [IsPortalPrivate](#isportalprivate) +- [ProductDocRenderer](#productdocrenderer) +- [ProviderUpstreamURL](#providerupstreamurl) +- [SplitStrings](#splitstrings) +- [TruncateString](#truncatestring) +- [TypeOfCredential](#typeofcredential) + +#### CanCreateOrganisation + +Returns true if user can create an organization. + +##### Example Usage +``` +{{ if CanCreateOrganisation req }} + ... +{{ end }} +``` + +#### Clients + +Returns the list of applications for the current user. Expects the request as argument. + +##### Client Attributes + +Accessible via `{{ range $client := Clients req }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ $client.ID }}` | Client ID | +| `{{ $client.Name }}` | Client name | +| `{{ $client.Description }}` | Client description | +| `{{ $client.RedirectURLs }}` | Client redirect URLs | +| `{{ $client.Credentials }}` | Array of client credentials | +| `{{ $client.AccessRequests }}` | Array of client access requests | + +##### Credential Attributes (Within client) + +Accessible via `{{ range $cred := $client.Credentials }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ $cred.ID }}` | Credential ID | +| `{{ $cred.Credential }}` | Credential | +| `{{ $cred.CredentialHash }}` | Credential hash | +| `{{ $cred.OAuthClientID }}` | OAuth client ID | +| `{{ $cred.OAuthClientSecret }}` | OAuth client secret | +| `{{ $cred.Expires }}` | Credential expiration | +| `{{ $cred.AccessRequestID }}` | Access request ID associated with the credential | + +##### Access Request Attributes (Within client) + +Accessible via `{{ range $acreq := $client.AccessRequests }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ $acreq.ID }}` | Access request ID | +| `{{ $acreq.Status }}` | Access request status | +| `{{ $acreq.UserID }}` | User ID associated with access request | +| `{{ $acreq.AuthType }}` | Access request auth type | +| `{{ $acreq.DCREnabled }}` | true if access request DCR enabled | +| `{{ $acreq.ProvisionImmediately }}` | true if provisioned immediately is enabled | +| `{{ $acreq.CatalogueID }}` | Catalogue ID | + +##### Product Attributes (Within access request) + +Accessible via `{{ range $product := $acreq.Products }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ $product.ID }}` | Product ID | +| `{{ $product.Name }}` | Product name | +| `{{ $product.DisplayName }}` | Product display name | +| `{{ $product.Path }}` | Product path | +| `{{ $product.Description }}` | Product description | +| `{{ $product.Content }}` | Product content | +| `{{ $product.AuthType }}` | Product auth type | +| `{{ $product.DCREnabled }}` | true if product DCR enabled | + +##### Plan Attributes (Within access request) + +Accessible via `{{ $acreq.Plan }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .Name }}` | Plan name | +| `{{ .DisplayName }}` | Plan display name | +| `{{ .Description }}` | Plan description | +| `{{ .AuthType }}` | Plan auth type | +| `{{ .Rate }}` | Plan rate | +| `{{ .Per }}` | Plan period | +| `{{ .QuotaMax }}` | Plan quota maximum | +| `{{ .QuotaRenewalRate }}` | Plan quota renewal rate | +| `{{ .AutoApproveAccessRequests }}` | true if auto-approve access requests is enabled | + +##### Example Usage +```html expandable +{{ range $client := Clients req }} +
+

Client: {{ $client.Name }}

+ {{ range $acreq := $client.AccessRequests }} +

Products:

+
    + {{ range $product := $acreq.Products }} +
  • + {{ $product.Name }} + {{ $product.Description }} +
    • + {{ end }} +
+

Plan:

+

Name: {{ $acreq.Plan.Name }}

+

Rate: {{ $acreq.Plan.Rate }} per {{ $acreq.Plan.Per }}

+

Quota Max: {{ $acreq.Plan.QuotaMax }}

+ {{ end }} +
+{{ end }} +``` + +#### CurrentUser + +The `CurrentUser` function returns the current user object if a user is logged in. It expects the request as an argument. + +#### User Attributes + +Accessible via `{{ $user := CurrentUser req }}` + +| Attribute/Method | Description | +| :------------------- | :------------- | +| `{{ $user.ID }}` | User ID | +| `{{ $user.First }}` | User name | +| `{{ $user.Last }}` | User surname | +| `{{ $user.Email }}` | User email | +| `{{ $user.OrganisationID }}` | User organization ID | +| `{{ $user.DisplayName }}` | User complete name | +| `{{ $user.IdentityProvider }}` | User provider (Portal or Tyk Identity Broker) | +| `{{ $user.GetOrganisationID }}` | User's organization ID | +| `{{ $user.IsAdmin }}` | true if user is an admin | +| `{{ $user.IsOrgAdmin }}` | true if user is an organization admin | +| `{{ $user.DisplayRole }}` | User's role | + +##### Example Usage +```html expandable +{{ $user := CurrentUser req }} +{{ if $user }} + +{{ else }} +

Please log in to view your account information.

+{{ end }} +``` + +#### FeaturedProducts + +Returns a list of featured products. + +##### Product Attributes + +Accessible via `{{ range FeaturedProducts }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .ID }}` | Product ID | +| `{{ .Name }}` | Product name | +| `{{ .DisplayName }}` | Product display name | +| `{{ .Path }}` | Product path | +| `{{ .ReferenceID }}` | Product reference ID | +| `{{ .Description }}` | Product description | +| `{{ .AuthType }}` | Product auth type | +| `{{ .Scopes }}` | Product scopes | +| `{{ .Logo.URL }}` | Product logo URL | +| `{{ .Feature }}` | true if the product is featured | +| `{{ .DCREnabled }}` | true if DCR is enabled | +| `{{ .ProviderID }}` | Provider ID | +| `{{ .APIDetails }}` | Array of API details associated with the product | +| `{{ .Catalogues }}` | Array of catalogues associated with the product | + +##### API Details Attributes (Within product) + +Accessible via `{{ range .APIDetails }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .Name }}` | API name | +| `{{ .Description }}` | API description | +| `{{ .APIType }}` | API type | +| `{{ .TargetURL }}` | API target URL | +| `{{ .ListenPath }}` | API listen path | +| `{{ .OASUrl }}` | API OAS URL | +| `{{ .Status }}` | "Active" if API status is active, otherwise "Inactive" | + +##### Catalogue Attributes (Within product) + +Accessible via `{{ range .Catalogues }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .Name }}` | Catalogue name | +| `{{ .VisibilityStatus }}` | Catalogue visibility status | + + +```html expandable +{{ $featured_products := FeaturedProducts }} +

Featured API Products

+

Explore our highlighted API offerings

+
+ {{ range $featured_products }} +
+ {{ if .Logo }} + {{ .Name }} logo + {{ end }} +
+ {{ .AuthType }} +

{{ .Name }}

+

{{ .Description }}

+
+
+ More Info +
+ {{ range .APIDetails }} + {{ if or (gt (.OASDocument.Base.Url | trim | length) 0) (gt (.OASUrl | trim | length) 0) }} + + {{ .Name }} + + {{ end }} + {{ end }} +
+
+
+ {{ end }} +
+``` + + +#### FilterUserInvites + +Returns a list of users that were invited to the current user's organization, if the user became an organization. +Expects the request as a parameter. + +##### User Attributes + +Accessible via `{{ range $invite := FilterUserInvites req }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ $invite.ID }}` | User ID | +| `{{ $invite.Email }}` | User email | +| `{{ $invite.First }}` | User first name | +| `{{ $invite.Last }}` | User last name | +| `{{ $invite.Role }}` | User role | +| `{{ $invite.JoinedAt }}` | User joined at time | +| `{{ $invite.Joined }}` | Whether the user has joined | +| `{{ $invite.Uactive }}` | Whether the user is active | + + +```html expandable +{{ $userInvites := FilterUserInvites req }} +{{ if $userInvites }} +

Invited Users

+ + + + + + + + + + + {{ range $invite := $userInvites }} + + + + + + + {{ end }} + +
NameEmailRoleStatus
{{ $invite.First }} {{ $invite.Last }}{{ $invite.Email }}{{ $invite.Role }} + {{ if $invite.Joined }} + Joined + {{ else if $invite.Uactive }} + Pending + {{ else }} + Inactive + {{ end }} +
+{{ else }} +

No pending invitations.

+{{ end }} +``` + + +#### FormatTime + +Formats a given time with a given format. + +##### Example Usage +```gotemplate expandable +{{ $user := CurrentUser req }} +{{ if $user}} +{{$time := FormatTime $user.CreatedAt "2 Jan, 2006 at 3:04:00 PM (MST)"}} +{/* Use $time or other variables here */} +... +{{end}} +``` + +#### GetCart + +Returns a map with the cart items for a given user ID. Expects the user ID as an argument. This function is useful for +retrieving and displaying the contents of a user's cart, including detailed information about the products, their +authentication types, and associated templates. + +##### Cart Item Attributes + +Accessible via `{{ range $key, $value := GetCart $user.ID }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ $value.AuthType }}` | Cart item auth type | +| `{{ $value.Catalogue }}` | Cart item catalogue | +| `{{ $value.DCREnabled }}` | true if cart order consists of DCR products | +| `{{ $value.Plan }}` | Cart item plan | +| `{{ $value.Products }}` | Cart item array of products | + +#### Plan Attributes (Within cart item) + +Accessible via `{{ $plan := $value.Plan }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .ID }}` | Plan ID | +| `{{ .PlanName }}` | Plan name | +| `{{ .FormatQuota }}` | Formatted quota information | +| `{{ .FormatRateLimit }}` | Formatted rate limit information | + +#### Catalogue Attributes (Within cart item) + +Accessible via `{{ $catalogue := $value.Catalogue }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .ID }}` | Catalogue ID | + +##### Product Attributes (Within cart item) + +Accessible via `{{ range $product := $value.Products }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .ID }}` | Product ID | +| `{{ .Name }}` | Product name | +| `{{ .DisplayName }}` | Product display name | +| `{{ .Path }}` | Product path | +| `{{ .Description }}` | Product description | +| `{{ .Content }}` | Product content | +| `{{ .AuthType }}` | Product auth type | +| `{{ .DCREnabled }}` | true if product DCR enabled | +| `{{ .AuthTypes }}` | List of product auth types | + +##### DCR Client Template Attributes (Within product) + +Accessible via `{{ range $template := $product.Templates }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .ID }}` | Template ID | +| `{{ .Name }}` | Template name | +| `{{ .GrantType }}` | Template grant type | +| `{{ .ResponseTypes }}` | Template response types | +| `{{ .TokenEndpointAuthMethod }}` | Template token endpoint auth method | +| `{{ .OktaAppType }}` | Template Okta app type | + +##### Example Usage +```html expandable +{{ $user := CurrentUser req }} +{{ if $user }} + {{ $cart := GetCart $user.ID }} + {{ if $cart }} +

Your Cart

+ {{ range $key, $value := $cart }} +
+

{{ $value.Catalogue.Name }}

+

Auth Type: {{ $value.AuthType }}

+ {{ range $product := $value.Products }} +
+

{{ $product.DisplayName }}

+

{{ $product.Description }}

+
+ {{ end }} +
+ {{ end }} + {{ else }} +

Your cart is empty.

+ {{ end }} +{{ end }} +``` + +#### GetCatalogueList + +Returns a list of catalogue names. Expects the request as parameter. + +##### Example Usage +```gotemplate +{{ range $key, $value := GetCatalogueList req }} + +{{ end }} +``` + +#### GetCataloguesForProduct + +Returns a list of products for a given user and product ID. Expects the request, a user and a product ID as parameters. + +##### Catalogue Attributes + +Accessible via `{{ range GetCataloguesForProduct req $user $product.ID }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .VisibilityStatus }}` | Catalogue visibility status | +| `{{ .Name }}` | Catalogue name | + +##### Example Usage +```html expandable +{{ $thisProduct := .product }} +{{ $user := CurrentUser req }} +{{ $catalogues_for_product := GetCataloguesForProduct req $user $thisProduct.ID }} +

Catalogues for {{ $thisProduct.Name }}

+
    + {{ range $catalogues_for_product }} +
  • + {{ .Name }} + (Visibility: {{ .VisibilityStatus }}) +
    • + {{ end }} +
+``` + +#### GetClientDescription + +Returns an application description given a credential ID. + +##### Example Usage +``` +{{ range $app.Credentials }} +... +{{ GetClientDescription .ID}} +{{end}} +``` + +#### GetClientName + +Returns an application name given a credential ID. + +##### Example Usage +``` +{{ range $app.Credentials }} +... +{{ GetClientName .ID}} +{{end}} +``` + +#### GetMenus + +Returns a map of all [menus](/portal/customization/menus). + +##### Example Usage +```html expandable +{{ if GetMenus.Primary }} + {{ range GetMenus.Primary.Children }} + {{ range .Children }} +
  • + {{ .Tag }} +
    • + {{ end }} + {{ end }} +{{ end }} +``` + +#### GetProducts + +Returns the list of products for the current user. Expects the request as an argument. + +##### Product Attributes + +Accessible via `{{ range $product := GetProducts req }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ $product.ID }}` | Product ID | +| `{{ $product.Name }}` | Product name | +| `{{ $product.DisplayName }}` | Product display name | +| `{{ $product.Path }}` | Product path | +| `{{ $product.ReferenceID }}` | Product reference ID | +| `{{ $product.Description }}` | Product description | +| `{{ $product.AuthType }}` | Product auth type | +| `{{ $product.Scopes }}` | Product scopes | +| `{{ $product.Logo.URL }}` | Product logo URL | +| `{{ $product.Feature }}` | true if the product is featured | +| `{{ $product.DCREnabled }}` | true if DCR is enabled | +| `{{ $product.ProviderID }}` | Provider ID | +| `{{ $product.APIDetails }}` | Array of API details associated with the product | +| `{{ $product.Catalogues }}` | Array of catalogues associated with the product | + +##### API Details Attributes (Within product) + +Accessible via `{{ range $api := $product.APIDetails }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ $api.Name }}` | API name | +| `{{ $api.Description }}` | API description | +| `{{ $api.APIType }}` | API type | +| `{{ $api.TargetURL }}` | API target URL | +| `{{ $api.ListenPath }}` | API listen path | +| `{{ $api.OASUrl }}` | API OAS URL | +| `{{ $api.Status }}` | "Active" if API status is active, otherwise "Inactive" | + +##### Catalogue Attributes (Within product) + +Accessible via `{{ range $catalogue := $product.Catalogues }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ $catalogue.Name }}` | Catalogue name | +| `{{ $catalogue.VisibilityStatus }}` | Catalogue visibility status | + + +```html expandable +{{ range GetProducts req }} +
    +
    + {{ if .Logo.URL }} + + {{ end }} +
    +
    +
    + {{ .AuthType }} +
    +

    {{ .ProductName }}

    +
    + {{ if .Description }} +

    {{ .Description }}

    + {{ end }} +
    +
    +
    + More Info +
    +
    +
    +
    +{{ end }} +``` +     + +#### IsPortalDisabled + +Returns true (exception: for admins is always enabled) if portal visibility was set to hidden. Expects the request as +parameter. + +##### Example Usage +``` +{{ $portalDisabled := IsPortalDisabled req }} +``` + +#### IsPortalPrivate + +Returns true (exception: for admins is always enabled) if portal visibility was set to private. Expects the request as +parameter. + +##### Example Usage +``` +{{ $portalPrivate := IsPortalPrivate req }} +``` + +#### ProductDocRenderer + +Returns the configured product OAS renderer (redoc or stoplight). + +##### Example Usage +``` +{{ $oas_template := ProductDocRenderer }} +``` + +#### ProviderUpstreamURL + +Returns the provider upstream URL for a given providerID. Expects the request and a provider ID as parameters. + +##### Example Usage +``` +{{ $upstreamURL := ProviderUpstreamURL req $thisProduct.ProviderID }} +``` + +#### SplitStrings + +Splits a given string with given separator and returns a slice of split strings. + +##### Example Usage +``` expandable +{{ range $app.Credentials }} +... +{{ range SplitStrings .GrantType "," }} +... +{{ end }} +{{ end }} +``` + +#### TruncateString + +Truncates a given string to a given length, returning the truncated string followed by three dots (…). + +##### Example Usage +``` +{{ TruncateString $api.Description 60 }} +``` + +#### TypeOfCredential + +Returns the credential type ("oAuth2.0" or "authToken") given the credential. + +##### Example Usage +``` expandable +{{ range $app.Credentials }} +... +{{ if eq (TypeOfCredential . ) "oAuth2.0" }} +... +{{ end }} +{{end}} +``` + + +## Email Templates + +This section provides a detailed overview of the email template data available in the Tyk Enterprise Developer Portal. +The Tyk Enterprise Developer Portal uses a variety of email templates for different purposes, such as user registration +and access request status or organization status updates. Each template has access to specific data or functions relevant +to its purpose. + +It's important to note that while email templates can include template data or specific template functions, they do not +have access to the global helper functions available in other portal templates. + +Please refer to [email workflow](/portal/customization/email-notifications) +for additional detail on email notifications sent by the portal. + + +### Available Email Templates + +- [Access Request Approve/Reject](#access-request-approvereject) +- [Access Request Submitted](#access-request-submitted) +- [Activate and Deactivate](#activate-and-deactivate) +- [New User Request](#new-user-request) +- [Organization Approve](#organization-approve) +- [Organization Reject](#organization-reject) +- [Organization Request](#organization-request) +- [Reset Password](#reset-password) +- [Targeted Invite](#targeted-invite) +- [Welcome User](#welcome-user) + +#### Access Request Approve/Reject + +**Template Paths**: +- `themes/default/mailers/approve.tmpl` +- `themes/default/mailers/reject.tmpl` + +These templates are used for sending notifications to users when their access requests are approved or rejected. + +##### Available Objects + +There's no data sent to these templates. + +##### Example Usage +``` +Hi, +The API Credentials you provisioned have been rejected. +Thanks, +The Team +``` + +#### Access Request Submitted + +**Template Path**: `themes/default/mailers/submitted.tmpl` + +This template is used for notifying administrators about pending access requests. + +##### Available Objects + +- `{{ .requests }}`: Returns the list of access requests pending approval. + +##### Access Request Attributes + +Accessible via `{{ range .requests }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ .PlanID }}` | Plan ID associated with access request | +| `{{ .Status }}` | Request status | +| `{{ .AuthType }}` | Request authentication type | +| `{{ .UserID }}` | User ID associated with the request | +| `{{ .ClientID }}` | Client ID associated with the request | +| `{{ .DCREnabled }}` | Indicates if DCR (Dynamic Client Registration) is enabled for the request | +| `{{ .ProvisionImmediately }}` | Indicates if provisioning is immediate for the request | +| `{{ .CatalogueID }}` | Catalogue ID associated with the request | + +##### Product Attributes (within Access Request) + +Accessible via `{{ range $product := $acreq.Products }}` + +| Attribute | Description | +| :----------- | :------------- | +| `{{ $product.ID }}` | Product ID | +| `{{ $product.Name }}` | Product name | +| `{{ $product.DisplayName }}` | Product display name | +| `{{ $product.Description }}` | Product description | +| `{{ $product.AuthType }}` | Product authentication type | +| `{{ $product.DCREnabled }}` | Indicates if DCR (Dynamic Client Registration) is enabled for the product | + +##### Example Usage +```html expandable +

    A new Access request has been submitted. Please log in to the administration dashboard to view the request.

    +
      + {{ range $acreq := .requests }} +
    • + Status: {{ $acreq.Status }}
      + User ID: {{ $acreq.UserID }}
      + Products: +
        + {{ range $product := $acreq.Products }} +
      • {{ $product.DisplayName }} ({{ $product.AuthType }})
        • + {{ end }} +
      +
      • + {{ end }} +
    +``` + + +#### Activate and Deactivate + +**Template Paths**: +- `themes/default/mailers/activate.tmpl` +- `themes/default/mailers/deactivate.tmpl` + +These templates are used for sending activation and deactivation notifications to users. + +##### Available Objects + +- `{{ .name }}`: Returns the user's full name. + +##### Example Usage +``` +Hi, {{.name}}
    +Your account has been activated. +``` + +#### New User Request + +**Template Path**: `themes/default/mailers/newuser.tmpl` + +This template is used for notifying administrators about new user registration requests pending activation. + +##### Available Objects + +- `{{ .user }}`: Returns the new user pending activation. + +#### User Attributes + +Accessible via `{{ .user }}` + +| Attribute/Method | Description | +| :------------------- | :------------- | +| `{{ .ID }}` | User ID | +| `{{ .First }}` | User name | +| `{{ .Last }}` | User surname | +| `{{ .Email }}` | User email | +| `{{ .OrganisationID }}` | User organization ID | +| `{{ .DisplayName }}` | User complete name | +| `{{ .IdentityProvider }}` | User provider (Portal or Tyk Identity Broker) | +| `{{ .GetOrganisationID }}` | User's organization ID | +| `{{ .IsAdmin }}` | true if user is an admin | +| `{{ .IsOrgAdmin }}` | true if user is an organization admin | +| `{{ .DisplayRole }}` | User's role | +| `{{ .Organisation.Name }}` | Organization name | +| `{{ .Teams }}` | Array of user teams | +| `{{ .Teams.ID }}` | Team ID | +| `{{ .Teams.Name }}` | Team name | +| `{{ .Teams.Default }}` | Indicates if the team is the default team (true/false) | + +##### Example Usage +``` expandable +

    There is a new user request pending. Please approve it from the admin console.

    +

    + Id: {{ .user.ID }}
    + User: {{ .user.DisplayName }} ({{ .user.Email }})
    + Role: {{ .user.Role }}
    + {{ if gt .user.OrganisationID 0 }} + Organisation: {{ .user.Organisation.Name }}
    + {{ else }} + Organisation: Administrators' organisation
    + {{ end }} + {{ if gt (len .user.Teams) 0 }} + Teams:
    +

      + {{ range .user.Teams }} +
    • {{ .Name }}
      • + {{ end }} +
    + {{ else }} + Teams: none + {{ end }} +

    +``` + +#### Organization Approve + +**Template Path**: `themes/default/mailers/organisation_request.tmpl` + +This template is used for notifying users that their organization creation request has been approved. + +##### Available Objects + +- `{{ site }}`: Returns the application host. + +##### Example Usage +``` +Hello, +The organization registration request has been approved. You can now manage your organization in your dashboard here: https://{{.site}}/portal/private/dashboard +Thanks, +The team +``` + +#### Organization Reject + +**Template Path**: `themes/default/mailers/organisation_reject.tmpl` + +This template is used for notifying users that their organization creation request has been rejected. + +##### Available Objects + +There's no data sent to this template. + +##### Example Usage +``` +Hello, +The organization registration request has been rejected. +Thanks, +The team +``` + +#### Organization Request + +**Template Path**: `themes/default/mailers/organisation_request.tmpl` + +This template is used for notifying administrators about new organization creation requests. + +##### Available Objects + +- `{{ .user }}`: Returns the user who made the request. +- `{{ .organisationName }}`: Returns the new organization name. + +#### User Attributes + +Accessible via `{{ .user }}` + +| Attribute/Method | Description | +| :------------------- | :------------- | +| `{{ .ID }}` | User ID | +| `{{ .First }}` | User name | +| `{{ .Last }}` | User surname | +| `{{ .Email }}` | User email | +| `{{ .OrganisationID }}` | User organization ID | +| `{{ .DisplayName }}` | User complete name | +| `{{ .IdentityProvider }}` | User provider (Portal or Tyk Identity Broker) | +| `{{ .GetOrganisationID }}` | User's organization ID | +| `{{ .IsAdmin }}` | true if user is an admin | +| `{{ .IsOrgAdmin }}` | true if user is an organization admin | +| `{{ .DisplayRole }}` | User's role | + +##### Example Usage +``` +There is a new organization registration request pending. Please approve it from the admin console. +The organization name: {{ .organisationName }}. +The user: {{ .user.DisplayName }} ({{ .user.Email }}). +``` + +#### Reset Password + +**Template Path**: `themes/default/mailers/auth/reset_password.tmpl` + +This template is used for sending password reset emails to users. + +##### Available Functions + +- `{{ current_user }}`: Returns the current user object. +- `{{ reset_password_url }}`: Returns the URL with the token for setting the password. + +##### User Attributes + +Accessible via `{{ current_user }}` + +| Attribute/Method | Description | +| :------------------- | :------------- | +| `{{ .ID }}` | User ID | +| `{{ .First }}` | User name | +| `{{ .Last }}` | User surname | +| `{{ .Email }}` | User email | +| `{{ .Role }}` | User role | +| `{{ .OrganisationID }}` | User organization ID | +| `{{ .DisplayName }}` | User complete name | +| `{{ .IdentityProvider }}` | User provider (Portal or Tyk Identity Broker) | +| `{{ .GetOrganisationID }}` | User's organization ID | +| `{{ .IsAdmin }}` | true if user is an admin | +| `{{ .IsOrgAdmin }}` | true if user is an organization admin | +| `{{ .DisplayRole }}` | User's role | + +##### Example Usage +``` expandable +{{ $user := current_user}} +

    Hello {{ $user.DisplayName }},

    +

    Someone has requested a link to change your password. You can do this through the link below.

    +

    {{reset_password_url}}

    +

    If you didn't request this, please ignore this email.

    +

    Your password won't change until you access the link above and create a new one.

    +``` + +#### Targeted Invite + +**Template Path**: `themes/default/mailers/auth/targeted_invite.tmpl` + +This template is used for sending targeted invitations to users. + +##### Available Functions + +- `{{ user }}`: Returns the targeted user object. +- `{{ team }}`: Returns the team name to which the user is being invited. +- `{{ invite_url }}`: Returns the URL with the token for setting the password. + +##### User Attributes + +Accessible via `{{ user }}` + +| Attribute/Method | Description | +| :------------------- | :------------- | +| `{{ .ID }}` | User ID | +| `{{ .First }}` | User name | +| `{{ .Last }}` | User surname | +| `{{ .Email }}` | User email | +| `{{ .Role }}` | User role | +| `{{ .OrganisationID }}` | User organization ID | +| `{{ .DisplayName }}` | User complete name | +| `{{ .IdentityProvider }}` | User provider (Portal or Tyk Identity Broker) | +| `{{ .GetOrganisationID }}` | User's organization ID | +| `{{ .IsAdmin }}` | true if user is an admin | +| `{{ .IsOrgAdmin }}` | true if user is an organization admin | +| `{{ .DisplayRole }}` | User's role | + +##### Example Usage +```html +{{ $u := user }} +Hi, {{ $u.DisplayName }}
    +

    Someone is inviting you to join {{ if $u.IsAdmin }}as an Administrator{{ else }}the {{ team }} team{{end }}. You can do this through the link below.

    +

    {{ invite_url }}

    +

    If you didn't request this, please ignore this email.

    +``` + +#### Welcome User + +**Template Paths**: +- `themes/default/mailers/welcome_admin.tmpl` +- `themes/default/mailers/welcome_dev.tmpl` + +These templates are used for sending welcome emails to new users, with separate templates for administrators and developers. + +##### Available Objects + +- `{{ .user }}`: Returns the user who made the request. Refer to the CurrentUser section for accessible attributes and methods. + +#### User Attributes + +Accessible via `{{ .user }}` + +| Attribute/Method | Description | +| :------------------- | :------------- | +| `{{ .ID }}` | User ID | +| `{{ .First }}` | User name | +| `{{ .Last }}` | User surname | +| `{{ .Email }}` | User email | +| `{{ .OrganisationID }}` | User organization ID | +| `{{ .DisplayName }}` | User complete name | +| `{{ .IdentityProvider }}` | User provider (Portal or Tyk Identity Broker) | +| `{{ .GetOrganisationID }}` | User's organization ID | +| `{{ .IsAdmin }}` | true if user is an admin | +| `{{ .IsOrgAdmin }}` | true if user is an organization admin | +| `{{ .DisplayRole }}` | User's role | +| `{{ .Organisation.Name }}` | organization name | +| `{{ .Teams }}` | Array of user teams | +| `{{ .Teams.ID }}` | Team ID | +| `{{ .Teams.Name }}` | Team name | +| `{{ .Teams.Default }}` | Indicates if the team is the default team (true/false) | + + +```html expandable +

    Welcome to Tyk Enterprise Developer Portal

    +

    Hello {{ .user.DisplayName }},

    +

    Your account has been created for the {{ .user.Organisation.Name }} organisation.

    +

    Your assigned teams:

    +
      + {{ range .user.Teams }} +
    • {{ .Name }}{{ if .Default }} (Default){{ end }}
      • + {{ end }} +
    +

    We're excited to have you on board!

    +``` +     diff --git a/portal/customization/themes.mdx b/portal/customization/themes.mdx new file mode 100644 index 00000000..c5857f0f --- /dev/null +++ b/portal/customization/themes.mdx @@ -0,0 +1,277 @@ +--- +title: "Customize Themes in Developer Portal" +'og:description': "How to customize themes in developer portal" +sidebarTitle: "Themes" +--- + +## What is a Theme? + +The Tyk Enterprise Developer Portal uses **themes** for customizing the live portal. We provide an out of the box theme that is using our own branding, it’s called the `default` theme. You are welcome to use it and modify it for your needs, yet if you want to start with a blank page, you can also create a completely new theme. + +The following section explains how they are structured and their main concepts. We recommend you to read this if you are creating your own theme, or making extensive changes to the ones we provide. + +## File Structure of a Theme + +Generally speaking, a theme defines an application’s styling, templates and scripts. +In the Tyk Developer Portal a `themes` folder is located in the root of the application and is the directory where each theme folder must be added. If you navigate to `path /themes/` you’ll see our default theme which has the following structure: + +Default Tyk Enterprise Portal theme structure + +- Manifest file (`theme.json`): It uses JSON syntax to define theme metadata (name, version and author) as well as a list of templates that are part of the theme. +- `assets`: It intended for static assets like CSS, JS or images that are used by the theme. All contents from this directory are mounted under the `/assets` path in the portal HTTP server. +- `layouts`: The layout is the top level view of your theme. +- `views`: The view is rendered as a part of a layout. Each view can be rendered using a different layout. +- `partials`: Partials provide an easier way to handle snippets of code that are reused across different views or layouts, for example if you want to inject a JS snippet that’s used in different places, you could set this code in a partial and include it anywhere by using the appropriate 'Go template directive'. In this way you could improve code readability and organize the theme in the most efficient way. + +### Manifest File + +This file should sit in the root of a theme and hold the theme configuration. You can define a name and your templates along other options such as the version and the author. + +You can find an example of the manifest within the `default` theme that is located in `/themes/default`. The syntax looks as follows: + +```json expandable +{ + "name": "default", + "version": "0.0.1", + "author": "Tyk Technologies Ltd. ", + "templates": [ + { + "name": "Content Page", + "template": "page", + "layout": "site_layout" + }, + { + "name": "Portal Home", + "template": "portal_home", + "layout": "portal_layout" + }, + { + "name": "Home", + "template": "home", + "layout": "portal_layout" + }, + { + "name": "Catalogue", + "template": "catalogue", + "layout": "portal_layout" + } + ] +} +``` + +The `templates` field establishes a list of available templates. Every template consists of three fields where `name` is a user-friendly name that will be seen on the Admin app when creating a page. `template` is a reference to the template file itself. `layout` is a reference to the layout that will be used to render the previously set template. + +To illustrate the current template hierarchy, this is what a typically rendered page would look like. The `layout` would be the top level template and base structure of the page: +Template structure + + +Also note that the Developer Portal will let you use not just multiple `layouts` and `views` but also any combination of them. These combinations are set in your manifest file (`theme.json`). + +Regarding `partials`, even though the illustration above shows two partials embedded on the `view` section, `partials` are intended for using in any place. You should be able to embed a `partial` directly into a layout, or even in multiple layouts. + +Content blocks are explored more deeply in the next sections. To summarise, its relationship with the above hierarchy is when rendering a particular page, a `layout`, a `view` and potentially a combination of partials get loaded from the theme directory. Content blocks are different because their content gets dynamically populated by database content. These contents are created from the Admin section. + +To be Concluded: + +- A layout is the wrapper of everything you want to include inside it. So, typically it would consist of tags such as ``, ``, ``, ``, and `<body>`. +- A `template` is what we would inject in a layout and specifically within the `<body>` of a layout. +- A `partial` can be, for example, the navigation menu so that you can inject it in the layout and it will be visible every time this layout is used + +### Go Templates + +All theme template files use the Go template syntax. You will find every file in the layouts and views. Partials directory uses the `.tmpl` file extension, which is the default Go template extension. Go templates work in a similar way to ERB or EJS templates by letting the user mix HTML code with dynamic values. Sample syntax is as follows: + +`{{ render β€œtop_nav” }}` + +The code sample above would execute the `render` template helper, which is a common function that’s used to inject code from other `views` into the current one. You may use this to embed content from other parts of the theme, typically `partials` or `views`. In this case, it will insert a `view` or `partial` named `top_nav` to the template where it’s used. + +The same delimiters `{{` and `}}` are used for all Go template directives. We’ll explore some of them in the upcoming sections. + +See the [Go package template documentation](https://pkg.go.dev/text/template#pkg-overview) for more information. + +### Content Blocks + +The Developer Portal themes use content blocks to facilitate content management. A content block is defined as a part of a `view` by using a particular template directive in combination with a name or ID to identify the given block. For example, if you check the `home` template in the default theme (`themes/default/views/home.tmpl`), you will find the following code: + +```go expandable +div class="container"> + <div class="row"> + <div class="col-sm-6"> + <div class="text-container"> + <h1>{{.page.Title}}</h1> + <p>{{.blocks.HeaderDescription.Content}}</p> + <a href="{{.blocks.HeaderButtonLink.Content}}" class="btn btn-primary">{{.blocks.HeaderButtonLabel.Content}}</a> + </div> +…. +``` + +There are four code references in the above snippet. In this example we have a header, some text and then a button that act as a link. Let's see what each one is and how it correlates with the UI. + +1. `{{ .page.Title }}`. This is the `Title` input in the form UI (Screenshot #1) +1. `{{ .blocks.HeaderDescription.Content }}`. This is the `HeaderDescription` input in the form UI (Screenshot #2) +2. `{{ .blocks.HeaderButtonLink.Content }}`. This is the `HeaderDescription` input in the form UI (Screenshot #3) +3. `{{ .blocks.HeaderButtonLabel.Content }}`. This is the `HeaderButtonLabel` input in the form UI (Screenshot #4) + +<img src="/img/dashboard/portal-management/enterprise-portal/go-template-ui.png" alt="Go template blocks and portal UI" /> + +This will display in your portal as following: + +<img src="/img/dashboard/portal-management/enterprise-portal/example-portal-content-block.png" alt="Example Portal content block" /> + +In order for a page to render properly the content manager will need to be aware of the content blocks that are required by a particular template. + +## Managing Themes + +The Tyk Enterprise Developer Portal enables the admin users and developers to manage themes and select which theme is visible in the portal. +To enable this capability, the portal has theme management UI. + +### Create a Theme +Follow the example below to create a new theme called "TestTheme" using the default theme as a blueprint: + +1. As an admin user, navigate to the Theme management UI and download the default theme. The Tyk Enterprise Developer Portal doesn't allow modifications to the default theme so that you will always have access to the vanilla theme. + <img src="/img/dashboard/portal-management/enterprise-portal/download-default-theme.png" alt="Download default theme" /> +2. Unzip the theme and rename it by modifying the Manifest file as described above. + <img src="/img/dashboard/portal-management/enterprise-portal/rename-a-theme.png" alt="Rename theme" /> +3. You can also modify other assets in the theme as described later in this guide. Once all modifications are done, you need to zip the theme and upload it to the portal. + <img src="/img/dashboard/portal-management/enterprise-portal/compress-a-theme.png" alt="Zip theme" /> +4. To upload the theme as an admin user, navigate to **Themes** and click on the **Add new theme** button. Please note that the size of individual files should not exceed 5 MB and the total size of all files in the theme should not exceed `PORTAL_MAX_UPLOAD_SIZE`. This parameter is [configurable](/product-stack/tyk-enterprise-developer-portal/deploy/configuration#portal_max_upload_size). + <img src="/img/dashboard/portal-management/enterprise-portal/add-a-new-theme.png" alt="Add new theme" /> +5. Then click on the **Add theme file** button. + <img src="/img/dashboard/portal-management/enterprise-portal/add-theme-file.png" alt="Add theme file" /> +6. Select the archive that you've created earlier and click on the **Save** button. + <img src="/img/dashboard/portal-management/enterprise-portal/save-new-theme.png" alt="Save new theme" /> +7. Now you should see a success message meaning the theme was successfully created. + <img src="/img/dashboard/portal-management/enterprise-portal/new-theme-is-created.png" alt="Theme is created" /> + +### Preview a Theme +The Tyk Enterprise Developer Portal enables the admin users to preview the theme before it gets reflected on the public-facing portal. This enables to review the changes that are made to the theme before exposing them to the developer community. +1. To preview a theme as an admin user, navigate to the **Themes** menu. Select a theme, and click on the **Preview** button. + <img src="/img/dashboard/portal-management/enterprise-portal/preview-theme-button.png" alt="Preview theme" /> +2. The previewer will open the selected theme in a new tab. Now you can browse your theme and review the changes. For the demonstration purposes, we've modified the API Catalog page so it displays "Modified catalog" instead of "Product Catalogs". + <img src="/img/dashboard/portal-management/enterprise-portal/theme-preview.png" alt="Preview theme" /> +3. Once the review is done, you can quit the preview by clicking on the **Quit preview button**. + <img src="/img/dashboard/portal-management/enterprise-portal/quit-theme-preview.png" alt="Quite theme preview" /> + +### Activate a Theme +The Tyk Enterprise Developer Portal enables you to have multiple themes at the same time but only one of them is active. +1. As an admin user, navigate to the **Themes** menu. The current status of each theme is displayed in the **Status** column. + <img src="/img/dashboard/portal-management/enterprise-portal/default-theme-is-current.png" alt="Default theme is the current theme" /> +2. To activate the new theme, click on the **Activate** button. + <img src="/img/dashboard/portal-management/enterprise-portal/activate-a-theme.png" alt="Activate theme" /> +3. The selected theme is now active and displayed to all API consumers on the Live portal. + <img src="/img/dashboard/portal-management/enterprise-portal/modified-theme-is-active.png" alt="Modified theme is activated" /> + +### Modify an Existing Theme +The Tyk Enterprise Developer Portal enables modification to any existing theme, except the default one. +1. To start modification of any existing theme, navigate to the **Themes** menu and download the theme package. + <img src="/img/dashboard/portal-management/enterprise-portal/download-a-theme.png" alt="Download existing theme" /> +2. Unzip the package, do any required modification and zip it back. You should keep the name of the theme. If you need to change the name of the theme, you will need to create a new theme as described above. +3. As an admin user, navigate to the **Themes** menu and select the modified theme. + <img src="/img/dashboard/portal-management/enterprise-portal/select-a-modified-theme.png" alt="Select modified theme" /> +4. Click on the **Add Theme File** button and select the theme archive. + <img src="/img/dashboard/portal-management/enterprise-portal/add-theme-file-to-existing-theme.png" alt="Add theme file" /> +5. Click on the **Save changes** button to save changes to the theme. + <img src="/img/dashboard/portal-management/enterprise-portal/save-changes-to-theme.png" alt="Save changes" /> +6. If the theme is the current changes to the Live portal, it will be applied immediately. Otherwise, you can preview and activate the theme as described above. + +## Upgrading Themes + +The Tyk Enterprise Developer Portal does not automatically update the default theme with each new release of the product, because doing so could result in the loss of customizations made by customers. +Therefore, customers are required to manually upgrade their themes to access the latest updates and fixes. We recommend using GitFlow for the latest theme updates. + +Alternatively, you can download the theme package from the **Releases** section of the [portal-default-theme](https://github.com/TykTechnologies/portal-default-theme) repository. +However, we advise against this method, as it requires you to merge your customized theme with the downloaded one, which is much simpler to accomplish via GitFlow. +Follow the guide below to obtain the latest version of the portal theme and merge it with your customized version. + +### Merge Latest Changes using Gitflow + +The default theme for the Tyk Enterprise Developer Portal is located in the [portal-default-theme](https://github.com/TykTechnologies/portal-default-theme) repository. +The `main` branch contains code corresponding to the latest stable release. If you wish to check out a specific release (e.g., v1.8.3), you can use tags: + +```console +git checkout tags/1.8.3 -b my-custom-theme branch +``` + +To organize local development in a way that facilitates seamless theme updates using git merge, follow the steps below: +1. Fork the `portal-default-theme` repo in [github](https://github.com/TykTechnologies/portal-default-theme). + <img src="/img/dashboard/portal-management/enterprise-portal/fork-portal-theme-repo.png" alt="Fork the portal-theme repo" /> + +2. Clone the forked repository: +```console +git clone https://github.com/my-github-profile/portal-default-theme && cd ./portal-default-theme +``` + +3. If you have an existing repository, check if you already have the `portal-default-theme` repo among your remotes before adding it. Execute the following command to check: +```console +git remote -v | grep 'TykTechnologies/portal-default-theme' +``` + +Skip the next step if you see the following: +```console +# portal-default-theme https://github.com/TykTechnologies/portal-default-theme (fetch) +# portal-default-theme https://github.com/TykTechnologies/portal-default-theme (push) +``` + +If the output of the above command is empty, proceed to step 5 to add the `portal-default-theme`. + +4. Add the `portal-default-theme` to the remotes by executing the following command: +```console +git remote add portal-default-theme https://github.com/TykTechnologies/portal-default-theme +``` + +5. Create a new local branch that tracks the remote `main` branch. That branch will mirror the latest changes from the `portal-default-theme` main. You will be using it to import the latest changes from the `portal-default-theme` to your custom theme: +```console +git fetch portal-default-theme main:portal-default-theme-main +``` + +If you have an existing local branch that tracks the `main` branch in the `portal-default-theme` repo, you can just pull the latest updates: +```console +git checkout portal-default-theme-main +git pull portal-default-theme main +``` + +6. To start customizing the theme, create a local develop branch from the `portal-default-theme-main`: +```console +git checkout portal-default-theme-main +git checkout -b dev-branch +``` + +7. Once the required customizations are completed, commit the changes to the `dev-branch`. + +8. Merge the latest updates from the `portal-default-theme` into the `dev-branch`. Please remember to always pull the latest changes from the `portal-default-theme-main` branch before merging: + - Checkout to the portal-default-theme-main and fetch the latest changes. + ```console + git checkout portal-default-theme-main + git pull portal-default-theme main + ``` + - Checkout the dev-branch and merge the changes from the portal-default-theme-main to retrieve the latest changes from the portal-default-theme repo with the customized theme. + ```console + git checkout dev-branch + git merge portal-default-theme-main + ``` + +Finally, address merge conflicts and commit changes. + + +<Note> +**You have successfully updated your custom theme** + +Now you can repeat the above described process when upgrading the portal version to make sure you have incorporated the latest theme changes to your customized theme. +</Note> + + +### Upload the Theme to the Portal +Once you have merged your local changes with the latest changes from the `portal-default-theme` repo, you need to create a zip archive with the customized theme and upload it to the portal. +1. Create a zip archive with the customized theme. Make sure you zip the content of the `src` folder and not the folder itself. To create a zip archive with the customized theme execute the following: + - cd to the `src` directory to make sure you: + ```console + cd ./src + ``` + - zip the content of the `src` directory: + ```console + zip -r9 default.zip + ``` + +2. Upload the theme package that is created in the previous step to the portal. You can use the portal's [Admin dashboard](/portal/customization/themes#create-a-theme) or the [admin API](/product-stack/tyk-enterprise-developer-portal/api-documentation/tyk-edp-api) to do it. +![image](https://github.com/TykTechnologies/tyk-docs/assets/14009/f0e547b2-b521-4c3e-97ce-fd3a2a3b170b) +3. Finally, you need to [activate](/portal/customization/themes#activate-a-theme) the theme so that it will be applied to the portal. diff --git a/portal/customization/user-model.mdx b/portal/customization/user-model.mdx new file mode 100644 index 00000000..bb3ba054 --- /dev/null +++ b/portal/customization/user-model.mdx @@ -0,0 +1,70 @@ +--- +title: "Customize User Model in Developer Portal" +'og:description': "How to customize user model in developer portal" +sidebarTitle: "User Model" +--- + +A **Model** in developer portal represents an physical entity. Currenlty, we only have the User model, which represent a user who will be consuming the API by signing up. + +In this section, you will learn how to customize the User model and the sign-up form for your API consumers. +Customizing the User model enables the storage of custom data attributes in the User profile. +Additionally, it allows these attributes to be optionally included in the credentials metadata (therefore accessible by the gateway runtime) and exposed in the user sign-up form. + +This feature enables the implementation of complex business logic when processing API requests. +For example, it is particularly useful when the quota for API calls needs to be distributed among all developers of consumer organizations. +In such cases, both the quota and the rate limit should be applied at the organization level, rather than according to individual credentials. +In this event, the organization ID should be known to the gateway in runtime. This feature helps to achieve that. + +## Add Custom Attributes to the User Model + +To customize the User model, navigate to the **Custom attributes** menu and then select the **User** model. Currently, it is possible to extend only the User model. In future releases we will add the same capabilities to other models. +<img src="/img/dashboard/portal-management/enterprise-portal/navigate-to-user-attributes.png" alt="Navigate to the User's attributes" /> + +To add a new attribute to the user model, click on the **Add Custom attribute** button and then fill in properties of the new attribute: +- **Attribute ID**: A string that consists of letters (a-zA-Z), numbers (0-9), dashes, and underscores. This is used to reference the attribute in the [Admin APIs](/product-stack/tyk-enterprise-developer-portal/api-documentation/tyk-edp-api) screen. +- **Attribute Label**: The attribute's name that is displayed in the UI. +- **Description**: Explains the intended usage of this attribute. It is also displayed in the UI. +- **Type of attribute**: The type of data that can be stored in this attribute. You cannot change the value of this field once the attribute is created. The following data types are acceptable: + - Boolean (true or false). + - Dropdown (a list of values). + - String. + - Number. +- **Validation Reg Exp**: A regular expression that is used to validate the value of this field. It is available for the **String** data type only. +- **Enable validation**: Determines if the portal should apply the regular expression defined in the **Validation Reg Exp** to validate the value of this attribute when creating or updating a user profile. It is available for the **String** data type only. +- **Dropdown Values**: A list of values for the attribute. It is available for the **Dropdown** data type only. +- Fields that define the attribute's behavior: + - **Write once read many**: Determines whether the value of the attribute can be changed after a user profile is created. This means that when **Write once read many** is enabled, the value of this attribute can be set only during the creation of a user profile. After the user profile is created, the value of this attribute cannot be edited, either through the admin APIs or via the Users UI. + - **Add to the key metadata**: Determines if the value of the attribute should be added to the metadata of Auth keys or OAuth2.0 clients when a user creates them. Keep in mind that credential-level metadata will be accessible in both the gateway runtime and gateway database. Please be cautious when handling personally identifiable information (PII) data. + - **Required**: Determines if this attribute is required to create a user profile. + - **Show on sign-up form**: Determines if this attribute should be visible in the sing-up form. +- **Behavior**: Determines if developers can view or edit this attribute. Possible values are: + - Developers can view and edit the attribute. + - Developers can only view the attribute. + - Developers cannot see the attribute. + +For the purpose of this guide, make sure to tick the **Required** and **Show on sign-up form** checkboxes and select the **Developers can only view the attribute** option. +<img src="/img/dashboard/portal-management/enterprise-portal/add-new-attribute-to-user-model.png" alt="Add a new attribute to the user model" /> + +The new attribute will be added to the user sign-up form, once you have created a new custom attribute and saved changes to the user model by clicking on the **Save** button. +<img src="/img/dashboard/portal-management/enterprise-portal/custom-attribute-in-the-sign-up-form.png" alt="Customized user sign-up form" /> + +## Default Attributes of User Model +By default, the portal assigns the following attributes to credentials metadata in the gateway when provisioning API credentials: +| Attribute | Name of the credential metadata field | Description | +| :----------------- | :--------------------------------------- | :----------------------------------------------------------------------------------- | +| Developer ID | DeveloperID | ID of the developer who created the credential | +| Application ID | ApplicationID | ID of the application to which it belongs | +| Organisation ID | OrganisationID | ID of the organization to which the developer who created the application belongs | +| Team IDs | TeamIDs | Array of team IDs to which the developer, who created the application, belongs | + +Additionally, it is possible to include other default attributes of the User model in the credential metadata fields. +However, it is important to remember that metadata at the credential level will be accessible both in the gateway runtime and in the gateway database. +Exercise caution when dealing with personally identifiable information (PII). Additional default attributes include: +| Attribute | Name of the credential metadata field | Description | +| :------------------- | :--------------------------------------- | :------------------------------------------------------------------------------------- | +| First name | First | First name of the developer who created the credential | +| Last name | Last | Last name of the developer who created the credential | +| Email | Email | Email name of the developer who created the credential | +| Role | Role | Array of team IDs to which the developer, who created the application, belongs | +| Organisation name | Organisation | Name of the organization to which the developer who created the application belongs | +| Teams name | TeamNames | Array of team names to which the developer, who created the application, belongs | diff --git a/portal/customization/webhooks.mdx b/portal/customization/webhooks.mdx new file mode 100644 index 00000000..5f70c749 --- /dev/null +++ b/portal/customization/webhooks.mdx @@ -0,0 +1,357 @@ +--- +title: "Customize Webhooks in Developer Portal" +'og:description': "How to customize webhooks in developer portal" +sidebarTitle: "Webhooks" +--- + +In this section, you will learn how to configure webhooks for events that occur within the portal. +Webhooks enable asynchronous integration with the portal by notifying third-party software about an event that has occurred. +This feature facilitates the implementation of complex business logic when the portal is integrated with third-party systems such as CRMs and ITSM systems. +Typical use cases for the webhooks include: +- An asynchronous approval that occurs externally (e.g., in a third-party CRM, ITSM, or another system managing approvals). In this scenario, an access request (such as an API product access request, an organization registration request, or a new developer profile in an inactive state) is created in the portal. The portal then informs the third-party system by calling a registered webhook. +- A follow-up action that needs to occur after a specific event in the portal. For example, after a developer profile is created, the customer must create a billing profile in their internal billing system (or a profile in a third-party billing engine such as Moesif, Lago, or a similar service) to automatically update and add this information into custom attributes. + +## Create a Webhook in Developer Portal + +The configuration process consists of two steps: +- Configure connectivity to the target endpoint by specify the Target URL, HTTP method, timeout, and request headers. +- Select types of events that should be sent to the target endpoint. + +1. **Configure the Target Endpoint** + + Each webhook delivers events to the **Target URL** via the specified **HTTP Method**. Additionally, it's possible to configure timeout header for requests. + + Finally, for each webhook it's possible to define HTTP headers that should be used for requests to the target URL via the **Headers** section. + To add a new header, click on the **Add Headers** button, specify **Name** and **Value** of the header. + + Note that you can test connectivity to the **Target URL** by clicking on the **Test Connection** button. + For testing connectivity, the portal sends a HEAD request to the specified target endpoint. + Please note that the connectivity is tested only with the HEAD method, and the test call does not include any headers defined in the **Headers** section. + + <img src="/img/dashboard/portal-management/enterprise-portal/edp-configure-webhook-channel.png" alt="Create new webhook channel" /> + + Once the target endpoint is configured, proceed to the next section to select the types of events that should be sent to that endpoint. + +2. **Select Event Types for the Webhook** + + To finish configuration, select types of events that should be sent to the **Target URL** and save the changes. Refer the docs below to know more about [supported event types](#supported-portal-events) + <img src="/img/dashboard/portal-management/enterprise-portal/edp-select-webhook-events-for-channel.png" alt="Select webhook events" /> + + +## Supported Portal Events + +The portal fires the following webhook events: +- [UserRegistered](/portal/customization/webhooks#new-user-registered) when a new user is registered. +- [UserAccountActivated](/portal/customization/webhooks#user-account-activated) when a user is activated. +- [UserAccountDeactivated](/portal/customization/webhooks#user-account-deactivated) when a user is deactivated. +- [PasswordReset](/portal/customization/webhooks#password-reset) when a user tries to reset a password. +- [ApplicationRegistered](/portal/customization/webhooks#new-application-registered) when a new API consumer application is created. +- [CredentialRegistered](/portal/customization/webhooks#new-credential-is-created) when a new API credential is created. +- [AccessRequestCreated](/portal/customization/webhooks#new-access-request-created) when a new API access request is created. +- [AccessRequestApproved](/portal/customization/webhooks#an-access-request-is-approved) when an API access request is approved. +- [AccessRequestRejected](/portal/customization/webhooks#an-access-request-is-rejected) when an API access request is rejected. +- [OrganizationRegistered](/portal/customization/webhooks#new-organization-registered) when an API consumer organization is created. +- [OrganizationRequestCreated](/portal/customization/webhooks#new-organization-registration-request-created) when a new API consumer organization registration request is created. +- [OrganizationRequestApproved](/portal/customization/webhooks#organization-registration-request-is-approved) when an API consumer organization registration request is approved. +- [OrganizationRequestRejected](/portal/customization/webhooks#organization-request-is-rejected) when an API consumer organization registration request is rejected. + +The complete list of events and their corresponding payloads is outlined below. + +### New User Registered +This event is fired whenever a new user is created via APIs, the admin UI, and the live portal UI (SSO or invite though the org dashboard or self-registration or invite code). + +Sample payload: +```json expandable +{ + "Event": "UserRegistered", + "Message": { + "ID": 29, + "Email": "developer@user.com", + "First": "FirstName", + "Last": "Lastname", + "OrgID": 1, + "Provider": "password", + "Status": "active", + "CreatedAt": "2024-04-22T16:38:54.068565+02:00", + "ByUser": 1, + "CustomAttributes": [ + { + "Identifier": "company-name", + "Value": "ACME" + } + ] + }, + "Timestamp": "2024-04-22T16:38:54.082037+02:00" +} +``` + +### User Account Activated +This event is fired whenever a user (either an admin or a developer) account is activated via APIs or the admin UI. + +Sample payload: +```json expandable +{ + "Event": "UserAccountActivated", + "Message": { + "ID": 7, + "Email": "devD1@tyk.io", + "First": "Test", + "Last": "User", + "OrgID": 7, + "Provider": "password", + "Status": "active", + "CreatedAt": "2024-04-22T15:46:40.128398Z", + "ByUser": 1, + "CustomAttributes": [ + { + "Identifier": "boolean-custom-attribute", + "Value": "false" + } + ] + }, + "Timestamp": "2024-04-22T17:52:22.673077+02:00" +} +``` + +### User Account Deactivated + +This event is fired whenever a user account is deactivated via APIs or the admin UI. + +Sample payload: +```json expandable +{ + "Event": "UserAccountDeactivated", + "Message": { + "ID": 7, + "Email": "test@user.io", + "First": "Test", + "Last": "User", + "OrgID": 7, + "Provider": "password", + "Status": "inactive", + "CreatedAt": "2024-04-22T15:46:40.128398Z", + "ByUser": 1, + "CustomAttributes": [ + { + "Identifier": "boolean-custom-attribute", + "Value": "false" + } + ] + }, + "Timestamp": "2024-04-22T17:51:22.24066+02:00" +} +``` + +### Password Reset + +This event is fired whenever a user tries to reset their password. + +Sample payload: +```json expandable +{ + "Event": "PasswordReset", + "Message": { + "ID": 7, + "Email": "test@user.io", + "First": "Test", + "Last": "User", + "OrgID": 7, + "Provider": "password", + "Status": "active", + "CreatedAt": "2024-04-22T15:46:40.128398Z", + "CustomAttributes": [ + { + "Identifier": "boolean-custom-attribute", + "Value": "false" + } + ] + }, + "Timestamp": "2024-04-22T17:58:10.223162+02:00" +} +``` + +### New Application Registered + +This event is fired whenever a new app is created via APIs, and the live portal UI (either via the checkout or the create app button in the developer’s dashboard). + +Sample payload: +```json expandable +{ + "Event": "ApplicationRegistered", + "Message": { + "ID": 1, + "Name": "New App", + "UserID": 1, + "CreatedAt": "2024-04-18T13:29:23.738726+02:00" + }, + "Timestamp": "2024-04-18T13:29:23.744826+02:00" +} +``` + +### New Credential Is Created + +This event is fired whenever a new credential is created via APIs, the admin UI (creation after approval) and the live portal UI. + +Sample payload: +```json expandable +{ + "Event": "CredentialRegistered", + "Message": { + "ID": 1, + "ByUser": 3, + "AccessRequestID": 1, + "AppID": 3, + "CreatedAt": "2024-04-18T13:48:08.489611+02:00" + }, + "Timestamp": "2024-04-18T13:48:08.494266+02:00" +} +``` + +### New Access Request Created + +This event is fired whenever a new access request is created via APIs and the live portal UI. + +Sample payload: +```json expandable +{ + "Event": "AccessRequestCreated", + "Message": { + "ID": 0, + "AppID": 1, + "ByUser": 2, + "Status": "approved", + "ProductIDs": [ + 1 + ], + "PlanID": 2, + "CreatedAt": "0001-01-01T00:00:00Z" + }, + "Timestamp": "2024-04-22T18:09:45.245357+02:00" +} +``` + +### An Access Request Is Approved + +This event is fired whenever an access request is approved or auto-approved via the admin APIs or admin UI. + +Sample payload: +```json expandable +{ + "Event": "AccessRequestApproved", + "Message": { + "ID": 1, + "AppID": 3, + "ByUser": 3, + "Status": "approved", + "ProductIDs": [ + 1 + ], + "PlanID": 2, + "CreatedAt": "2024-04-18T13:36:02.769109+02:00" + }, + "Timestamp": "2024-04-18T13:48:08.508925+02:00" +} +``` + +### An Access Request Is Rejected + +This event is fired whenever an access request is rejected via the admin APIs or the admin UI. + +Sample payload: +```json expandable +{ + "Event": "AccessRequestRejected", + "Message": { + "ID": 6, + "AppID": 7, + "ByUser": 3, + "Status": "rejected", + "ProductIDs": [], + "PlanID": 2, + "CreatedAt": "2024-04-18T14:40:15.81038+02:00" + }, + "Timestamp": "2024-04-18T14:40:28.998297+02:00" +} +``` + +### New Organization Registered + +This event is fired whenever a new consumer organization is created via the admin APIs, the live portal ([the become an organization flow](/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/manage-api-consumers)) or the admin UI. + +Sample payload: +```json expandable +{ + "Event": "OrganisationRegistered", + "Message": { + "ID": 8, + "Name": "Organisation added from Admin UI", + "CreatedAt": "2024-04-18T16:12:09.8437+02:00" + }, + "Timestamp": "2024-04-18T16:12:09.849045+02:00" +} +``` + +### New Organization Registration Request Created + +This event is fired whenever a new organization request is created via the live portal ([the become an organization flow](/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/manage-api-consumers)) or the admin UI. + +Sample payload: +```json expandable +{ + "Event": "OrganisationRequestCreated", + "Message": { + "Name": "Organisation added from Live Portal (the become an org flow)", + "AdminEmail": "dev@tyk.io", + "AdminID": 3, + "ByUser": 3, + "TeamIDs": [], + "Status": "pending", + "CreatedAt": "2024-04-18T16:13:50.766139+02:00" + }, + "Timestamp": "2024-04-18T16:13:50.796234+02:00" +} +``` + +### Organization Registration Request Is Approved + +This event is fired whenever an organization registration request is approved by an admin user. + +Sample payload: +```json expandable +{ + "Event": "OrganisationRequestApproved", + "Message": { + "ID": 11, + "Email": "dev@tyk.io", + "First": "Developer", + "Last": "User", + "OrgID": 2, + "Provider": "password", + "Status": "inactive", + "CreatedAt": "2024-04-24T15:26:04.312618088Z", + "CustomAttributes": [] + }, + "Timestamp": "2024-04-24T15:26:04.329072196Z" +} +``` + +### Organization Request Is Rejected + +This event is fired whenever a new organization request is rejected by an admin user. + +Sample payload: +```json expandable +{ + "Event": "OrganisationRequestRejected", + "Message": { + "Name": "ACME", + "AdminEmail": "dev@tyk.io", + "AdminID": 17, + "ByUser": 17, + "TeamIDs": [], + "Status": "rejected", + "CreatedAt": "2024-04-18T16:27:34.012613+02:00" + }, + "Timestamp": "2024-04-18T16:27:50.504654+02:00" +} +``` diff --git a/portal/install.mdx b/portal/install.mdx new file mode 100644 index 00000000..b98236f4 --- /dev/null +++ b/portal/install.mdx @@ -0,0 +1,1005 @@ +--- +title: "Install and Configure Developer Portal" +'og:description': "Installation guide for the Tyk Enterprise Developer Portal" +tags: ['Developer Portal', 'Tyk', 'Install Tyk Enterprise Developer Portal', 'Bootstrap Tyk Enterprise Developer Portal'] +sidebarTitle: "Install and Configure" +--- + +import { ResponsiveGrid } from '/snippets/ResponsiveGrid.mdx'; + + +<Note> +**Tyk Enterprise Developer Portal** + +If you are interested in getting access contact us at [support@tyk.io](<mailto:support@tyk.io?subject=Tyk Enterprise Portal Beta>) +</Note> + + +Tyk Enterprise Developer Portal is available as a Docker container. To install Tyk Enterprise Developer Portal, you need to launch the Docker image for the portal with a database to store the portal metadata. +Optionally, you may decide to use S3 to store the portal CMS assets (image and theme files) + +This guide explains how to install and bootstrap the Tyk Enterprise Developer Portal. On average, it should take around 5-10 minutes to install it depending on your setup. + +## Enterprise Developer Portal Components + +<img src="/img/dashboard/portal-management/enterprise-portal/portal-deployment-diagram.png" alt="Portal deployment diagram" /> +<br/> + +The portal deployment comprises of three main components: +- The portal application itself +- The portal's main database that stores metadata related to the portal, such as API products, plans, developers, applications, and more +- The asset storage, which stores CMS assets such as images, themes, and OpenAPI specification files. The assets could reside in the portal's main database or separately in an S3 bucket or filesystem volume. + +Optionally, there could be three additional components: +- **3rd party identity provider.** To [enable oAuth2.0 for your API Products](/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/dynamic-client-registration), you'll need to utilize an OpenID-compliant third-party identity provider. +It's essential to note that the [Tyk Stack](/tyk-stack) doesn't include third-party identity providers, so you should refer to your Identity Provider's documentation for instructions on configuring and deploying it. +This component is optional and required only for enabling oAuth2.0 +- **[Tyk Identity Broker](/api-management/external-service-integration)**. You only need this component if you want to configure Single Sign-On for the Tyk Enterprise Developer Portal. +For more guidance on this topic, please consult [the Single Sign-On section](/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/enable-sso) of the documentation +- **Email server**. The portal is capable of sending notifications to both admin users and developers when specific events happen within the portal. +To enable this feature, you need to specify a connection configuration to an email server or service, and configure other email settings. +You can choose to use a server that is installed on your premises or an SMTP-compatible SaaS product. +For step-by-step instructions, please refer to [the Email Settings section](/portal/customization/email-notifications) + +## Portal Installation Process + +The portal installation process comprises two steps: +1. **Install the portal application.** To install the portal and launch it in the bootstrap mode, you need to configure your portal instance by specifying settings such as TLS, log level, and database connection. +For further guidance on launching the portal, please refer to one of the installation options: [Docker container](/portal/install#docker), [Docker Compose](/portal/install#docker-compose), [Helm chart](/portal/install#using-legacy-helm-chart), or [RPM package](/portal/install#linux-redhat-centos). +2. **[Bootstrap the portal](#bootstrapping-enterprise-developer-portal)** After you've launched the portal, it will wait for you to provide credentials for the super admin user before it starts accepting traffic. +Once you've created the super admin user, the portal will complete its installation process by creating the necessary database structure and initialising the required assets for its operations. You can [bootstrap](#bootstrapping-enterprise-developer-portal) the portal either through the UI or using the bootstrap API. +Please refer to [the Bootstrapping section](/portal/install#bootstrapping-enterprise-developer-portal) for implementing this step. + +## Installation Options for Enterprise Developer Portal + +The Tyk Enterprise Developer Portal supports multiple installation flavors. Check out the guides below to deploy the portal on the platform that suits you best. + +<ResponsiveGrid> + + +<Card href="/portal/install#docker" img="/img/docker.png"> +**Read time: 10 mins** + +Install with Docker +</Card> + +<Card href="/portal/install#docker-compose" img="/img/docker.png"> +**Read time: 10 mins** + +Install with Docker Compose +</Card> + +<Card href="/portal/install#using-new-helm-chart" img="/img/k8s.png"> +**Read time: 10 mins** + +Install on Kubernetes +</Card> + +<Card href="/portal/install#linux-redhat-centos" img="/img/redhat-logo2.png"> +**Read time: 10 mins** + +Install on Red Hat +</Card> + + +</ResponsiveGrid> + +### Docker + +This section explains how to install Tyk Enterprise Developer Portal in a container using Docker. +Depending on your preferences, you can use MariaDB, MySQL, PostgreSQL or SQLite as database. + +In this recipe, the database and the portal container will run on the same network, with the database storing its data on a volume. The portal's CMS assets (images, files and themes) are stored in the database, although this guide provides links to the documentation to use a persistent volume or an S3 bucket as a storage medium for CMS assets. +Additionally, all settings for the Portal are configured using an env-file. + + +<Warning> +**Note** + +This document is just an example. Customize all fields, including the username, password, root password, database name and more. + +Be sure to update the connection DSN in the env-file accordingly. +</Warning> + + + +**Prerequisites** +To successfully install the Tyk Enterprise Developer Portal with Docker, you should have installed [Docker](https://docs.docker.com/get-docker/) on the machine where you want to install the portal. + +#### Using PostgreSQL + +1. **Create a network for the portal deployment** + + To start with, you need to create a Docker network for communication between the database and the portal. Execute the following command to create it: + ```console expandable + docker network create tyk-portal + ``` + +2. **Create an init script for PostgreSQL** + + To initialize a PostgreSQL database, you need to create an init script that will later be used to launch the PostgreSQL instance. + Copy the content below to a file named `init.sql`, which you will need in the next step. + ```sql + -- init.sql + -- Creating user + CREATE USER admin WITH ENCRYPTED PASSWORD 'secr3t'; + CREATE DATABASE portal; + GRANT ALL PRIVILEGES ON DATABASE portal TO admin; + ``` + +3. **Create the database volume and launch the database** + + The next step is to launch the PostgreSQL database for the portal. To achieve this, create a data volume for the database first: + ```console + docker volume create tyk-portal-postgres-data + ``` + + Then launch the PostgreSQL instance by executing the following command: + ```container + docker run \ + -d \ + --name tyk-portal-postgres \ + --restart on-failure:5 \ + -e POSTGRES_PASSWORD=secr3t \ + -e PGDATA=/var/lib/postgresql/data/pgdata \ + --mount type=volume,source=tyk-portal-postgres-data,target=/var/lib/postgresql/data/pgdata \ + --mount type=bind,src=$(pwd)/init.sql,dst=/docker-entrypoint-initdb.d/init.sql \ + --network tyk-portal \ + -p 5432:5432 \ + postgres:10-alpine + ``` + **Note** + + +<Warning> +The above PostgreSQL configuration is an example. You can customize deployment of your PostgreSQL instance. Please refer to [the PostgreSQL documentation](https://www.postgresql.org/docs/current/installation.html) for further guidance. +</Warning> + + +4. **Create an environment variables file** + + Creating an environment variables file to specify settings for the portal is the next step. + This is optional, as you can alternatively specify all the variables using the -e option when starting your deployment. + + Here is an example of a sample environment file. For a comprehensive reference of environment variables, please refer to the [configuration section](/product-stack/tyk-enterprise-developer-portal/deploy/configuration) in the Tyk Enterprise Developer Portal documentation. + ```ini + PORTAL_HOSTPORT=3001 + PORTAL_DATABASE_DIALECT=postgres + PORTAL_DATABASE_CONNECTIONSTRING=host=tyk-portal-postgres port=5432 dbname=portal user=admin password=secr3t sslmode=disable + PORTAL_DATABASE_ENABLELOGS=false + PORTAL_THEMING_THEME=default + PORTAL_STORAGE=db + PORTAL_LICENSEKEY=<your-license-here> + ``` + + Once you have completed this step, you are ready to launch the portal application with PostgreSQL in a Docker container. + +5. **Pull and launch the portal container** + + To pull and launch the portal using Docker, use the command provided below. + Ensure that you replace `<tag>` with the specific version of the portal you intend to launch before executing the command, e.g. `tykio/portal:v1.7` for the portal v1.7. You can browse all available versions on [Docker Hub](https://hub.docker.com/r/tykio/portal/tags) and in the [release notes section](/developer-support/release-notes/portal#170-release-notes). + ```console + docker run -d \ + -p 3001:3001 \ + --env-file .env \ + --network tyk-portal \ + --name tyk-portal \ + tykio/portal:<tag> + ``` + + This command will launch the portal on localhost at port 3001. Now, you can bootstrap the portal and start managing your API products. + +6. **Bootstrap the portal** + + Now the portal is running on port 3001, but it needs to be bootstrapped by providing credentials for the super admin user since it's the first time you are launching it. + Follow the [bootstrapping section](#bootstrapping-enterprise-developer-portal) of the documentation to bootstrap the portal via the UI or the admin API. + +7. **Clean up** + + If you want to clean up your environment or start the installation process from scratch, execute the following commands to stop and remove the portal container: + ```console + docker stop tyk-portal + docker rm tyk-portal + docker stop tyk-portal-postgres + docker rm tyk-portal-postgres + docker volume rm tyk-portal-postgres-data + ``` + +#### Using MySQL + +1. **Create a network for the portal deployment** + + To start with, you need to create a Docker network for communication between the database and the portal. Execute the following command to create it: + ```console + docker network create tyk-portal + ``` + +2. **Create the database volume and launch the database** + + The next step is to launch the MySQL database for the portal. To achieve this, create a data volume for the database first: + ```console + docker volume create tyk-portal-mysql-data + ``` + + Then launch the MySQL instance by executing the following command: + ```console + docker run \ + -d \ + --name tyk-portal-mysql \ + --restart on-failure:5 \ + -e MYSQL_ROOT_PASSWORD=sup3rsecr3t \ + -e MYSQL_DATABASE=portal \ + -e MYSQL_USER=admin \ + -e MYSQL_PASSWORD=secr3t \ + --mount type=volume,source=tyk-portal-mysql-data,target=/var/lib/mysql \ + --network tyk-portal \ + -p 3306:3306 \ + mysql:5.7 --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci --sql-mode=ALLOW_INVALID_DATES + ``` + +<Warning> +**Note** + +The above MySQL configuration is an example. You can customize deployment of your MySQL instance. + +Please refer to the [MySQL documentation](https://dev.mysql.com/doc/refman/5.7/en/charset-applications.html) for further guidance. +</Warning> + + + +<Note> +SQLite support will be deprecated from Tyk 5.7.0. To avoid disrupution, please transition to PostgreSQL, MongoDB or one of the listed compatible alternatives. +</Note> + + +3. **Create an environment variables file** + + Creating an environment variables file to specify settings for the portal is the next step. + This is optional, as you can alternatively specify all the variables using the -e option when starting your deployment. + + Here is an example of a sample environment file. For a comprehensive reference of environment variables, please refer to the [configuration](/product-stack/tyk-enterprise-developer-portal/deploy/configuration) section in the Tyk Enterprise Developer Portal documentation. + ```ini + MYSQL_ROOT_PASSWORD=sup3rsecr3t + MYSQL_DATABASE=portal + MYSQL_USER=admin + MYSQL_PASSWORD=secr3t + PORTAL_HOSTPORT=3001 + PORTAL_DATABASE_DIALECT=mysql + PORTAL_DATABASE_CONNECTIONSTRING=admin:secr3t@tcp(tyk-portal-mysql:3306)/portal?charset=utf8mb4&parseTime=true + PORTAL_DATABASE_ENABLELOGS=false + PORTAL_THEMING_THEME=default + PORTAL_STORAGE=db + PORTAL_LICENSEKEY=<your-license-here> + ``` + + Once you have completed this step, you are ready to launch the portal application with MySQL in a Docker container or via Docker Compose. + +4. **Pull and launch the portal container** + + To pull and launch the portal using Docker, use the command provided below. + Ensure that you replace `<tag>` with the specific version of the portal you intend to launch before executing the command, e.g. `tykio/portal:v1.7` for the portal v1.7. + You can browse all available versions on [Docker Hub](https://hub.docker.com/r/tykio/portal/tags) and in the [release notes](/developer-support/release-notes/portal#170-release-notes) section. + ```console + docker run -d \ + -p 3001:3001 \ + --env-file .env \ + --network tyk-portal \ + --name tyk-portal \ + --mount type=bind,src=/tmp/portal/themes,dst=/opt/portal/themes \ + --mount type=bind,src=/tmp/portal/system,dst=/opt/portal/public/system \ + tykio/portal:<tag> + ``` + + This command will launch the portal on localhost at port 3001. Now, you can bootstrap the portal and start managing your API products. + +5. **Bootstrap the portal** + + Now the portal is running on port 3001, but it needs to be bootstrapped by providing credentials for the super admin user since it's the first time you are launching it. Follow the [bootstrapping](#bootstrapping-enterprise-developer-portal) section of the documentation to bootstrap the portal via the UI or the admin API. + +6. **Clean up** + + If you want to clean up your environment or start the installation process from scratch, execute the following commands to stop and remove the portal container: + ```console + docker stop tyk-portal + docker rm tyk-portal + docker stop tyk-portal-mysql + docker rm tyk-portal-mysql + docker volume rm tyk-portal-mysql-data + ``` + +#### Using Sqlite + + +<Warning> +SQLite is useful for quick deployment and testing, however we don't recommend using it in production. +</Warning> + + + +<Note> +Tyk no longer supports SQLite as of Tyk 5.7.0. To avoid disruption, please transition to [PostgreSQL](/planning-for-production/database-settings#postgresql), [MongoDB](/planning-for-production/database-settings#mongodb), or one of the listed compatible alternatives. +</Note> + + +1. **Create a volume for the portal's database** + + To start with, you need to create a single volume for sqlite: + ```console + mkdir -p /tmp/portal/db + chmod -R o+x,o+w /tmp/portal + ``` + +2. **Create an environment variables file** + + Creating an environment variables file to specify settings for the portal is the next step. + This is optional, as you can alternatively specify all the variables using the -e option when starting your deployment. + + Here is an example of a sample environment file. For a comprehensive reference of environment variables, please refer to the [configuration](/product-stack/tyk-enterprise-developer-portal/deploy/configuration)] section in the Tyk Enterprise Developer Portal documentation. + ```ini + PORTAL_HOSTPORT=3001 + PORTAL_DATABASE_DIALECT=sqlite3 + PORTAL_DATABASE_CONNECTIONSTRING=db/portal.db + PORTAL_DATABASE_ENABLELOGS=false + PORTAL_THEMING_THEME=default + PORTAL_STORAGE=db + PORTAL_LICENSEKEY=<your-license-here> + ``` + + Once you have completed this step, you are ready to launch the portal application with SQLite in a Docker container. + +3. **Pull and launch the portal container** + + To pull and launch the portal using Docker, use the command provided below. + Ensure that you replace `<tag>` with the specific version of the portal you intend to launch before executing the command, e.g. `tykio/portal:v1.7` for the portal v1.7. + You can browse all available versions on [Docker Hub](https://hub.docker.com/r/tykio/portal/tags) and in the [release notes](/developer-support/release-notes/portal#170-release-notes) section. + ```console + docker run -d \ + -p 3001:3001 \ + --env-file .env \ + --mount type=bind,src=/tmp/portal/db,dst=/opt/portal/db \ + --mount type=bind,src=/tmp/portal/themes,dst=/opt/portal/themes \ + --mount type=bind,src=/tmp/portal/system,dst=/opt/portal/public/system \ + --name tyk-portal \ + tykio/portal:<tag> + ``` + + This command will launch the portal on localhost at port 3001. Now, you can bootstrap the portal and start managing your API products. + +4. **Bootstrap the portal** + + Now the portal is running on port 3001, but it needs to be bootstrapped by providing credentials for the super admin user since it's the first time you are launching it. Follow the [bootstrapping](#bootstrapping-enterprise-developer-portal) section of the documentation to bootstrap the portal via the UI or the admin API. + +5. **Clean Up** + + If you want to clean up your environment or start the installation process from scratch, execute the following commands to stop and remove the portal container: + ```console + docker stop tyk-portal + docker rm tyk-portal + ``` + + Since the SQLite data is persisted in the file system, you need to remove the following file for a complete deletion of the portal: + ```console + rm -rf /tmp/portal/db + ``` + +### Docker Compose + +This section provides a clear and concise, step-by-step recipe for launching the Tyk Enterprise Developer Portal in a container using Docker Compose. +Depending on your preferences, you can use MariaDB, MySQL, PostgreSQL or SQLite as database. + +In this recipe, the database and the portal containers will run on the same network, with the database storing it's data on a volume. The portal's CMS assets (images, files and themes) are stored in the database, although this guide provides links to the documentation to use a persistent volume or an S3 bucket as a storage medium for CMS assets. +Additionally, all settings for the Portal are configured using an env-file. + + +<Warning> +**Note** + +This document is just an example. Customize all fields, including the username, password, root password, database name and more. +</Warning> + + + +**Prerequisites** + +To successfully install the Tyk Enterprise Developer Portal with Docker Compose, you should have installed the following software: +- [Docker](https://docs.docker.com/get-docker/) +- [Docker Compose](https://docs.docker.com/compose/install/) + +#### Using PostgreSQL + +1. **Create an init script for PostgreSQL** + + To initialize a PostgreSQL database, you need to create an init script that will later be used to launch the PostgreSQL instance. + Copy the content below to a file named `init.sql`, which you will need in the next step. + ```sql + -- init.sql + -- Creating user + CREATE USER admin WITH ENCRYPTED PASSWORD 'secr3t'; + CREATE DATABASE portal; + GRANT ALL PRIVILEGES ON DATABASE portal TO admin; + ``` + +2. **Create an environment variables file for configuring the portal and the database** + + Creating an environment file to specify settings for the portal is the next step. + + Here is an example of a sample environment file. For a comprehensive reference of environment variables, please refer to the [configuration section](/product-stack/tyk-enterprise-developer-portal/deploy/configuration) in the Tyk Enterprise Developer Portal documentation. + ```ini + PORTAL_HOSTPORT=3001 + PORTAL_DATABASE_DIALECT=postgres + PORTAL_DATABASE_CONNECTIONSTRING=host=tyk-portal-postgres port=5432 dbname=portal user=admin password=secr3t sslmode=disable + PORTAL_DATABASE_ENABLELOGS=false + PORTAL_THEMING_THEME=default + PORTAL_LICENSEKEY=<your-license-here> + PORTAL_STORAGE=db + ``` + + Once you have completed this step, you are ready to launch the portal application with PostgreSQL via Docker Compose. + +3. **Create a docker-compose file** + + Before launching the portal using docker-compose, you will need to create a `docker-compose.yaml` file. An example of the portal's docker-compose file is provided below, which you can use as a starting point and further customize to meet your specific requirements. + + Ensure that you replace `<tag>` with the specific version of the portal you intend to launch before executing the command, e.g. `tykio/portal:v1.7` for the portal v1.7. You can browse all available versions on [Docker Hub](https://hub.docker.com/r/tykio/portal/tags) and in the [release notes section](/developer-support/release-notes/portal#170-release-notes). + + ```yaml + version: '3.6' + services: + tyk-portal: + depends_on: + - tyk-portal-postgres + image: tykio/portal:<tag> + networks: + - tyk-portal + ports: + - 3001:3001 + environment: + - PORTAL_DATABASE_DIALECT=${PORTAL_DATABASE_DIALECT} + - PORTAL_DATABASE_CONNECTIONSTRING=${PORTAL_DATABASE_CONNECTIONSTRING} + - PORTAL_THEMING_THEME=${PORTAL_THEMING_THEME} + - PORTAL_THEMING_PATH=${PORTAL_THEMING_PATH} + - PORTAL_LICENSEKEY=${PORTAL_LICENSEKEY} + - PORTAL_STORAGE=${PORTAL_STORAGE} + + tyk-portal-postgres: + image: postgres:10-alpine + volumes: + - tyk-portal-postgres-data:/var/lib/postgresql/data/pgdata + - ${PWD}/init.sql:/docker-entrypoint-initdb.d/init.sql + networks: + - tyk-portal + environment: + - POSTGRES_PASSWORD=secr3t + - PGDATA=/var/lib/postgresql/data/pgdata + + volumes: + tyk-portal-postgres-data: + + networks: + tyk-portal: + ``` + +4. **Pull and launch the portal container using docker-compose** + + To launch the portal using docker-compose, execute the command provided below. + ```console + docker-compose --env-file .env up -d + docker-compose --env-file .env up -d + ``` + + This command will launch the portal on localhost at port 3001. Now, you can bootstrap the portal and start managing your API products. + +5. **Bootstrap the portal** + + Now the portal is running on port 3001, but it needs to be bootstrapped by providing credentials for the super admin user since it's the first time you are launching it. Follow the [bootstrapping section](#bootstrapping-enterprise-developer-portal) of the documentation to bootstrap the portal via the UI or the admin API. + +6. **Clean up** + + If you want to clean up your environment or start the installation process from scratch, execute the following commands to stop and remove the portal container: + ```console + docker-compose down + docker-compose down + ``` + +#### Using MySQL + +1. **Create an environment variables file for configuring the portal and the database** + + The first step is to create an environment file to specify settings for the portal. + + Here is an example of a sample environment file. For a comprehensive reference of environment variables, please refer the [configuration section](/product-stack/tyk-enterprise-developer-portal/deploy/configuration) in the Tyk Enterprise Developer Portal documentation. + ```ini + MYSQL_ROOT_PASSWORD=sup3rsecr3t + MYSQL_DATABASE=portal + MYSQL_USER=admin + MYSQL_PASSWORD=secr3t + PORTAL_HOSTPORT=3001 + PORTAL_DATABASE_DIALECT=mysql + PORTAL_DATABASE_CONNECTIONSTRING=admin:secr3t@tcp(tyk-portal-mysql:3306)/portal?charset=utf8mb4&parseTime=true + PORTAL_DATABASE_ENABLELOGS=false + PORTAL_THEMING_THEME=default + PORTAL_STORAGE=db + PORTAL_LICENSEKEY=<your-license-here> + ``` + + Once you have completed this step, you are ready to launch the portal application with MySQL via Docker Compose. + +2. **Create a docker-compose file** + + Before launching the portal using docker-compose, you will need to create a `docker-compose.yaml` file. + An example of the portal's docker-compose file is provided below, which you can use as a starting point and further customize to meet your specific requirements. + + Ensure that you replace `<tag>` with the specific version of the portal you intend to launch before executing the command, e.g. `tykio/portal:v1.7` for the portal v1.7. + You can browse all available versions on [Docker Hub](https://hub.docker.com/r/tykio/portal/tags) and in the [release notes section](/developer-support/release-notes/portal#170-release-notes). + + ```yaml + version: '3.6' + services: + tyk-portal: + depends_on: + - tyk-portal-mysql + image: tykio/portal:<tag> + networks: + - tyk-portal + ports: + - 3001:3001 + environment: + - PORTAL_DATABASE_DIALECT=${PORTAL_DATABASE_DIALECT} + - PORTAL_DATABASE_CONNECTIONSTRING=${PORTAL_DATABASE_CONNECTIONSTRING} + - PORTAL_THEMING_THEME=${PORTAL_THEMING_THEME} + - PORTAL_THEMING_PATH=${PORTAL_THEMING_PATH} + - PORTAL_LICENSEKEY=${PORTAL_LICENSEKEY} + - PORTAL_STORAGE=${PORTAL_STORAGE} + + tyk-portal-mysql: + image: mysql:5.7 + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + volumes: + - tyk-portal-mysql-data:/var/lib/mysql + networks: + - tyk-portal + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MYSQL_DATABASE=${MYSQL_DATABASE} + - MYSQL_USER=${MYSQL_USER} + - MYSQL_PASSWORD=${MYSQL_PASSWORD} + + volumes: + tyk-portal-mysql-data: + + networks: + tyk-portal: + ``` + +3. **Pull and launch the portal container using docker-compose** + + To launch the portal using docker-compose, execute the command provided below. + ```console + docker-compose --env-file .env up -d + docker-compose --env-file .env up -d + ``` + + This command will launch the portal on localhost at port 3001. Now, you can bootstrap the portal and start managing your API products. + +4. **Bootstrap the portal** + + Now the portal is running on port 3001, but it needs to be bootstrapped by providing credentials for the super admin user since it's the first you are launching it. Follow the [bootstrapping section](#bootstrapping-enterprise-developer-portal) of the documentation to bootstrap the portal via the UI or the admin API. + +5. **Clean up** + + If you want to clean up your environment or start the installation process from scratch, execute the following commands to stop and remove the portal container: + ```console + docker-compose down + docker-compose down + ``` + +#### Using Sqlite + +1. **Create an environment variables file for configuring the portal and the database** + + Creating an environment file to specify settings for the portal is the next step. + This is optional, as you can alternatively specify all the variables using the -e option when starting your deployment. + + Here is an example of a sample environment file. For a comprehensive reference of environment variables, please refer to the [configuration section(/product-stack/tyk-enterprise-developer-portal/deploy/configuration)] in the Tyk Enterprise Developer Portal documentation. + ```ini + PORTAL_HOSTPORT=3001 + PORTAL_DATABASE_DIALECT=sqlite3 + PORTAL_DATABASE_CONNECTIONSTRING=db/portal.db + PORTAL_DATABASE_ENABLELOGS=false + PORTAL_THEMING_THEME=default + PORTAL_THEMING_PATH=./themes + PORTAL_LICENSEKEY=<your-license-here> + PORTAL_STORAGE=db + ``` + + Once you have completed this step, you are ready to launch the portal application with SQLite via Docker Compose. + +2. **Create a docker-compose file** + + Before launching the portal using docker-compose, you will need to create a `docker-compose.yaml` file. + An example of the portal's docker-compose file is provided below, which you can use as a starting point and further customize to meet your specific requirements. + ```yaml + version: '3.6' + services: + tyk-portal: + image: tykio/portal:<tag> + volumes: + - /tmp/portal/db:/opt/portal/db + ports: + - 3001:3001 + environment: + - PORTAL_DATABASE_DIALECT=${PORTAL_DATABASE_DIALECT} + - PORTAL_DATABASE_CONNECTIONSTRING=${PORTAL_DATABASE_CONNECTIONSTRING} + - PORTAL_THEMING_THEME=${PORTAL_THEMING_THEME} + - PORTAL_THEMING_PATH=${PORTAL_THEMING_PATH} + - PORTAL_LICENSEKEY=${PORTAL_LICENSEKEY} + - PORTAL_STORAGE=${PORTAL_STORAGE} + ``` + +3. **Pull and launch the portal container using docker-compose** + + To launch the portal using docker-compose, execute the command provided below. + ```console + docker-compose --env-file .env up -d + docker-compose --env-file .env up -d + ``` + + This command will launch the portal on localhost at port 3001. Now, you can bootstrap the portal and start managing your API products. + +4. **Bootstrap the portal** + + Now the portal is running on port 3001, but it needs to be bootstrapped by providing credentials for the super admin user since it's the first time you are launching it. Follow the [bootstrapping section](#bootstrapping-enterprise-developer-portal) of the documentation to bootstrap the portal via the UI or the admin API. + +5. **Clean up** + + If you want to clean up your environment or start the installation process from scratch, execute the following commands to stop and stop and remove the portal container: + ```console + docker-compose down + docker-compose down + ``` + + Since the SQLite data is persisted in the mounted volume (`/tmp/portal/db` in the above example), to completely erase the deployment, you will also need to delete it for a complete clean-up: + ```console + rm -rf /tmp/portal/db + rm -rf /tmp/portal/db + ``` + +### Kubernetes + +#### Using New Helm Chart + +There are two ways to install Tyk Enterprise Developer Portal. You can enable `global.components.devPortal` during Tyk Self-Managed deployment by following the [Tyk Self-Managed installation instruction](/product-stack/tyk-charts/tyk-stack-chart) using our `tyk-stack` umbrella chart. It will install Tyk Enterprise Developer Portal together with Tyk Gateway and Dashboard in the same namespace. + +Alternatively, you can install Tyk Enterprise Developer Portal as standalone component using our `tyk-dev-portal` chart. This page provides a clear and concise, step-by-step guide for installing the Tyk Enterprise Developer Portal as standalone component using the new helm chart. + +To install the portal using helm charts, you need to take the following steps: + +- Create the `tyk-dev-portal-conf` secret +- Specify config settings for the portal in `values.yaml` +- Launch the portal using the helm chart + +1. **Create the `tyk-dev-portal-conf` secret** + + Make sure the `tyk-dev-portal-conf` secret exists in your namespace. + This secret will automatically be generated if Tyk Dashboard instance was bootstrapped with [tyk-boostrap](https://artifacthub.io/packages/helm/tyk-helm/tyk-bootstrap) component chart + and `bootstrap.devPortal` was set to `true` in the `values.yaml`. + + If the secret does not exist, you can create it by running the following command. + + ```bash + kubectl create secret generic tyk-dev-portal-conf -n ${NAMESPACE} \ + --from-literal=TYK_ORG=${TYK_ORG} \ + --from-literal=TYK_AUTH=${TYK_AUTH} + ``` + + The fields `TYK_ORG` and `TYK_AUTH` are the Tyk Dashboard _Organization ID_ and the Tyk Dashboard API _Access Credentials_ respectively. These can be obtained under your profile in the Tyk Dashboard. + +2. **Config settings** + + You must set the following values in the `values.yaml` or with `--set {field-name}={field-value}` using the helm upgrade command: + + | Field Name | Description | + | ---------- | ----------- | + | `global.adminUser.email` and `global.adminUser.password` | Set portal admin username and email for bootstrapping | + | `global.secrets.devPortal` | Enable portal bootstrapping by providing secret name | + | `license` | Tyk license key for your portal installation | + | `storage.type` | Portal storage type, e.g. *fs*, *s3* and *db* | + | `image.tag` | Enterprise Portal version. You can get the latest version image tag from [Docker Hub](https://hub.docker.com/r/tykio/portal/tags) | + | `database.dialect` | Portal database dialect, e.g. *mysql*, *postgres* and *sqlite3* | + | `database.connectionString`| Connection string to the Portal's database, e.g. for the *mysql* dialect: `admin:secr3t@tcp(tyk-portal-mysql:3306)/portal?charset=utf8mb4&parseTime=true` | + + In addition to `values.yaml`, you can also define the environment variables described in the [configuration section](/product-stack/tyk-enterprise-developer-portal/deploy/configuration) to further customize your portal deployment. These environment variables can also be listed as a name value list under the `extraEnvs` section of the helm chart. + +3. **Launch the portal using the helm chart** + + Run the following command to update your infrastructure and install the developer portal: + + ```bash + helm install tyk-dev-portal tyk-helm/tyk-dev-portal -f values.yaml -n tyk + ``` + +###### Configuration +Please refer to this [guide](/product-stack/tyk-charts/tyk-stack-chart) for an explanation of all configuration options. + +> **Note**: Helm chart supports Enterprise Portal v1.2.0+. + +#### Using Legacy Helm Chart + + +<Warning> +**Note** + +It is recommended to use new helm charts instead of legacy charts. Guide for new charts can be found [here](/portal/install#using-new-helm-chart) +</Warning> + + +To install the portal using helm charts, you need to take the following steps: + +- Create the `tyk-enterprise-portal-conf` secret +- Specify config settings for the portal in `values.yaml` +- Launch the portal using the helm chart + +This guide provides a clear and concise, step-by-step recipe for installing the Tyk Enterprise Developer Portal using helm charts. + +1. **Create the `tyk-enterprise-portal-conf` secret** + + Make sure the `tyk-enterprise-portal-conf` secret exists in your namespace. This secret will automatically be generated during the Tyk Dashboard bootstrap if the `dash.enterprisePortalSecret` value is set to `true` in the `values.yaml`. + + If the secret does not exist, you can create it by running the following command. + + ```bash + kubectl create secret generic tyk-enterprise-portal-conf -n ${NAMESPACE} \ + --from-literal=TYK_ORG=${TYK_ORG} \ + --from-literal=TYK_AUTH=${TYK_AUTH} + ``` + + Where `TYK_ORG` and `TYK_AUTH` are the Tyk Dashboard Organization ID and the Tyk Dashboard API Access Credentials respectively. Which can be obtained under your profile in the Tyk Dashboard. + +2. **Config settings** + + + + <Note> + Tyk no longer supports SQLite as of Tyk 5.7.0. To avoid disruption, please transition to [PostgreSQL](/planning-for-production/database-settings#postgresql), [MongoDB](/planning-for-production/database-settings#mongodb), or one of the listed compatible alternatives. + </Note> + + + You must set the following values in the `values.yaml` or with `--set {field-name}={field-value}` with the helm upgrade command: + + | Field Name | Description | + | ---------- | ----------- | + | `enterprisePortal.enabled` | Enable Portal installation | + | `enterprisePortal.bootstrap` | Enable Portal bootstrapping | + | `enterprisePortal.license`| Tyk license key for your portal installation | + | `enterprisePortal.storage.type`| Portal database dialect, e.g *mysql*, *postgres* or *sqlite3* | + | `enterprisePortal.storage.connectionString` | Connection string to the Portal's database, e.g for the mysql dialect: `admin:secr3t@tcp(tyk-portal-mysql:3306)/portal?charset=utf8mb4&parseTime=true` | + + In addition to values.yaml, you can also define the environment variables described in the [configuration section](/product-stack/tyk-enterprise-developer-portal/deploy/configuration) to further customize your portal deployment. These environment variables can also be listed as a name value list under the `extraEnvs` section of the helm chart. + +3. **Launch the portal using the helm chart** + + Run the following command to update your infrastructure and install the developer portal: + + ```bash + helm upgrade tyk-pro tyk-helm/tyk-pro -f values.yaml -n tyk + ``` + + + + <Note> + In case this is the first time you are launching the portal, it will be necessary to bootstrap it before you can use it. For detailed instructions, please refer to the [bootstrapping documentation](#bootstrapping-enterprise-developer-portal). + </Note> + + +> **Note**: Helm chart supports Enterprise Portal v1.2.0+. + + +<h3 id="linux-redhat-centos">Red Hat (RHEL / CentOS)</h3> +This guide provides a step-by-step recipe for launching the Tyk Enterprise Developer Portal using an RPM package in Red Hat environment (RHEL / CentOS). + + +<Warning> +**Note** + +This document is just an example. Customize all fields, including the username, password, root password, database name and more. + +Be sure to update the connection DSN in the env-file accordingly. +</Warning> + + +**Prerequisites** + +To successfully install the Tyk Enterprise Developer Portal using RPM, your environment should satisfy the following requirements: +- Connectivity to [packagecloud.io](https://packagecloud.io). If your environment doesn't have connectivity to packagecloud, you will need to download the portal package and copy it to the target host. +- RPM Package Manager should be installed on the host machine. + +**Steps for Installation** + +1. **Download the portal package** + + To start with, you need to download the portal package from [packagecloud.io](https://packagecloud.io). To keep things organized, first create a directory where all installation assets (packages and config files) will be stored: + ```console + mkdir ~/portal-install + cd ~/portal-install + ``` + + Next, download the portal package from [packagecloud.io](https://packagecloud.io/tyk/portal-unstable) by executing the command below. + Ensure to replace package-version with actual package version e.g. https://packagecloud.io/tyk/portal-unstable/packages/el/8/portal-1.7.0-1.x86_64.rpm/download.rpm?distro_version_id=205 for the portal v1.7.0 for x86_64. + ```console + wget --content-disposition "https://packagecloud.io/tyk/portal-unstable/packages/<package-version>" + ``` + +2. **Install the portal package** + + Once the package is downloaded, you need to install using RPM. Execute the below command to so. Once again, ensure to replace `portal-1.7.0-1.x86_64.rpm` with an actual filename of the package you have downloaded on the previous step. + ```console + sudo rpm -i portal-1.7.0-1.x86_64.rpm + ``` + +3. **Update the configuration file with your license** + + + + <Note> + Tyk no longer supports SQLite as of Tyk 5.7.0. To avoid disruption, please transition to [PostgreSQL](/planning-for-production/database-settings#postgresql), [MongoDB](/planning-for-production/database-settings#mongodb), or one of the listed compatible alternatives. + </Note> + + + Before starting the portal service, you need to configure the portal. Once the rpm package has been installed, the portal configuration file will be located in `/opt/portal/portal.conf`. + Initially, the config file is filled with the default values. The minimal configuration change to start the portal is to add the `LicenseKey` property to the config file. + The below sample configuration will start the portal on portal 3001 with SQLite as a database, no TLS enabled, and all CMS assets (images, theme files, etc.) are stored in the filesystem. + You can, however, customize the provided example and make more suitable for your need using the [configuration](/product-stack/tyk-enterprise-developer-portal/deploy/configuration) reference. + ```json + { + "HostPort": 3001, + "LicenseKey": "<your-license-here>", + "Database": { + "Dialect": "sqlite3", + "ConnectionString": "portal.db", + "EnableLogs": false + }, + "Blog": { + "Enable": true + }, + "Site": { + "Enable": true + }, + "Forms": { + "Enable": false + }, + "StoreSessionName": "portal-store-session-name", + "PortalAPISecret": "123456", + "Storage": "fs", + "S3": { + "AccessKey": "your-access-key-here", + "SecretKey": "your-secret-key-here", + "Region": "s3-region", + "Endpoint": "if-any", + "Bucket": "your-bucket-here", + "ACL": "", + "PresignURLs": true + }, + "TLSConfig": { + "Enable": false, + "InsecureSkipVerify": false, + "Certificates":[ + { + "Name": "localhost", + "CertFile": "portal.crt", + "KeyFile": "portal.key" + } + ] + } + } + ``` + +4. **Start the portal service** + + Now when the portal package is installed and the configuration is updated, it is time to start the portal by executing the following command: + ```console + sudo systemctl start portal.service + ``` + + To check status and log of the portal execute the following command: + ```console + systemctl status portal.service + ``` + +5. **Bootstrap the portal** + + Now the portal is running on port 3001, but it needs to be bootstrapped by providing credentials for the super admin user since it's the first you are launching it. Follow the [bootstrapping](#bootstrapping-enterprise-developer-portal) section of the documentation to bootstrap the portal via the UI or the admin API. + +## Bootstrapping Enterprise Developer Portal + +When launching the Tyk Enterprise Developer portal for the first time, it starts in a special bootstrap mode, which is required to create the first admin user who will act as the super admin. +After launching the portal, you can bootstrap it using either the portal UI or the bootstrap API. + +This section explains how to bootstrap the portal using both the portal UI and the bootstrap API. + +### Bootstrapping the Portal via the UI +After launching the portal for the first time, you can use its UI to bootstrap it. The portal will display a form that allows you to create a super admin user and set their password. + +Navigate to the portal UI in your browser to start bootstrapping the portal. You should see the following: +<img src="/img/dashboard/portal-management/enterprise-portal/portal-bootstrap-ui-empty.png" alt="Portal bootstrap UI" /> + +Enter the admin email, password, first name, and last name. Then click on the `Register to Developer portal` button to complete the bootstrapping process. + +The bootstrap process should take no longer than a couple of seconds, so almost instantly the portal will display the following page, which confirms the successful bootstrap. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-bootstrap-successful.png" alt="Successful bootstrapping" /> + +Click on the `Login` button to proceed to the login page, where you can use the newly created super admin credentials to log in to the portal. + +### Bootstrapping the Portal via the API +The second approach to bootstrap the portal is through the bootstrap API, which allows you to programmatically bootstrap the portal. + +To bootstrap the portal via an API call, call the bootstrap API: +```shell +curl --location 'http://<your-portal-host>:<your-portal-port>/portal-api/bootstrap' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "username":"super-admin@tyk.io", + "password": "tyk123", + "first_name":"John", + "last_name":"Doe" +}' +``` expandable + +The bootstrap API accepts the following parameters: +- **username** - email of the super admin, it is also used as their login +- **password** - the super admin login password +- **first_name** - first name of the super admin +- **last_name** - first name of the super admin + +The bootstrap process should take no longer than a couple of seconds. You will receive the following response as a confirmation of the successful bootstrapping: +```json +{ + "code": "OK", + "message": "Bootstrapped user successfully", + "data": { + "api_token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJQcm92aWRlciI6Im5vbmUiLCJVc2VySUQiOiIkMmEkMTAkREF0czZhZTY0ZEZXSkFTbnR2OS8yLmMxcS91VTFhbTRGYk53RVJhTE1Ed2c0NHFsSXJnMkMifQ.ExTNl6UvjQA6WqrPE-7OkSNCBBixc2NGMnh3dnlk5Nw" + } +} +``` + + +<Note> +**Take a note of the api_token field** + +You will need this to call other Portal APIs. +</Note> + + +### Login as the super admin +After you have bootstrapped the portal, either via the UI or the bootstrap API, you can use the super admin's login credentials to log in to the portal. +Open the portal UI in your browser and click on the 'Login' button to open the login page. +<img src="/img/dashboard/portal-management/enterprise-portal/navigate-to-the-login-page.png" alt="Open the login page" /> +<br/> + +On the login page, enter the super admin credentials for logging into the portal: +<img src="/img/dashboard/portal-management/enterprise-portal/login-page-after-bootstrapping.png" alt="Open the login page" /> + +<br/> + + +<Note> +**Congratulations!** + + +Now you have a fully functional portal. +</Note> + + +<br/> + +You can continue configuring and customizing it either via the UI or the portal admin API. Please refer to [the Tyk Enterprise Developer Portal Concepts section](/portal/overview/concepts) for further guidance. + +## Environment Variable Reference + +For global configurations of the Developer Portal deployment refer this [config file](/product-stack/tyk-enterprise-developer-portal/deploy/configuration). + +## API Documentation + +The Dashboard exposes two APIs: + +1. [Enterprise Developer Portal API](/product-stack/tyk-enterprise-developer-portal/api-documentation/tyk-edp-api) +2. [Enterprise Developer Portal Admin API](/product-stack/tyk-enterprise-developer-portal/api-documentation/list-of-endpoints/portal-1.13.0-list-of-endpoints) diff --git a/portal/overview/concepts.mdx b/portal/overview/concepts.mdx new file mode 100644 index 00000000..05d87233 --- /dev/null +++ b/portal/overview/concepts.mdx @@ -0,0 +1,72 @@ +--- +title: "Core Concepts of Developer Portal" +'og:description': "Understand the fundamental concepts behind the Tyk Developer Portal, including APIs, access control, and customisation." +tags: ['Developer Portal', 'Tyk'] +sidebarTitle: "Core Concepts" +--- + +This section provides an overview of the most common concepts for the Developer Portal. When starting out with the Developer Portal, we recommend reading this through, ensuring you have a basic understanding of what these terms refer to. + +<iframe width="560" height="315" src="https://www.youtube.com/embed/0xlJDXKrbSw" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + +### API Products + +An API product is usually a grouping of API resources which have a value proposition to the API consumer. For example a 'Weather API' might bundle current weather with historical and forecasted weather APIs. + +### API Plans + +A plan is a way for API providers to create multiple quotas and/or rate limiting options for API consumers. It could be based on price, or commercial agreements with the API consumer. + +### Provisioning Request + +When an external api-consumer requests access to subscribe to a plan, a provisioning request will be sent to the portal admin, to either approve it or reject it. + +### Apps + +An app can be viewed as a simple wrapper for one or more sets of access credentials issued to an API-consumer. Multiple requests can be added into one single app. Users can manage those credentials (e.g. rotate keys) from this app section. + +### Access Credentials + +This is the unified naming for any API Keys, Tokens or Secrets provisioned for a specific app. + +### API Consumers + +API consumers are all external portal users/developers that are consuming and requesting access to APIs. + +This section includes: +- **Users**: Individual external API consumers +- **Teams**: Groups of API consumers +- **Organizations**: Grouping teams of API consumers + +Here is a potential set-up + +<img src="/img/dashboard/portal-management/enterprise-portal/portal-org-team-user-example.png" alt="A sample org set-up" /> + +### Catalogs + +Catalogs enable the publishing API products and plans based on visibility and access requirements. The catalog can be set to public or private. As an admin you can customize the audience of a private catalog at a team level, allowing you to create completely custom catalogs if needed. + +Here’s an example of how you could set up catalogs for the users above: + +<img src="/img/dashboard/portal-management/enterprise-portal/portal-catalogue-sample-set-up.png" alt="A sample catalogue set-up" /> + +### Admin users + +The internal users of the admin app. + +### Developer Portal + +When referring to the Developer Portal, we’re referring to the portal website in which external developers (what we refer to as API-consumers) can browse, view and request access to APIs. + +#### Using a policy in Tyk Self-Managed to create your API product + +To create an API Product you need to create a policy which enforces only Access rights. + +<img src="/img/dashboard/portal-management/enterprise-portal/portal-import-policy-as-product.png" alt="Using policy to create an API Product" /> + + +#### Using a policy in Tyk Self-Managed to create your plan + +To create an API Product you need to create a policy which enforces only quota and rate limit. + +<img src="/img/dashboard/portal-management/enterprise-portal/portal-import-policy-as-product.png" alt="Using policy to create a plan" /> diff --git a/portal/overview/getting-started.mdx b/portal/overview/getting-started.mdx new file mode 100644 index 00000000..4afcb914 --- /dev/null +++ b/portal/overview/getting-started.mdx @@ -0,0 +1,377 @@ +--- +title: "Getting Started with Developer Portal" +'og:description': "Get started quickly with setting up and using the Tyk Developer Portal." +tags: ['Developer Portal', 'Tyk'] +sidebarTitle: "Getting Started" +--- + +To get started with the Enterprise Developer portal, you need to accomplish four steps: + +1. [Connect to a provider (Tyk Self Managed)](/portal/overview/getting-started#connect-to-a-provider) +2. [Create and import API Products and Plans in your Tyk Self Managed Instance](/portal/overview/getting-started#create-api-products-and-plans) +3. [Create catalogs and developer audiences](/portal/overview/getting-started#organization-and-catalog) +4. [Publish API Products and Plans to the Developer portal so that API consumers can access them](/portal/overview/getting-started#optional-customize-visual-appearance-of-api-products-and-plans) + +After the last step, you can start serving APIs to your developer audience. + +## Install Developer Portal + +Refer the [install guide](/portal/install#installation-options-for-enterprise-developer-portal). + +## Connect to a Provider + +The first step in getting started with the developer portal is to connect the portal to a provider. Currently, the Tyk Enterprise Developer Portal supports only the Tyk Dashboard as an API Provider, with the ability to connect multiple instances of the Tyk Dashboard to the portal. +When the connection is established, the portal will import policies as API Products to the portal. The [Getting started guide](/portal/overview/getting-started#connect-to-a-provider) explains how to set up a policy and import it to the portal. + +<iframe width="560" height="315" src="https://www.youtube.com/embed/8KJSVACD-j4" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + +**Prerequisites** + +- A Tyk Self-Managed [installation](/tyk-self-managed/install) +- The Enterprise portal installed. +- A login for the portal admin app. + +**Steps for Configuration** + +1. Go to the provider section in the portal admin dashboard +2. Click **Add provider** +3. Add your provider details + +| Field | Description | +| :-------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Name | This is an internal reference to the provider. | +| Provider type (disabled) | This refers to the type of provider; however, the only supported provider at this stage is Tyk Self-Managed. | +| URL | The URL refers to the provider host URL for your Tyk Self-Managed installation. Within the Tyk instance, the URL can be simply copied. | +| Gateway URL | The gateway URL refers to the URL that the portal developers will use for requesting queries and accessing credentials. +| Secret | The Secret can be fetched from the Tyk Self-Managed / Tyk analytics dashboard. The procedure is as follows: Go to the Tyk Dashboard. Navigate to *Users*. Select a user with the permissions you want to bring on to the portal. You can find the secret under *API Access Credentials. (Optional)*. You can find the organization id listed under *Organization ID* if your use case requires this. Please note that the Portal will share the same permissions that the user selected to provide the secret. +| Organization ID | The org id is required in order to connect to your installation as a provider. It can be found in the user profile within the Tyk Dashboard. | +| Policy tags | This is an optional field that can be used to define which policies from Tyk will be imported to the portal. If a tag is defined here, it needs to also be defined in the Policy section in the [Tyk Dashboard](/portal/overview/getting-started#create-and-import-an-api-product-from-tyk). If this field is left empty in both this provider section and in the policies within Tyk, then all policies will be imported from the Tyk instance. How to include the label in the policy section inside Tyk, is explained in [Publish API Products and plans](/portal/overview/getting-started#publish-an-api-product) for the public-facing portal. | + +4. Save your changes + +##### How to find the Secret and Org ID inside your Tyk Dashboard? + +1. Select **Users** from the **System Management** section. +2. In the users list, click **Edit** for your user. +3. The Secret is the **Tyk Dashboard API Access Credentials**. The **Organization ID** is underneath **Reset key**. <img src="/img/2.10/user_api_id.png" alt="API key location" /> +4. Select **APIs** from the **System Management** section. +5. From the **Actions** menu for your API, select **Copy API ID**. + +## Create API Products and Plans + +There are two ways of creating API Products and Plans in the Developer Portal: + +1. [Automatically create an API Product or Plan](#import-api-product-and-plan) in the Developer Portal by importing it from Tyk when synchronising the provider. +2. [Manually create an API Product or Plan](#manually-create-api-product-and-plan) in the Developer Portal. (Only from version 1.13.0) + +**Prerequisites** + +- A Tyk Self-Managed [installation](/tyk-self-managed/install) +- Tyk Self-Managed [added as a provider](/portal/overview/getting-started#connect-to-a-provider) +- Have APIs [created in your Tyk installation](/api-management/gateway-config-managing-classic#create-an-api). + +### Import API Product and Plan + +When integrating with Tyk, the Tyk policies will be imported into the Developer Portal. Depending on the configuration that’s been set in the policy section, the policy will either be imported as an API Product or a Plan. For further details check the [portal key concepts](/portal/overview/concepts) document. + +#### Create and Import an API Product from Tyk + +<iframe width="560" height="315" src="https://www.youtube.com/embed/rIGnIQ235As" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + +API Products are partitioned policies that provide an ACL but not quota/rate limits. +The following steps explain how to create and import an API product from Tyk, assuming you have one or more APIs already created: + +1. From your Tyk Self-Managed installation, go to **Policies** and click **Add policy**. +2. Select which APIs you want to add to your API product. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-add-policy.png" alt="Create a policy" /> + +3. From the **Access Rights** drop-down list, select one or more APIs to include in your policy. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-select-api-to-include-into-policy.png" alt="Add an API into the policy" /> + +4. Under **Global limits and Quota**, select **Enforce access rights**. Ensure **Enforce usage quota** and **Enforce rate limit** are **not** selected. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-enforce-access-rights.png" alt="Enforce access rights" /> + +5. From the **Configurations** tab, add the information needed under name and settings. +6. From the **Tags** tab, a tag can be added to tell the portal this should be imported. If you have specified a specific label in the Provider section within the Developer portal when adding Tyk, the way the portal would know which Policies to import can be specified here. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-add-tags.png" alt="Add tags to the policy" /> + +7. To import the API Products into the Developer portal, from the Tyk Portal admin app, click **Synchronise**. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-sync-with-dashboard.png" alt="Sync with the Tyk Pro" /> + +#### Create and Import Plans from Tyk + +<iframe width="560" height="315" src="https://www.youtube.com/embed/XYlaqy3UuNg" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + +Plans are policies that implement rate limit or quota, or both, but do **NOT** include the ACL. +To create a Plan for the developer portal, follow the same steps as for creating an API Product. However, within the Global limits and quota in the Policies, configure the policy as follows: + +1. From your Tyk Self-Managed installation, go to **Policies** and click **Add policy**. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-add-policy.png" alt="Create a policy" /> + +2. Select an API. Please note that this a required field. The purpose of the policy explained in this guide is to control allowance, so you can select any API here. +3. Under **Global limits and Quota**, select **Enforce usage quota** and **Enforce rate limit**. Ensure **Disable rate-limiting** and **Unlimited requests** are **not** selected so you can set these limits. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-enforce-quota.png" alt="Enforce quota and rate limit" /> + +4. Click **Synchronise** to import the plans into the Developer portal, from the Tyk Portal admin app. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-sync-with-dashboard.png" alt="Sync with the Tyk Pro" /> + +### Manually Create API Product and Plan + +From version 1.13.0, the Developer Portal allows you to create API Products and Plans from the portal dashboard for admins. When manually creating an API Product or Plan, the corresponding policy will be created in the Tyk Self-Managed selected provider. + +#### Create API Product + +When creating an API Product in the Developer Portal, a partitioned policy that provides an ACL but not quota/rate limits will be created in the Tyk Self-Managed selected provider. The following steps explain how to create an API Product in the Developer Portal: + +1. From the Tyk Portal admin app, go to **API Products** and click **Add new API Product**. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-add-api-product.png" alt="Add an API Product" /> + +2. Select a unique name for the API Product and complete the [product details for customization](/portal/overview/getting-started#optional-customize-visual-appearance-of-api-products-and-plans) in the **Details** tab. The product name will be the name assigned to the created policy in the Tyk Self-Managed selected provider +<img src="/img/dashboard/portal-management/enterprise-portal/portal-product-details.png" alt="Add an API Product" /> + +3. Select **Provider**, **Authentication**, and **APIS** in the **API's** tab. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-product-apis.png" alt="Add APIs" /> + +4. Add API specifications in the **Documentation** tab. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-product-api-specs.png" alt="Add API Specifications" /> + +5. Add Product Guides in the **"Getting Started" guides** tab. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-product-guides.png" alt="Add Product Guides" /> + +6. Complete DCR settings in the **Dynamic Client Registration** tab (Only for JWT selected APIs). +<img src="/img/dashboard/portal-management/enterprise-portal/portal-product-dcr.png" alt="Add DCR settings" /> + +7. Save changes. + + + + <Note> + **Note:** + + If no APIs are selected, you can still add api specifications, and guides creating a documentation only product. Documentation only products are Developer Portal products that do not have any APIs associated with them and thus no policies will be created in the Tyk Self-Managed selected provider. Documentation only products are useful for creating documentation for APIs that are not yet created or published. Specs and guides will be shown in the external portal as a regular product and the selected `Specification Alias` will be used as the reference for each spec. + <img src="/img/dashboard/portal-management/enterprise-portal/portal-doc-only-product.png" alt="Add a Documentation only Product" /> + </Note> + + +#### Create Plan + +When creating a Plan in the Developer Portal, a partitioned policy that implements rate limit or quota, or both, but do **NOT** include the ACL will be created in the Tyk Self-Managed selected provider. The following steps explain how to create a Plan in the Developer Portal: + +1. From the Tyk Portal admin app, go to **Plans** and click **Add new Plan**. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-add-plan.png" alt="Add a Plan" /> + +2. Choose a **Provider** and a unique **Name** for the plan. The plan name will be the name assigned to the created policy in the Tyk Self-Managed selected provider. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-plan-details.png" alt="Add Plan Details" /> + +3. Complete Plan limits. Select **Usage quota**, **Rate limit**, and **Key expiration** for the plan. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-plan-limits.png" alt="Add Plan Limits" /> + +4. Complete Advanced settings (optional). Set **Scopes** (for DCR), **Access Request Frequency**, and **Credential metadata** for the plan. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-plan-advanced-settings.png" alt="Add Plan Advanced Settings" /> + +5. Save changes. + + +## Publish API Products and Plans + +In this section, you will learn how to publish the API products and plans to the public-facing portal so that API Consumers can access them. + +<iframe width="560" height="315" src="https://www.youtube.com/embed/9CA1iY-VTjo" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + +**Prerequisites** + +- A Tyk Self-Managed [installation](/tyk-self-managed/install) +- Tyk Self-Managed [added as a provider](/portal/overview/getting-started#connect-to-a-provider) +- [Created and imported API Products and Plans from Tyk](/portal/overview/getting-started#create-api-products-and-plans) + +### Publish an API product + +Follow these steps below how to publish an API Product to a catalog: + +1. From the **API Product** section, Click an API product to open the details. + + <img src="/img/dashboard/portal-management/enterprise-portal/portal-publish-product.png" alt="Edit the API Product's metadata" /> + +2. Edit the metadata as needed. + +| Field | Description | +| :-------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Catalog display name | This is the name that will be displayed in the external catalog. | +| Featured API Product | Tick this option if you want the API Product to appear on the homepage under β€œFeatured products”. | +| Description | Short description about what this API Product is. | +| Content | This section appears on the API Product overview page, the rich text editor enables you to add more information about the API Product e.g. use cases, features, etc. | +| Image | An image can be added to the API Product. Supported formats are JPG and PNG. | +| Organization ID | The org id is required in order to connect to Tyk as a provider. It can be found in the user profile within the Tyk Dashboard. | +| Catalogs | Select an existing catalog that this product should be available from. | +| App registration configs | An experimental feature, it works only for oAuth2.0-enabled APIs | +| API resources | This section lists all APIs available within this product, you can add OAS documentation for each API. | + +3. Select a catalog to publish the API product. If you want to create a custom catalog. + + <img src="/img/dashboard/portal-management/enterprise-portal/portal-select-catalogue-for-api-product.png" alt="Edit the API Product's metadata" /> + +4. Navigate to **Catalogs** to view the available catalog. + + <img src="/img/dashboard/portal-management/enterprise-portal/portal-available-catalogues.png" alt="Edit the API Product's metadata" />s + +### Publish a Plan + +In order for developers to be able to request access to an API Product and retrieve credentials, a minimum of one plan needs to be available within the same catalog as the API Product. + +Follow these steps below to knowhow to publish a plan: + +1. From the **Plans** section, select a plan to open the details. + + <img src="/img/dashboard/portal-management/enterprise-portal/portal-select-a-plan.png" alt="Edit the plan's metadata" /> + + +2. Edit the metadata as needed + +| Field | Description | +| :-------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Catalog display name | This is the name that will be displayed in the external catalog. | +| Plan allowance | This section describes what quota and limit is set for this plan. These values can be updated within the β€˜policy section’ in the Tyk dashboard. | +| Catalogs | Select an existing catalog that this product should be available from. | +| Auto-approve provisioning request | Under plan settings, you can choose to select this option which means whenever an API-consumer requests access to an API product(s) using this plan, it will be approved automatically. | +| JWT Scope | An experimental feature, it works only for oAuth2.0 enabled APIs | + +3. Click **Save changes**. The plan will now be available within selected catalog(s). + +## Organization and Catalog + +In the Tyk Enterprise Developer Portal, Organizations and Catalogs are used to segment the user base and make different APIs available to different user audiences according to the business model. +For example, assume the following scenario: + +- Account Management API is available only to trusted partners +- Payment API is available to all developers + +Subsequently, two catalogs can be created for these two APIs. + +In the below example, an API Provider offers two API Products (the Accounts API and Payment API) with two plans (the Free plan and Enterprise plan) to their customers. +Customers subscribed to the enterprise plan can use both APIs, offering a higher user limit. Conversely, customers subscribed to the Free plan (individual developers or hobbyists) only have visibility of the Payment API. + +To achieve that, the API Provider uses two catalogs to implement their business model so that they can offer different experiences for different customer audiences. This section explains how to achieve that using the Tyk Enterprise Developer Portal. +<img src="/img/dashboard/portal-management/enterprise-portal/org-catalogue-product-relation.png" alt="Relationship between catalogs, API Products, plans, teams, and organizations" /> + +### Create Organization and Team + +The Tyk Enterprise Developer Portal uses Organization and Catalogs to segment access to APIs and plans. Therefore, the first thing is to create an organization for your customers. If you don't want to provision organizations manually, you can leverage the [Admin APIs](/product-stack/tyk-enterprise-developer-portal/api-documentation/tyk-edp-api) or enable the [self-service organization registration](/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/manage-api-consumer-organisations). +In this guide, we will create the **B2B customer** organization to fulfill the above business model: +1. To create an organization for the **B2B customer**, navigate to the **Organizations** menu and click on the **Add new organization** button. + <img src="/img/dashboard/portal-management/enterprise-portal/navigate-to-organisations.png" alt="Navigate to the Organizations menu" /> + +2. Enter the name of your new organization and click on the **Save** button. A new default-team will be automatically created and associated with your new organization. + <img src="/img/dashboard/portal-management/enterprise-portal/create-b2b-customer-org.png" alt="Add a new Organization" /> + + +<Note> +You can edit the default team name by navigating to **Teams** and opening up the team associated with the organization you created. This will allow you to edit the team name as required. +</Note> + + +### Create Catalogs + +1. To create catalogs, navigate to the catalogs menu in the Tyk Enterprise Developer Portal. + +The default catalogs that are featured when the portal is [bootstrapped](/portal/install#bootstrapping-enterprise-developer-portal) are: +- **Public** catalogs are available to all developers. +- **Private** catalogs are available only to logged in developers who have been assigned with access. + +You can create new catalogs by clicking on the **Add new catalog** button or use the default catalogs. + +<img src="/img/dashboard/portal-management/enterprise-portal/navigate-to-catalogues.png" alt="Navigate the to catalogues menu" /> + +2. To add a new catalog, click on the **Add new catalog** button. Then specify the name of the catalog and select its type: **Private** or **Public**. + Since the public catalog already exists, in this guide you need to create only an additional private catalog called **Enterprise catalog** for the **B2B customer** who will have extended access rights compared to other developers. + <img src="/img/dashboard/portal-management/enterprise-portal/specify-name-of-catalogue.png" alt="Create a catalogue" /> + + +<Note> +While it is possible to create multiple public catalogs, we do not advise doing so. This is because multiple public catalogs will share the same level of access. +</Note> + + +3. Once the catalog is created, add a developer audience to it by clicking on the **Add Team** button and selecting an appropriate developer team (**B2B customer All users** in this example). + Finally, add plans and API Products to the created catalog so that the selected developer teams can view them. + <img src="/img/dashboard/portal-management/enterprise-portal/add-team-products-and-plans.png" alt="Add teams, products, and plans to the catalogue" /> + +You can achieve the same result by navigating to the **API Products** menu. Adding an API Product to a catalog through the **Catalogs** and the **API Products** menus will produce the same result. +<img src="/img/dashboard/portal-management/enterprise-portal/publish-products-to-catalogues.png" alt="Adding a product to a catalogue through the API Products menu" /> + + +<Note> +**Congratulations** + +You have successfully added a catalog and associated a team with it. Furthermore, you have allocated plans and API products to this catalog. +</Note> + + +## (Optional) Customize Visual Appearance of API Products and Plans + +In this section, you will learn how to customize the visual appearance of API Products and plans with the Tyk Enterprise Developer Portal. That includes: +- The display name for API Products and plans. +- The description and logo of API Products that will be displayed on the API Product catalog page. +- Tags for API Products to match them with related blog posts. + +### Customize API Products + +To customize the visual appearance of API Products: +1. Navigate to the **API Products** menu, select the product that you want to customize (the Payment API in example below). In this menu you can customize the following aspects of API Products: + + - **Catalog display name**: This is the name that will be displayed in the external catalog. + - **Featured API Product**: Tick this option for the API Product to appear on the home page under the **Featured products** section. + - **Description/Description in the catalogue page**: A short description about what this API Product is. It is displayed in the catalog and on the API Product page. + - **Content/Description in the product details page**: A long text description that appears on the API Product overview page, the rich text editor enables you to add more information about the API Product e.g. use cases, features, etc. + - **Image**: An API Product logo that is displayed on the catalog and on the API Product pages. From version 1.13.0 you can also upload a preview image for the calalogue page. + - **Tags**: The tags are used to match the API Product with the related blog posts that have the same tags in common. + + From that page you can also manage [OAuth2.0 settings](/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/dynamic-client-registration) and add [Get started guides](/tyk-stack/tyk-developer-portal/enterprise-developer-portal/getting-started-with-enterprise-portal/manage-get-started-guides-for-api-products) to your API Products, which is covered in the relevant sections. + <img src="/img/dashboard/portal-management/enterprise-portal/customize-product.png" alt="Customize visual appearance of an API Product" /> + +2. Save changes by clicking on the **Save** button. You should now be able to preview how the API Product will be displayed in the catalog: + + <img src="/img/dashboard/portal-management/enterprise-portal/product-live-portal.png" alt="View the API Product in the catalogue" /> + + + + <Note> + From version 1.13.0 you can customize the api product in the `Details` tab section of a product. + <img src="/img/dashboard/portal-management/enterprise-portal/portal-product-customize.png" alt="Customize visual appearance of an API Product" /> + </Note> + + +### Customize Plans + +1. To customize visual appearance of plans, open the **Plans** menu and select the plan you want to customize. You can customize the following settings: + + - **Catalog display name**: The name of the catalog that will be displayed in the API Product page. + - **Scope**: Scope for the [OAuth2.0 settings](/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/dynamic-client-registration). Please refer to the [OAuth2.0 documentation](/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/dynamic-client-registration) for further guidance. + - **Catalogs**: The catalogs in which this plan is available. Catalogs and organization are [covered](/portal/overview/getting-started#organization-and-catalog) later within this getting started guide. + - **Auto-approve settings for access requests**: Determines if access requests for this plan should be approved automatically. + - **Access request frequency**: Defines how often developers can request access to a specific plan. This way the admins can prevent developers from creating too many keys and abusing their automatically approved plans. + + + + <Note> + Scope and Access request frequency settings are located under Plan's view Advanced settings from version 1.13.0. + <img src="/img/dashboard/portal-management/enterprise-portal/portal-plan-advanced-settings.png" alt="Add Plan Advanced Settings" /> + </Note> + + + + <img src="/img/dashboard/portal-management/enterprise-portal/customize-plan.png" alt="Customize visual appearance of a plan" /> + +2. Customize the plan's visual appearance as required and then click on the **Save** button. Now you can view the plan in the API Product page: + + <img src="/img/dashboard/portal-management/enterprise-portal/plan-live-portal.png" alt="Customize visual appearance of a plan" /> + + + + <Note> + **Congratulations** + + You have now customized the visual appearance of your API Product and plan. By following the above steps you can customize visual appearance of your other API Products and plans. + </Note> + + + diff --git a/portal/overview/intro.mdx b/portal/overview/intro.mdx new file mode 100644 index 00000000..2b425847 --- /dev/null +++ b/portal/overview/intro.mdx @@ -0,0 +1,56 @@ +--- +title: "Introduction to Tyk Developer Portal" +'og:description': "Learn what the Tyk Developer Portal is, its key features, and how it supports API management." +tags: ['Developer Portal', 'Tyk'] +sidebarTitle: "Introduction" +--- + + +<Note> +**Tyk Enterprise Developer Portal** + +If you are interested in getting access contact us at [support@tyk.io](<mailto:support@tyk.io?subject=Tyk Enterprise Portal Beta>) +</Note> + + + +## What is Enterprise Developer Portal? + +The Tyk Enterprise Developer Portal is the most flexible and straightforward way for API providers to publish, monetize and drive the adoption of APIs. It provides a full-fledged CMS-like system that enables you to serve all stages of API adoption: from customization the look and feel to exposing APIs and enabling third-party developers to register and use your APIs. + +The Tyk Enterprise Developer Portal enables you to: + +* Completely customize look and feel of the portal. +* Combine multiple APIs into API product and supply them with OpenAPI documentation and tutorials. +* Create multiple organizations and teams to segment your developer audience. +* Create multiple API catalogs to tailor visibility of API products and plans to different audiences. +* Integrate with the most popular Identity providers via Dynamic client registration. +* Fully control the flow of the developer sign-up and enrollment. + +<iframe width="560" height="315" src="https://www.youtube.com/embed/UMLkTKMzGXg" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + +## Where does it fit? + +Tyk Developer Portal enables multiple instances of Tyk Manager, also referred to as a `Provider` because we will soon include other API Managers and Gateways! Each provider provides a list of Policies, APIs and Keys. +In turn, when the API consumer makes a request and it is approved, the Portal issues a provisioning request to the relevant control plane to issue a key. + +<img src="/img/dashboard/portal-management/enterprise-portal/portal-diagram-api-providers.png" alt="Developer portal and Tyk Manager relationship" /> + +## Workflows for Portal Management + +These workflows are designed to help organizations streamline collaboration between developers and content managers in managing the Tyk Developer Portal. + +### Developer Workflow + +For organizations with developers customizing pages layout and other technical aspects of the portal pages, we are recommending the following workflow. + +<img src="/img/dashboard/portal-management/enterprise-portal/portal-fe-develop-flow.png" alt="Developer workflow" /> + +### Content Manager Workflow + +For organizations with content manager(s) managing the developer portal content, we are recommending the following workflow. + +<img src="/img/dashboard/portal-management/enterprise-portal/portal-content-manager-flow.png" alt="Content manager workflow" /> + +The Tyk Developer portal supports the workflow of content managers who're responsible for editing and managing page content. +The purpose of highlighting this flow is to give recommendations on how to organize effective workflows between front end engineers and content managers. Where front end engineers are building page templates and content managers are managing the pages and the content. \ No newline at end of file diff --git a/portal/resource-migration.mdx b/portal/resource-migration.mdx new file mode 100644 index 00000000..9acd5e92 --- /dev/null +++ b/portal/resource-migration.mdx @@ -0,0 +1,289 @@ +--- +title: "Migrate Resources Between Environments" +'og:description': "Learn how to migrate resources between environments in the Tyk Enterprise Developer Portal." +tags: ['Tyk Developer Portal', 'Enterprise Portal', 'Email', 'Notifications'] +sidebarTitle: "Resource Migration" +--- + +## Migrate Resources Between Environments + +This guide explains how to migrate Developer Portal resources (API Products, Plans, Tutorials etc.) between different portal environments. + +This capability was made possible with introduction of [Custom IDs](#custom-ids-in-developer-portal) (more on this later) in v1.13. + +## Prerequisites + +Before you begin, make sure the following are true: + +- Your Tyk Developer Portal version is 1.13 or later. +- All resources in your source environment have **Custom IDs** (CIDs) assigned. Resources created after version 1.13 automatically include a CID, while resources from earlier versions receive CIDs through the portal's startup process. +- You have admin access to both the source and target environments. + +## Custom IDs in Developer Portal + +Starting with Portal 1.13, we introduced **Custom IDs (CIDs)** - additional persistent identifiers that work alongside database IDs to provide stable references across environments and recreations. While database IDs remain the primary internal identifiers, CIDs provide a reliable way to track and maintain relationships between resources across different environments. + +### The Role of Database IDs and CIDs + +Resources in the Tyk Developer Portal use both types of identifiers: +- **Database IDs**: Primary internal identifiers that are automatically generated and managed by the database. +- **Custom IDs (CIDs)**: Additional stable identifiers that remain consistent across environments. + +### The Problem with Database-Generated IDs + +Before Portal 1.13, resources were identified solely by database-generated IDs. While this worked for single-environment setups, it caused challenges when: +- Migrating resources between environments. +- Recreating or restoring resources. +- Maintaining relationships between connected resources. + +For example, if you recreated an API product that was linked to a plan, the product would receive a new database ID. This would break the connection between the product and plan, requiring manual intervention to fix the relationship. + +### Benefits of Custom IDs (CIDs) + +Custom IDs solve these problems by providing: + +- Persistent identification across environments. +- Stable reference points for resource relationships. +- Reliable migration and synchronization capabilities. + +Resources that now support CIDs include: + +- OAuth Providers and Client Types +- Products, Plans, Tutorials, and OAS Documents +- Organisations and Teams +- Pages and Content Blocks + +These resources are now easily transferable between environments, with their relationships preserved via CIDs, ensuring smooth migrations and consistent management. + +### Automatic CID Assignment + +When upgrading to Tyk Portal 1.13 from an earlier version, the portal automatically runs a **background process** to assign CIDs to resources created in previous versions. This process also runs every time the portal starts, ensuring any new resources without CIDs are retroactively assigned one, whether after an upgrade or a fresh installation. + +You can fetch a specific organisation using either its database ID or CID. For example, to fetch the "foo" organisation: + +**Using database ID:** +```bash +curl -X GET 'http://localhost:3001/portal-api/organisations/27' \ + -H "Authorization: ${TOKEN}" \ + -H 'Content-Type: application/json' \ + -H 'Accept: application/json' +``` + +**Using CID (recommended):** +```bash +curl -X GET 'http://localhost:3001/portal-api/organisations/2sG5umt8rGHMiwjcgaHXxwExt8O' \ + -H "Authorization: ${TOKEN}" \ + -H 'Content-Type: application/json' \ + -H 'Accept: application/json' +``` + +While both methods work, using CIDs is recommended as they remain consistent across environments. + + +## Step-by-Step Instructions + +In this guide, we'll walk through the process of migrating selected organisations and their teams from one environment (Environment A) to another (Environment B). This involves exporting data from the source environment and importing it into the target environment. + +<br /> + +<Note> +This guide only migrates the `Organization` and `Teams` resources from the developer portal; the same process must be repeated for other resources. +</Note> + + +### Example Scenario +- **Source**: Environment A at `https://portal-env-a.example.com` +- **Target**: Environment B at `https://portal-env-b.example.com` +- **Goal**: Migrate organisations and their associated teams + +### Setting Up Environment Variables + +Before running the migration scripts, you'll need to set up authentication tokens for both environments. You can find these tokens in the Developer Portal UI: + +1. Log in to the Developer Portal as an admin +2. Click on your user profile in the top right corner +3. Copy **API credential** + +```bash +# For Environment A (source) +export ENV_A_TOKEN="your-source-environment-token" + +# For Environment B (target) +export ENV_B_TOKEN="your-target-environment-token" +``` + +### Export Organisations from Environment A + +To start, you'll want to gather the relevant data from Environment A. This ensures you have everything you need for a smooth migration. The data is saved into a JSON file, making it easy to handle during the import process. + +Here's an example of how you can export organisations from Environment A: + +```bash expandable +# Fetch organisations from Environment A +response=$(curl -s -H "Authorization: ${ENV_A_TOKEN}" \ + -H "Content-Type: application/json" \ + -H "Accept: application/json" \ + "https://portal-env-a.example.com/organisations?page=1&per_page=50") + +# Process each organisation +echo "$response" | jq -c '.[] | select(.Name != "Default Organisation") | del(.ID, .CreatedAt, .UpdatedAt, .Teams)' > data/organisations.json +``` + +### Export Teams from Environment A + +After exporting organisations, the next step is to export the teams associated with each organisation. We exclude default teams since these are created automatically by the portal, and dealing with them could lead to conflicts. The data is saved into JSON files for structured storage and easy access during the import process. + +Here's an example of how you can export teams from Environment A: + +```bash expandable +# Read each organisation and fetch its teams +while IFS= read -r org; do + org_cid=$(echo "$org" | jq -r '.CID') + echo "Fetching teams for organisation CID: $org_cid..." + + # Fetch teams for the organisation + teams_response=$(curl -s -H "Authorization: ${ENV_A_TOKEN}" \ + -H "Content-Type: application/json" \ + -H "Accept: application/json" \ + "https://portal-env-a.example.com/organisations/$org_cid/teams?page=1&per_page=50") + + # Process each team + echo "$teams_response" | jq -c '.[] | select(.Name | endswith("All users") | not) | del(.Users)' > "data/teams_${org_cid}.json" +done < data/organisations.json +``` + +### Import Organisations to Environment B + +Now, let's move those organisations into Environment B, one by one. The goal here is to recreate the organisational structure in Environment B accurately. By using the JSON files, you ensure that each organisation is imported correctly, keeping the relationships intact from Environment A. + +Here's an example of how you can import organisations into Environment B: + +```bash expandable +# Read each organisation and import it +while IFS= read -r org; do + org_cid=$(echo "$org" | jq -r '.CID') + echo "Importing organisation CID: $org_cid..." + + # Import the organisation + curl -s -o /dev/null -w "%{http_code}" -X POST \ + -H "Authorization: ${ENV_B_TOKEN}" \ + -H "Content-Type: application/json" \ + -H "Accept: application/json" \ + -d "$org" "https://portal-env-b.example.com/organisations" +done < data/organisations.json +``` + +### Import Teams to Environment B + +After importing organisations, the next step is to import the teams associated with each organisation. This ensures that the organisational structure is accurately recreated in Environment B. + +Here's an example of how you can import teams into Environment B: + +```bash expandable +# Read each team file and import the teams +for file in data/teams_*.json; do + [[ -e "$file" ]] || continue + while IFS= read -r team; do + org_cid=$(basename "$file" | sed 's/teams_\(.*\)\.json/\1/') + team_cid=$(echo "$team" | jq -r '.CID') + echo "Importing team CID: $team_cid for organisation CID: $org_cid..." + + # Import the team + curl -s -o /dev/null -w "%{http_code}" -X POST \ + -H "Authorization: ${ENV_B_TOKEN}" \ + -H "Content-Type: application/json" \ + -H "Accept: application/json" \ + -d "$team" "https://portal-env-b.example.com/organisations/$org_cid/teams" + done < "$file" +done +``` + +### Verify the Migration + +After completing the migration, follow these steps to verify that everything was imported correctly: + +1. **Compare Organisation Counts** + - Check that the number of organisations in Environment B matches what you exported from Environment A + - Verify that each organisation's details (name, status, etc.) are correct + +2. **Verify Team Structure** + - Ensure all teams were created under their correct organisations + - Check that team configurations (permissions, settings) were preserved + +Example of verification script: + +```bash expandable +#!/bin/bash + +# Track total number of mismatches found +errors=0 +echo "Starting verification..." + +# === Organisation Verification === +echo "Checking organisations..." + +# Get organisations from source data file +# Format: "CID Name" for each organisation, sorted for comparison +source_orgs=$(jq -r '.[] | .CID + " " + .Name' data/organisations.json | sort) + +# Get organisations from target environment via API +# Exclude default organisation and format same as source +target_orgs=$(curl -s -H "Authorization: ${ENV_B_TOKEN}" \ + -H "Accept: application/json" \ + "https://portal-env-b.example.com/organisations" | \ + jq -r '.[] | select(.Name != "Default Organisation") | .CID + " " + .Name' | sort) + +# Compare organisation lists +# diff will show: < for missing in target, > for extra in target +if [ "$source_orgs" != "$target_orgs" ]; then + echo "❌ Organisation mismatch!" + diff <(echo "$source_orgs") <(echo "$target_orgs") || true + ((errors++)) +else + echo "βœ… Organisations match" +fi + +# === Team Verification === +echo -e "\nChecking teams..." + +# Iterate through each organisation to check its teams +while IFS= read -r org; do + # Split organisation line into CID and Name + org_cid=$(echo "$org" | cut -d' ' -f1) + org_name=$(echo "$org" | cut -d' ' -f2-) + + # Get teams from source data file for this organisation + source_teams=$(jq -r '.[] | .Name' "data/teams_${org_cid}.json" | sort) + + # Get teams from target environment for this organisation + # Exclude auto-generated "All users" teams + target_teams=$(curl -s -H "Authorization: ${ENV_B_TOKEN}" \ + -H "Accept: application/json" \ + "https://portal-env-b.example.com/organisations/$org_cid/teams" | \ + jq -r '.[] | select(.Name | endswith("All users") | not) | .Name' | sort) + + # Compare team lists for this organisation + if [ "$source_teams" != "$target_teams" ]; then + echo "❌ Team mismatch in '$org_name'" + diff <(echo "$source_teams") <(echo "$target_teams") || true + ((errors++)) + else + echo "βœ… Teams match in '$org_name'" + fi +done <<< "$source_orgs" + +# === Final Status === +# Exit with appropriate code: 0 for success, 1 for any errors +if [ $errors -eq 0 ]; then + echo -e "\nβœ… SUCCESS: Migration verified" + exit 0 +else + echo -e "\n❌ FAILURE: Found $errors error(s)" + exit 1 +fi +``` + +If you find any discrepancies, you may need to: +- Review the migration logs +- Re-run the import for specific resources +a \ No newline at end of file diff --git a/product-stack/tyk-charts/overview.mdx b/product-stack/tyk-charts/overview.mdx new file mode 100644 index 00000000..d7a7432b --- /dev/null +++ b/product-stack/tyk-charts/overview.mdx @@ -0,0 +1,25 @@ +--- +title: "Helm Chart Overview" +'og:description': "Explains an overview of Tyk charts" +sidebarTitle: "Overview" +tags: ['Tyk charts', 'helm charts', 'helm', 'charts', 'kubernetes', 'k8s'] +--- + +Tyk is working to provide a new set of helm charts, and will progressively roll them out at [tyk-charts](https://github.com/TykTechnologies/tyk-charts). It will provide component charts for all Tyk Components, as well as umbrella charts as reference configurations for open source and Tyk Self Managed users. Please check out the latest status from our [Github repository](https://github.com/TykTechnologies/tyk-charts). + +## Quick start guides +- [Quick Start with Tyk OSS Chart](/apim/open-source/installation#quick-start-with-helm-chart) +- [Quick Start with Tyk Data Plane Chart for Cloud Hybrid Gateways](/tyk-cloud#deploy-hybrid-gateways) +- [Quick Start with Tyk Stack Chart and PostgreSQL](/tyk-self-managed/install#install-tyk-stack-with-helm-chart-postgresql) +- [Quick Start with Tyk Stack Chart and MongoDB](/tyk-self-managed/install#install-tyk-stack-with-helm-chart-mongodb) + +## Which chart is right for you? + +| Umbrella Charts | Descriptions | +| :----------------- | :------------- | +| [tyk-stack](/product-stack/tyk-charts/tyk-stack-chart) | Deploys Tyk stack including Tyk Dashboard, Gateway, Enterprise Portal, and Pump | +| [tyk-control-plane](/product-stack/tyk-charts/tyk-control-plane-chart) | Deploys Tyk control plane including Tyk Dashboard, MDCB, Management Gateway, Enterprise Portal and Pump. Tyk control plane manages and configures distributed data planes in separate clusters / regions. | +| [tyk-data-plane](/product-stack/tyk-charts/tyk-data-plane-chart) | Deploys Tyk data plane including Tyk Gateway and Pump. Data planes are managed by Tyk Dashboard and MDCB in control plane. | +| [tyk-oss](/product-stack/tyk-charts/tyk-oss-chart) | Deploys Open source Tyk Gateway and Pump | + +Learn more about [Tyk Licensing and Deployment models](/tyk-self-managed#tyk-self-managed-pricing). diff --git a/product-stack/tyk-charts/tyk-control-plane-chart.mdx b/product-stack/tyk-charts/tyk-control-plane-chart.mdx new file mode 100644 index 00000000..b37b5008 --- /dev/null +++ b/product-stack/tyk-charts/tyk-control-plane-chart.mdx @@ -0,0 +1,923 @@ +--- +title: "Tyk Control Plane Chart" +'og:description': "Explains the Tyk Control Plane Chart" +sidebarTitle: "Tyk Control Plane Chart" +tags: ['Tyk Control Plane chart', 'Control Plane Chart', 'helm charts', 'helm', 'charts', 'kubernetes', 'k8s'] +--- + +The `tyk-control-plane` provides the default deployment of Tyk control plane on a Kubernetes cluster. It will deploy all required Tyk components with the settings provided in the values.yaml file. + +## What components are deployed with Tyk Control Plane Chart? + +It includes: +- Tyk Gateway, an Open Source Enterprise API Gateway (supporting REST, GraphQL, TCP and gRPC protocols). +- Tyk Dashboard, a license based component that provides a graphical management interface and analytics platform for Tyk. +- Tyk MDCB, a license based component that performs management and synchronisation of distributed clusters of Tyk API Gateways. +- Tyk Pump, an analytics purger that moves the data generated by your Tyk nodes to any back-end. +- Tyk Developer Portal, a full-fledged CMS-like system for API providers to publish, monetize and drive the adoption of APIs. + +Learn more about Tyk control plane at [MDCB components](/api-management/mdcb#mdcb-components). + +By default, this chart installs the following components as sub-charts on a [Kubernetes](https://kubernetes.io/) cluster using the [Helm](https://helm.sh/) package manager. + +| Component | Enabled by Default | Flag | +| :--------------------------------- | :-------------------- | :----------------------------- | +| Tyk Gateway | true | n/a | +| Tyk Dashboard | true | n/a | +| Tyk MDCB | true | n/a | +| Tyk Pump | false | global.components.pump | +| Tyk Enterprise Developer Portal | false | global.components.devPortal | +| Tyk Operator | false | global.components.operator | + +To enable or disable each component, change the corresponding enabled flag. + +Also, you can set the version of each component through `image.tag`. You could find the list of version tags available from [Docker hub](https://hub.docker.com/u/tykio). + +## Prerequisites + +- [Kubernetes 1.19+](https://kubernetes.io/docs/setup/) +- [Helm 3+](https://helm.sh/docs/intro/install/) +- [Redis](/apim/open-source/installation#configure-legacy-tyk-headless-helm-chart) should already be installed or accessible by the gateway and dashboard. +- [MongoDB](https://www.mongodb.com) or [PostgreSQL](https://www.postgresql.org) should already be installed or accessible by dashboard. Please consult the list of [supported versions](/api-management/dashboard-configuration#supported-database) that are compatible with Tyk. + + + + <Note> + If you want to enable Tyk Developer Portal, please use PostgreSQL. MongoDB is not supported in Developer Portal. + </Note> + + +## Tyk Control Plane Installations +### Installing The Chart + +To install the chart from Helm repository in namespace `tyk` with the release name `tyk-cp`, issue the following commands: +```bash +helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ +helm repo update +helm show values tyk-helm/tyk-control-plane > values.yaml +``` + +For further documentation relating to *helm* command usage, please refer to the [helm docs](https://helm.sh/docs/helm/). + +At a minimum, modify `values.yaml` for the following settings: +1. [Set Redis connection details](#set-redis-connection-details-required) +2. [Set Mongo or PostgreSQL connection details](#set-mongo-or-postgresql-connection-details-required) +3. [Tyk Dashboard License](#tyk-dashboard-license-required) +4. [Tyk MDCB License](#tyk-mdcb-license-required) +5. If you would like to use Developer Portal, an additional license is required: [Tyk Developer Portal License](#tyk-developer-portal-license-required) + +By default, the chart would expose MDCB service as ClusterIP. If you would like to access the control plane from outside the cluster, please change the service type `tyk-mdcb.mdcb.service.type` to NodePort or LoadBalancer. + +Then just run: +```bash +helm install tyk-cp tyk-helm/tyk-control-plane -n tyk --create-namespace -f values.yaml +``` + +Follow the installation output to obtain connection details to Tyk MDCB, and use that to configure Tyk Data Planes using [tyk-data-plane](/product-stack/tyk-charts/tyk-data-plane-chart) chart. + +### Uninstalling The Chart + +```bash +helm uninstall tyk-cp -n tyk +``` +This removes all the Kubernetes components associated with the chart and deletes the release. + +### Upgrading Chart + +```bash +helm upgrade tyk-cp tyk-helm/tyk-control-plane -n tyk -f values.yaml +``` + +## Configuration + +To get all configurable options with detailed comments, issue the following command: + +```bash +helm show values tyk-helm/tyk-control-plane > values.yaml +``` + +You can update any value in your local `values.yaml` file and use `-f [filename]` flag to override default values during installation. +Alternatively, you can use `--set` flag to set it in Tyk installation. See [Using Helm](https://helm.sh/docs/intro/using_helm/) for examples. + +To configure Tyk components, users can utilize both config files and [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/). Notably, environment variables take precedence over config files. To maintain simplicity and consistency, the Tyk Helm Charts deploy components with an empty config file while setting container environment variables based on user-defined [values](https://helm.sh/docs/chart_best_practices/values/). This approach ensures seamless integration with Kubernetes practices, allowing for efficient management of configurations. For a comprehensive overview of available configurations, please refer to the [configuration documentation](/tyk-oss-gateway/configuration). + +### Bootstrapping + +By default, the chart executes a [bootstrapping job](https://github.com/TykTechnologies/tyk-k8s-bootstrap) immediately after installation. This process ensures the presence of a valid dashboard license and initializes key components such as tyk-dashboard, tyk-portal, and tyk-operator, enabling them for immediate use. + +The bootstrapping job uses three distinct applications acting as Helm chart hooks: + +| **Bootstrapping Component** | **Description** | +| :----------------------------- | :----------------- | +| `bootstrap-pre-install` | - Runs as a pre-install hook. <br /> - Validates the Tyk Dashboard license key to ensure proper installation prerequisites. | +| `bootstrap-post-install` | - Executes post-installation. <br /> - Sets up an organization and admin user in the Tyk Dashboard. <br /> - Creates Kubernetes secrets required by Tyk Operator and Tyk Enterprise Portal. <br /> **Note**: If an existing organization and admin user are found in the database, the bootstrapping job will not set up a new organization and user. The Kubernetes secrets will not contain the expected Org ID or API key. Please update the Secret with existing credentials in this case. | +| `bootstrap-pre-delete` | - Functions as a pre-delete hook. <br /> - Cleans up resources, ensuring no residual configurations remain post-uninstallation. | + +Key Notes on Bootstrapping: + +- Bootstrapping is triggered **only during** a `helm install` and does not run during a `helm upgrade`. +- If `global.components.bootstrap` is set to `false`, only the dashboard license check will be performed. + +Handling Bootstrapping Failures: + +- If the bootstrapping process fails, check the logs from the bootstrap pods to diagnose the issue. +- Once reviewed, you can safely delete the bootstrap pods. +- To re-trigger the bootstrapping process after a failure, you must run `helm uninstall` and start the installation process anew. + +### Setting Environment Variables +Should any environment variables not be set by the Helm Chart, users can easily add them under the `extraEnvs` section within the charts for further customization. Values set under `extraEnvs` would take precedence over all configurations. + +Example of setting extra environment variable to gateway: +```yaml +tyk-gateway: + gateway: + extraEnvs: + - name: TYK_GW_LOGLEVEL + value: debug +``` + +An example is listed below for setting extra [environment variable using ConfigMap data](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#define-container-environment-variables-using-configmap-data), using gateway: +```yaml expandable +tyk-gateway: + gateway: + extraEnvs: + - name: CONFIG_USERNAME + valueFrom: + configMapKeyRef: + name: backend-user + key: backend-username +``` + +An example is listed below for setting extra [environment variable using secret data](https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#define-container-environment-variables-using-secret-data), using gateway: +```yaml expandable +tyk-gateway: + gateway: + extraEnvs: + - name: SECRET_USERNAME + valueFrom: + secretKeyRef: + name: backend-user + key: backend-username +``` + +In the above example, an extra environment variable `SECRET_USERNAME` will be added to the Gateway container, with a value of `backend-username` associated with the secret `backend-user`. It is useful if you want to access secret data from [Tyk Gateway configuration file (tyk.conf) or API definitions](/tyk-configuration-reference/kv-store). + +### Set Redis Connection Details (Required) + +Tyk uses Redis for distributed rate-limiting and token storage. You may use the [Bitnami chart](https://github.com/bitnami/charts/tree/main/bitnami/redis) or Tyk's [simple-redis](https://artifacthub.io/packages/helm/tyk-helm/simple-redis) to install chart for POC purpose. + +Set the following values after installing Redis: + +| Name | Description | +| :------ | :------------- | +| `global.redis.addrs` | Redis addresses | +| `global.redis.pass` | Redis password in plain text | +| `global.redis.passSecret.name` | If global.redis.pass is not provided, you can store it in a secret and provide the secret name here | +| `global.redis.passSecret.keyName` | key name to retrieve redis password from the secret | + +***Recommended: via *Bitnami* chart*** + +For Redis you can use these rather excellent charts provided by [Bitnami](https://github.com/bitnami/charts/tree/main/bitnami/redis). +Copy the following commands to add it: + +```bash +helm upgrade tyk-redis oci://registry-1.docker.io/bitnamicharts/redis -n tyk --create-namespace --install --version 19.0.2 +``` + + +<Note> +Please make sure you are installing Redis versions that are supported by Tyk. Please refer to Tyk docs to get list of [supported versions](/tyk-self-managed#redis). +</Note> + + +Follow the notes from the installation output to get connection details and password. + +``` + Redis(TM) can be accessed on the following DNS names from within your cluster: + + tyk-redis-master.tyk.svc.cluster.local for read/write operations (port 6379) + tyk-redis-replicas.tyk.svc.cluster.local for read-only operations (port 6379) + + export REDIS_PASSWORD=$(kubectl get secret --namespace tyk tyk-redis -o jsonpath="{.data.redis-password}" | base64 --decode) +``` + +The Redis address as set by Bitnami is `tyk-redis-master.tyk.svc.cluster.local:6379` + +You can reference the password secret generated by Bitnami chart by `--set global.redis.passSecret.name=tyk-redis` and `--set global.redis.passSecret.keyName=redis-password`, or just set `global.redis.pass=$REDIS_PASSWORD` + +***Evaluation only: via *simple-redis* chart*** + +Another option for Redis, to get started quickly, is to use our [simple-redis](https://artifacthub.io/packages/helm/tyk-helm/simple-redis) chart. + + +<Warning> +Please note that these provided charts must never be used in production or for anything +but a quick start evaluation only. Use [Bitnami Redis](https://github.com/bitnami/charts/tree/main/bitnami/redis) or [Redis Enterprise operator](https://docs.redis.com/latest/kubernetes/deployment/) in any other case. +We provide this chart, so you can quickly deploy *Tyk gateway*, but it is not meant for long term storage of data. +</Warning> + + +```bash +helm install redis tyk-helm/simple-redis -n tyk +``` + +The Tyk Helm Chart can connect to `simple-redis` in the same namespace by default. You do not need to set Redis address and password in `values.yaml`. + +### Set Mongo or PostgreSQL Connection Details (Required) +If you have already installed MongoDB or PostgreSQL, you can set the connection details in `global.mongo` and `global.postgres` section of values file respectively. + +If not, you can use these rather excellent charts provided by Bitnami to install MongoDB or PostgreSQL: + +**Mongo Installation** + +```bash +helm install tyk-mongo bitnami/mongodb --set "replicaSet.enabled=true" -n tyk --version 15.1.3 +``` + +Then follow notes from the installation output to get connection details and update them in `values.yaml` file. + + +<Note> +Bitnami MongoDB image is not supported on darwin/arm64 architecture. +</Note> + + + +<Note> +Please make sure you are installing MongoDB versions that are supported by Tyk. Please refer to Tyk docs to get list of [supported versions](/api-management/dashboard-configuration#supported-database). +</Note> + + + +<Note> +Important Note regarding MongoDB: + +This helm chart enables the `PodDisruptionBudget` for MongoDB with an arbiter replica-count of 1. If you intend to perform system maintenance on the node where the MongoDB pod is running and this maintenance requires the node to be drained, then this action will be prevented due to the the replica count being 1. + +Increase the replica count in the helm chart deployment to a minimum of 2 to remedy this issue. +</Note> + + +Configure `global.mongo.mongoURL` and `global.storageType` as below. You should replace `pass` in the connection string with the MONGODB_ROOT_PASSWORD you obtain from the installation output notes. + +```yaml expandable +global: + # Set mongo connection details if you want to configure mongo pump. + mongo: + # The mongoURL value will allow you to set your MongoDB address. + # Default value: mongodb://mongo.{{ .Release.Namespace }}.svc:27017/tyk_analytics + # mongoURL: mongodb://mongo.tyk.svc:27017/tyk_analytics + + # If your MongoDB has a password you can add the username and password to the url + mongoURL: mongodb://root:pass@tyk-mongo-mongodb.tyk.svc:27017/tyk_analytics?authSource=admin + + # mongo-go driver is supported for Tyk 5.0.2+. + # We recommend using the mongo-go driver if you are using MongoDB 4.4.x+. + # For MongoDB versions prior to 4.4, please use the mgo driver. + # Since Tyk 5.3 the default driver is mongo-go. + driver: mongo-go + + # Connection URL can also be set using a secret. Provide the name of the secret and key below. + # connectionURLSecret: + # name: "" + # keyName: "" + + # Enables SSL for MongoDB connection. MongoDB instance will have to support that. + # Default value: false + useSSL: false + + # Choose the storageType for Tyk. [ "mongo", "postgres" ] + storageType: &globalStorageType mongo +``` + +**PostgreSQL Installation** +```bash +helm install tyk-postgres bitnami/postgresql --set "auth.database=tyk_analytics" -n tyk --version 12.12.10 +``` + +Follow the notes from the installation output to get connection details. + + +<Note> +Please make sure you are installing PostgreSQL versions that are supported by Tyk. Please refer to Tyk docs to get list of [supported versions](/api-management/dashboard-configuration#supported-database). +</Note> + + +```yaml expandable +global: + # Postgres connection string parameters. + postgres: + # host corresponds to the host name of postgres + host: tyk-postgres-postgresql.tyk.svc + # port corresponds to the port of postgres + port: 5432 + # user corresponds to the user of postgres + user: postgres + # password corresponds to the password of the given postgres user in selected database + password: + # database corresponds to the database to be used in postgres + database: tyk_analytics + # sslmode corresponds to if postgres runs in sslmode (https) + sslmode: disable + # Connection string can also be set using a secret. Provide the name of the secret and key below. + # connectionStringSecret: + # name: "" + # keyName: "" +``` + +### Protect Confidential Fields with Kubernetes Secrets + +In the `values.yaml` file, some fields are considered confidential, such as `APISecret`, connection strings, etc. + +Declaring values for such fields as plain text might not be desired. Instead, for certain fields, Kubernetes secrets can be referenced, and the chart will +[define container environment variables using secret data](https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#define-container-environment-variables-using-secret-data). + +This section describes how to use Kubernetes secrets to declare confidential fields. + +***Tyk Dashboard and Developer Portal Admin*** + +If Tyk Dashboard bootstrapping is enabled, the admin user will be created according to the `global.adminUser` field. + +All admin credentials can also be set through Kubernetes secret. + + +<Note> +Once `global.adminUser.useSecretName` is declared, it takes precedence over `global.adminUser.firstName`, +`global.adminUser.lastName`, `global.adminUser.email` and `global.adminUser.password`. + +If `global.adminUser.useSecretName` is in use, please add all keys mentioned below to the secret. +</Note> + + +***Tyk Dashboard Admin First Name*** + +It can be configured via `global.adminUser.firstName` as a plain text or Kubernetes secret which includes `adminUserFirstName` key in it. Then, this secret must be referenced via `global.adminUser.useSecretName`. + + +***Tyk Dashboard Admin Last Name*** + +It can be configured via `global.adminUser.lastName` as a plain text or Kubernetes secret which includes `adminUserLastName` key in it. Then, this secret must be referenced via `global.adminUser.useSecretName`. + +***Tyk Dashboard and Developer Portal Admin Email*** + +It can be configured via `global.adminUser.email` as a plain text or Kubernetes secret which includes `adminUserEmail` key in it. Then, this secret must be referenced via `global.adminUser.useSecretName`. + +***Tyk Dashboard and Developer Portal Admin Password*** + +It can be configured via `global.adminUser.password` as a plain text or Kubernetes secret which includes `adminUserPassword` key in it. Then, this secret must be referenced via `global.adminUser.useSecretName`. + +***APISecret*** + +The `global.secrets.APISecret` field configures a [header value](/tyk-oss-gateway/configuration#secret) used in every interaction with Tyk Gateway API. + +It can be configured via `global.secrets.APISecret` as a plain text or Kubernetes secret which includes `APISecret` key in it. Then, this secret must be referenced via `global.secrets.useSecretName`. + +```yaml +global: + secrets: + APISecret: CHANGEME + useSecretName: "mysecret" # where mysecret includes `APISecret` key with the desired value. +``` + +***AdminSecret*** + +The `global.secrets.AdminSecret` field sets a [secret](/tyk-dashboard/configuration#admin_secret) for Admin API. + +It can be configured via `global.secrets.AdminSecret` as a plain text or Kubernetes secret which includes `AdminSecret` key in it. Then, this secret must be referenced via `global.secrets.useSecretName`. + +```yaml +global: + secrets: + useSecretName: "mysecret" # where mysecret includes `AdminSecret` key with the desired value. +``` + + +<Note> +Once `global.secrets.useSecretName` is declared, it takes precedence over `global.secrets.APISecret` and `global.secrets.AdminSecret`. +</Note> + + +***Dashboard License*** + +In order to refer to a Tyk Dashboard license through a Kubernetes secret, please use `global.secrets.useSecretName`, where the secret should contain a key called `DashLicense`. + +***MDCB License*** + +In order to refer to a Tyk MDCB license through a Kubernetes secret, please use `tyk-mdcb.mdcb.useSecretName`, where the secret should contain a key called `MDCBLicense`. + +***MDCB Secret*** + +In order to set the secret for accessing MDCB secure HTTP endpoints through a Kubernetes secret, please use tyk-`mdcb.mdcb.useSecretName`, where the secret should contain a key called `securitySecret`. + +***Tyk Developer Portal License*** + +In order to refer to a Tyk Developer Portal license through a Kubernetes secret, please use `tyk-dev-portal.useSecretName`, where the secret should contain a key called `DevPortalLicense`. + +***Tyk Developer Portal Admin Password*** + +In order to refer to a Tyk Developer Portal's admin password through a Kubernetes secret, +please use `global.adminUser.useSecretName`, where the secret should contain a key called `adminUserPassword`. + +***Tyk Developer Portal Storage Connection String*** + +In order to refer to a Tyk Developer Portal connection string to the selected database through a Kubernetes secret, please use `tyk-dev-portal.useSecretName`, where the secret should contain a key called `DevPortalStorageConnectionString`. + + +<Note> +If `tyk-dev-portal.useSecretName` is in use, please add all keys mentioned to the secret. +</Note> + + +***Tyk Developer Portal AWS S3 Access Key ID*** + +In order to refer to a Tyk Developer Portal AWS S3 Access Key ID through a Kubernetes secret, please use `tyk-dev-portal.useSecretName`, where the secret should contain a key called `DevPortalAwsAccessKeyId`. + + +<Note> +If `tyk-dev-portal.useSecretName` is in use, please add all keys mentioned to the secret. +</Note> + + +***Tyk Developer Portal AWS S3 Secret Access Key*** + +In order to refer Tyk Developer Portal connection string to the selected database through Kubernetes secret, +please use `tyk-dev-portal.useSecretName`, where the secret should contain a key called +`DevPortalAwsSecretAccessKey`. + + +<Note> +If `tyk-dev-portal.useSecretName` is in use, please add all keys mentioned to the secret. +</Note> + + +***Redis Password*** + +The Redis password can also be provided via a secret. Store the Redis password in Kubernetes secret and refer to this secret via `global.redis.passSecret.name` and `global.redis.passSecret.keyName` field, as follows: + +```yaml +global: + redis: + passSecret: + name: "yourSecret" + keyName: "redisPassKey" +``` + +***MongoDB or Postgres connection strings*** + +Storage connection strings can also be provided via a secret. + +For MongoDB, suppose you have a secret named `yourSecret` and you have the mongo connection URL stored in key `mongoConnectionURLkey`. Store the connection string in Kubernetes secret and refer to this secret via `global.mongo.connectionURLSecret.name` and `global.mongo.connectionURLSecret.keyName` field, as follows: + +- MongoDB: +```yaml +global: + mongo: + connectionURLSecret: + name: "yourSecret" + keyName: "mongoConnectionURLkey" +``` + +For Postgres, suppose you have a secret named `yourSecret` and you have the postgres connection string stored in key `postgreConnectionURLkey`. Store the connection string in Kubernetes secret and refer to this secret via `global.postgres.connectionStringSecret.name` and `global.postgres.connectionStringSecret.keyName` field, as follows: + +- Postgres: +```yaml +global: + postgres: + connectionStringSecret: + name: "yourSecret" + keyName: "postgreConnectionURLkey" +``` + +**_Tyk Operator License_** + +It can be configured via `global.license.operator` as a plain text or Kubernetes secret which includes `OperatorLicense` key in it. Then, this secret must be referenced via `global.secrets.useSecretName`. + +Note: If you are using `global.secrets.useSecretName`, you must configure the operator license in the referenced Kubernetes secret. `global.license.operator` will not be used in this case. + +### Gateway Configurations + + +<Note> +In Tyk control plane, Tyk Gateway acts as a management gateway that is used for creation of keys and certificates. It does not service API requests. It is important to ensure there is no public access to it and it must not be sharded (tagged) as it β€œbelongs” to the whole Tyk installation. +</Note> + + +Configure the following details below, inside the `tyk-gateway` section. + +#### Update Tyk Gateway Version +Set version of gateway at `tyk-gateway.gateway.image.tag`. You can find the list of version tags available from [Docker hub](https://hub.docker.com/r/tykio/tyk-gateway/tags). Please check [Tyk Release notes](/developer-support/release-notes/gateway) carefully while upgrading or downgrading. + +#### Enabling TLS + +*Enable TLS* + +We have provided an easy way to enable TLS via the `global.tls.gateway` flag. Setting this value to true will automatically enable TLS using the certificate provided under tyk-gateway/certs/. + +*Configure TLS secret* + +If you want to use your own key/cert pair, please perform the following steps: +1. Create a TLS secret using your cert and key pair. +2. Set `global.tls.gateway` to true. +3. Set `tyk-gateway.gateway.tls.useDefaultTykCertificate` to false. +4. Set `tyk-gateway.gateway.tls.secretName` to the name of the newly created secret. + +*Add Custom Certificates* + +To add your custom Certificate Authority(CA) to your containers, you can mount your CA certificate directly into /etc/ssl/certs folder. + +```yaml expandable + extraVolumes: + - name: self-signed-ca + secret: + secretName: self-signed-ca-secret + extraVolumeMounts: + - name: self-signed-ca + mountPath: "/etc/ssl/certs/myCA.pem" + subPath: myCA.pem +``` + +#### Enabling gateway autoscaling +You can enable autoscaling of the gateway by `--set tyk-gateway.gateway.autoscaling.enabled=true`. By default, it will enable the `Horizontal Pod Autoscaler` resource with target average CPU utilization at 60%, scaling between 1 and 3 instances. To customize those values you can modify the `tyk-gateway` section of `values.yaml` as shown below: + +```yaml expandable +tyk-gateway: + gateway: + autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 30 +``` + +Built-in rules include `tyk-gateway.gateway.autoscaling.averageCpuUtilization` for CPU utilization (set by default at 60%) and `tyk-gateway.gateway.autoscaling.averageMemoryUtilization` for memory (disabled by default). In addition to that you can define rules for custom metrics using `tyk-gateway.gateway.autoscaling.autoscalingTemplate` list: + +```yaml expandable +tyk-gateway: + gateway: + autoscaling: + autoscalingTemplate: + - type: Pods + pods: + metric: + name: nginx_ingress_controller_nginx_process_requests_total + target: + type: AverageValue + averageValue: 10000m +``` + +### Pump Configurations + +To enable Pump, set `global.components.pump` to true and configure the following parameters included below inside the `tyk-pump` section. + +{/* BEGIN import from pump doc */} + +| Pump | Configuration | +| :--------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------- | +| Prometheus Pump (Default) | Add the value `prometheus` to the `tyk-pump.pump.backend` entry and add connection details for Prometheus under `tyk-pump.pump.prometheusPump`. | +| Mongo Pump | Add `mongo` to `tyk-pump.pump.backend` and add connection details for mongo under `global.mongo`. | +| Mongo Selective Pump | Add `mongo-selective` to `tyk-pump.pump.backend` and add connection details for mongo under `global.mongo`. | +| Mongo Aggregate Pump | Add `mongo-aggregate` to `tyk-pump.pump.backend` and add connection details for mongo under `global.mongo`. | +| Postgres Pump | Add `postgres` to `tyk-pump.pump.backend` and add connection details for postgres under `global.postgres`. | +| Postgres Aggregate Pump | Add `postgres-aggregate` to `tyk-pump.pump.backend` and add connection details for postgres under `global.postgres`. | +| Uptime Pump | Set `tyk-pump.pump.uptimePumpBackend` to `mongo` or `postgres` or `""` | +| Other Pumps | Add the required environment variables in `tyk-pump.pump.extraEnvs` | + + + +<Note> +For additional information on Tyk Pump configurations, refer to the +[Setup Dashboard Analytics](/api-management/tyk-pump#setup-dashboard-analytics) documentation. + +To explore the list of supported backends for Tyk Pump, please visit [Pump Backends](/api-management/tyk-pump#external-data-stores). +</Note> + + +#### Prometheus Pump +Add `prometheus` to `tyk-pump.pump.backend` and add connection details for Prometheus under `tyk-pump.pump.prometheusPump`. + +We also support monitoring using Prometheus Operator. All you have to do is set `tyk-pump.pump.prometheusPump.prometheusOperator.enabled` to true. + +This will create a _PodMonitor_ resource for your Pump instance. + +```yaml expandable + # prometheusPump configures Tyk Pump to expose Prometheus metrics. + # Please add "prometheus" to .Values.pump.backend in order to enable Prometheus Pump. + prometheusPump: + # host represents the host without port, where Tyk Pump serve the metrics for Prometheus. + host: "" + # port represents the port where Tyk Pump serve the metrics for Prometheus. + port: 9090 + # path represents the path to the Prometheus collection. For example /metrics. + path: /metrics + # customMetrics allows defining custom Prometheus metrics for Tyk Pump. + # It accepts a string that represents a JSON object. For instance, + # + # customMetrics: '[{"name":"tyk_http_requests_total","description":"Total of API requests","metric_type":"counter","labels":["response_code","api_name","method","api_key","alias","path"]}, { "name":"tyk_http_latency", "description":"Latency of API requests", "metric_type":"histogram", "labels":["type","response_code","api_name","method","api_key","alias","path"] }]' + customMetrics: "" + # If you are using prometheus Operator, set the fields in the section below. + prometheusOperator: + # enabled determines whether the Prometheus Operator is in use or not. By default, + # it is disabled. + # Tyk Pump can be monitored with PodMonitor Custom Resource of Prometheus Operator. + # If enabled, PodMonitor resource is created based on .Values.pump.prometheusPump.prometheusOperator.podMonitorSelector + # for Tyk Pump. + enabled: false + # podMonitorSelector represents a podMonitorSelector of your Prometheus resource. So that + # your Prometheus resource can select PodMonitor objects based on selector defined here. + # Please set this field to the podMonitorSelector field of your monitoring.coreos.com/v1 + # Prometheus resource's spec. + # + # You can check the podMonitorSelector via: + # kubectl describe prometheuses.monitoring.coreos.com <PROMETHEUS_POD> + podMonitorSelector: + release: prometheus-stack +``` + +#### Mongo pump +To enable Mongo pump, add `mongo` to `tyk-pump.pump.backend` and add connection details for mongo under `global.mongo`. See [Mongo Installation](#set-mongo-or-postgresql-connection-details-required) section above. + +By default, it will enable Mongo Aggregate, Mongo Graph Pump and Mongo Selective Pump. + + +#### SQL Pump +To enable SQL pump, add `postgres` to `tyk-pump.pump.backend` and add connection details for postgres under `global.postgres`. See [PostgresSQL Installation](#set-mongo-or-postgresql-connection-details-required) section above. + +By default, it will enable Postgres Aggregate, Postgres Graph Aggregate, SQL Pump and SQL graph pump. + + +#### Uptime Pump +Uptime Pump can be configured by setting `pump.uptimePumpBackend` in values.yaml file. It supports the following values: +1. mongo: Used to set mongo pump for uptime analytics. Mongo Pump should be enabled. +2. postgres: Used to set postgres pump for uptime analytics. Postgres Pump should be enabled. +3. empty: Used to disable uptime analytics. + +```yaml + # uptimePumpBackend configures uptime Tyk Pump. ["", "mongo", "postgres"]. + # Set it to "" for disabling uptime Tyk Pump. By default, uptime pump is disabled. + uptimePumpBackend: "" +``` + +#### Other Pumps +To setup other backends for pump, refer to this [document](https://github.com/TykTechnologies/tyk-pump/blob/master/README.md#pumps--back-ends-supported) and add the required environment variables in `pump.extraEnvs` + + +### Tyk Dashboard Configurations + +#### Tyk Dashboard License (Required) + +Tyk Dashboard License is required. It can be set up in `global.license.dashboard` or through secret `global.secrets.useSecretName`. The secret should contain a key called DashLicense. + +```yaml expandable +global: + license: + # The license key needed by Tyk Dashboard to work. + # + # NOTE: If you do not want to store license as a plain text in the file, you can use a Kubernetes secret + # that stores the dashboard license. Please see `.global.secrets.useSecretName`. + dashboard: "" +``` + +#### Enabling Dashboard TLS + +Assuming that TLS certificates for the Tyk Dashboard are available in the Kubernetes Secret `tyk-dashboard-tls`, follow these steps to enable TLS: + +1. Set `global.tls.dashboard` to `true`. +2. Set `tyk-dashboard.dashboard.tls.secretName` to the name of the Kubernetes secret containing TLS certificates for the Tyk Dashboard, in this case, `tyk-dashboard-tls`. +3. Define certificate configurations in `tyk-dashboard.dashboard.tls.certificates`, which generates `TYK_DB_HTTPSERVEROPTIONS_CERTIFICATES` for the Tyk Dashboard. + +Optional Steps, if needed: +- Modify the secret mount path on the Tyk Dashboard Pod via `tyk-dashboard.dashboard.tls.certificatesMountPath`. +- If necessary, either enable `insecureSkipVerify` via `tyk-dashboard.dashboard.tls.certificates`, or mount CA information through `tyk-dashboard.dashboard.extraVolumes` and `tyk-dashboard.dashboard.extraVolumeMounts`. +- If the `tyk-bootstrap` chart is used to bootstrap the Tyk Dashboard, ensure that bootstrap app can validate certificate of Tyk Dashboard or enable `insecureSkipVerify` in the `tyk-bootstrap` chart. +- If the Tyk Gateway connects to the Tyk Dashboard, confirm that the Tyk Gateway has appropriate certificates for connecting to the Tyk Dashboard + +#### Audit Log Configurations + +You can manage audit logging for Tyk Dashboard via `auditLogs`: + +- `auditLogs.enabled`: Enables or disables audit logging. It sets corresponding Dashboard environment variable `TYK_DB_AUDIT_ENABLED`. Disabled by default. +- `auditLogs.type`: Specifies the storage type for audit logs (`db` or `file`). It sets corresponding Dashboard environment variable `TYK_DB_AUDIT_STORETYPE`. Set to `file` by default. +- `auditLogs.format`: Defines the format of audit log files (`json` or `text`). It sets corresponding Dashboard environment variable `TYK_DB_AUDIT_FORMAT`. Set to `text` by default. +- `auditLogs.path`: Sets the path to the audit log file. It sets corresponding Dashboard environment variable `TYK_DB_AUDIT_PATH`. Set to "" by default. +- `auditLogs.enableDetailedRecording`: Enables detailed logging, including HTTP requests (headers only) and full HTTP responses. It sets corresponding Dashboard environment variable `TYK_DB_AUDIT_DETAILEDRECORDING`. Disabled by default. + +<h4 id="opa-configurations">OPA Configurations</h4> +You can manage OPA (Open Agent Policy) for Tyk Dashboard via `opa`: + +- `opa.enabled`: Enables OPA support. It sets corresponding Dashboard environment `TYK_DB_SECURITY_OPENPOLICY_ENABLED`. Disabled by default. +- `opa.debug`: Activates OPA debug mode for detailed logging of policy execution. It sets corresponding Dashboard environment `TYK_DB_SECURITY_OPENPOLICY_DEBUG`. Disabled by default. +- `opa.api`: Enables OPA API mode to manage policies via the Dashboard API. It sets corresponding Dashboard environment `TYK_DB_SECURITY_OPENPOLICY_ENABLEAPI`. Disabled by default. +- `opa.allowAdminPasswordReset`: Required if OPA is enabled with its default policies. It sets corresponding Dashboard environment `TYK_DB_SECURITY_ALLOWADMINRESETPASSWORD`. Enabled by default. + +### Tyk MDCB Configurations + +#### Tyk MDCB License (Required) + +Tyk MDCB requires a license to be set at `tyk-mdcb.mdcb.license`. This field is mandatory and must be configured. + +To enhance security and avoid storing plaintext values for the MDCB license directly in the Helm value file, an alternative approach is available. You can store the license in a Kubernetes Secret and reference it externally. + +Set the license in the Kubernetes Secret and provide the secret's name through `tyk-mdcb.mdcb.useSecretName`. The Secret must contain a key named `MDCBLicense`. + +#### Tyk MDCB Listen Port + +The `tyk-mdcb.mdcb.listenPort` field represents a RPC port which worker Tyk Gateways will connect to. +Setting `tyk-mdcb.mdcb.listenPort` field opens a port on MDCB container and MDCB service targets this port. +It is used to set `TYK_MDCB_LISTENPORT` + +#### Tyk MDCB HTTP Port + +The HTTP port for Tyk MDCB is configurable via the `tyk-mdcb.mdcb.httpPort` field. +This port enables MDCB to accept standard HTTP requests, such as health checks. + +It also defines the path for liveness and readiness probes. +It is used to set `TYK_MDCB_HTTPPORT` in MDCB 2.6.0+ or `TYK_MDCB_HEALTHCHECKPORT` in MDCB 2.5.x or prior. + +#### Enabling secured HTTP endpoints + +The [MDCB OAS API](/tyk-mdcb-api) has secured HTTP endpoints, like `/dataplanes` which return list of gateway nodes connected. By default, this endpoint is disabled to avoid unintended leakage of data plane information. To enable this endpoint, set `tyk-mdcb.mdcb.security.enableHttpSecureEndpoints` to `true`. It is used to set `TYK_MDCB_SECURITY_ENABLEHTTPSECUREENDPOINTS`. Also, you need to set a secret that can be used to access this endpoint via `tyk-mdcb.mdcb.security.secret` field. + +#### Enabling MDCB TLS + +Assuming that TLS certificates for the Tyk MDCB are available in the Kubernetes Secret `mdcb-tls-secret`, follow these steps to enable TLS for RPC connection: +1. Set `tyk-mdcb.mdcb.tls.useSSL` to true. +2. Set `tyk-mdcb.mdcb.tls.secretName` to the name of the Kubernetes secret containing TLS certificates for the Tyk MDCB, in this case, `mdcb-tls-secret`. + +```yaml expandable +tyk-mdcb: + mdcb: + tls: + # enables ssl for mdcb + useSSL: true + + # the path to where the keys will be mounted in the pod + certificatesMountPath: "/etc/certs" + + # location to pem encoded private key + certificateKeyFile: "/etc/certs/tls.key" + + # location to pem encoded certificate + certificateCertFile: "/etc/certs/tls.crt" + + # the name of the secret + secretName: "mdcb-tls-secret" + + # the name of the volume + volumeName: "mdcb-tls-secret-volume" +``` + +To enable TLS for MDCB HTTP connection, use `tyk-mdcb.mdcb.httpServerOptions`. The configuration includes settings such as `useSSL`, `certificateKeyFile`, `certificateCertFile` and `minVersion`. For other [HTTP server options](/tyk-multi-data-centre/mdcb-configuration-options#http_server_options), users can utilize [extraEnvs](#setting-environment-variables) to configure additional parameters. + +```yaml expandable +tyk-mdcb: + mdcb: + # defines the SSL/TLS settings for the http server where the healthcheck is exposed + httpServerOptions: + # if enabled then the endpoints will be served over https + useSSL: true + certificateKeyFile: /path-to-cert-keyfile + certificateCertFile: /path-to-certfile + + # For TLS 1.0 use 769, for TLS 1.1 use 770, for TLS 1.2 use 771, for TLS 1.3 use 772 + minVersion: 771 +``` + +### Tyk Bootstrap Configurations + +To enable [bootstrapping](#bootstrapping), set `global.components.bootstrap` to `true`. It would run [tyk-k8s-bootstrap](https://github.com/TykTechnologies/tyk-k8s-bootstrap) to bootstrap `tyk-control-plane` and to create Kubernetes secrets that can be utilized in Tyk Operator and Tyk Developer Portal. + + +<Note> +During bootstrapping, admin user needs to reset its password. It may be denied by Dashboard OPA rules if [OPA](/api-management/dashboard-configuration#extend-permissions-using-open-policy-agent-opa) was enabled. Please disable OPA during the initial bootstrapping or set Dashboard configuration [TYK_DB_SECURITY_ALLOWADMINRESETPASSWORD](/tyk-dashboard/configuration#securityallow_admin_reset_password) to true. +</Note> + + +#### Bootstrapped Environments + +If Tyk is already bootstrapped, the application will bypass the creation of the Tyk Organization and Admin User, proceeding directly with the creation of Kubernetes Secrets. + +Given that the Kubernetes Secrets require values for `TYK_AUTH` and `TYK_ORG`, it is essential to provide these values through the respective environment variables, called `TYK_K8SBOOTSTRAP_TYK_ADMIN_AUTH` for `TYK_AUTH` and `TYK_K8SBOOTSTRAP_TYK_ORG_ID` for `TYK_ORG`. + +Ensure that these environment variables are set appropriately to `postInstall` hook for bootstrapped environments. + +### Tyk Developer Portal Configurations + +To enable Tyk Developer Portal, set `global.components.devPortal` to true, and configure below inside `tyk-dev-portal` section. + +#### Tyk Developer Portal License (Required) + +Tyk Developer Portal License is required. It can be set up in `tyk-dev-portal.license` or through secret `global.secrets.useSecretName`. The secret should contain a key called `DevPortalLicense`. + +```yaml +tyk-dev-portal: + # Tyk Developer Portal license. + license: "" +``` + +#### Tyk Developer Portal Database + + +<Note> +Tyk no longer supports SQLite as of Tyk 5.7.0. To avoid disruption, please transition to [PostgreSQL](/planning-for-production/database-settings#postgresql), [MongoDB](/planning-for-production/database-settings#mongodb), or one of the listed compatible alternatives. +</Note> + + +By default, Tyk Developer Portal use `sqlite3` to store portal metadata. If you want to use a different SQL Database, please modify the section below. + +```yaml expandable +tyk-dev-portal: + database: + # This selects the SQL dialect to be used + # The supported values are mysql, postgres and sqlite3 + dialect: "sqlite3" + connectionString: "db/portal.db" + enableLogs: false + maxRetries: 3 + retryDelay: 5000 +``` + +#### Storage Settings + +Tyk Developer Portal supports different storage options for storing the portal's CMS assets such as images, theme files and Open API Specification files. Please see the [Developer Portal Storage settings](/product-stack/tyk-enterprise-developer-portal/deploy/configuration#portal-settings) page for all the available options. + +If you use the file system as storage, please set `tyk-dev-portal.storage.type` to `fs`, and configure `tyk-dev-portal.storage.persistence` to mount an existing persistent volume to Tyk Developer Portal. + +If you use [AWS S3](https://aws.amazon.com/s3/) as storage, please set `tyk-dev-portal.storage.type` to `s3`, and configure `tyk-dev-portal.storage.s3` section with credentials to access AWS S3 bucket. + +If you use database as storage, please set `tyk-dev-portal.storage.type` to `db`, and configure `tyk-dev-portal.database` section with database connection details. + +```yaml expandable +tyk-dev-portal: + # Sensitive configuration of Portal could be set using k8s secret + # You can set following fields: + # - DevPortalLicense - Sets LicenseKey for Developer Portal + # - DevPortalStorageConnectionString - Sets connectionString for Developer Portal + # - DevPortalAwsAccessKeyId - Sets AWS S3 Access Key ID + # - DevPortalAwsSecretAccessKey - Sets AWS S3 Secret Access Key + useSecretName: "" + # The hostname to bind the Developer Portal to. + hostName: tyk-dev-portal.org + # Developer Portal license. + license: "" + # Developer portal can be deployed as StatefulSet or as Deployment + kind: StatefulSet + storage: + # User can set the storage type for portal. + # Supported types: fs, s3, db + type: "db" + # Configuration values for using s3 as storage for Tyk Developer Portal + # In case you want to provide the key ID and access key via secrets please + # refer to the existing secret inside the helm chart or the + # .Values.useSecretName field + s3: + awsAccessKeyid: your-access-key + awsSecretAccessKey: your-secret-key + region: sa-east-1 + endpoint: https://s3.sa-east-1.amazonaws.com + bucket: your-portal-bucket + acl: private + presign_urls: true + persistence: + mountExistingPVC: "" + storageClass: "" + accessModes: + - ReadWriteOnce + size: 8Gi + annotations: {} + labels: {} + selector: {} + database: + # This selects the SQL dialect to be used + # The supported values are mysql, postgres and sqlite3 + dialect: "sqlite3" + connectionString: "db/portal.db" + enableLogs: false + maxRetries: 3 + retryDelay: 5000 +``` + +#### Other Configurations + +Other [Developer Portal configurations](/product-stack/tyk-enterprise-developer-portal/deploy/configuration) can be set by using environment variables with `extraEnvs` fields, e.g.: + +```yaml +tyk-dev-portal: + extraEnvs: + - name: PORTAL_LOG_LEVEL + value: debug +``` + +### Tyk Operator Configurations + +Tyk Operator is a licensed component that requires a valid key for operation. +Please refer to the [Tyk Operator Installation Guide](/api-management/automations/operator#install-and-configure-tyk-operator) +for detailed information on the installation and upgrade processes. + +Prior to installing Tyk Operator, ensure that a valid license key is provided by setting `global.license.operator` field in values.yaml file. You can set license key via a Kubernetes secret using `global.secrets.useSecretName` field. The secret should contain a key called `OperatorLicense`. + +In order to enable installing Tyk Operator along-side Tyk Control Plane installation, please set `global.components.operator` +to `true`. + +All other configurations related to Tyk Operator are available under `tyk-operator` section of `values.yaml` file. + +> Tyk Operator needs a cert-manager to be installed. Ensure that cert-manager is installed as described in the +> official documentation: [Installing Tyk Operator](/api-management/automations/operator#install-and-configure-tyk-operator). diff --git a/product-stack/tyk-charts/tyk-data-plane-chart.mdx b/product-stack/tyk-charts/tyk-data-plane-chart.mdx new file mode 100644 index 00000000..6d36ea18 --- /dev/null +++ b/product-stack/tyk-charts/tyk-data-plane-chart.mdx @@ -0,0 +1,510 @@ +--- +title: "Tyk Data Plane Chart" +'og:description': "Install and configure Tyk Data Plane" +sidebarTitle: "Tyk Data Plane Chart" +tags: ['Tyk Data Plane chart', 'Data Plane Chart', 'helm charts', 'helm', 'charts', 'kubernetes', 'k8s'] +--- + +The `tyk-data-plane` chart provides the default deployment of a Tyk data plane for Tyk Self Managed MDCB or Tyk Cloud users. It will deploy the data plane components that remotely connect to a MDCB control plane. + +## What components are deployed with Tyk Data Plane Chart? + +It includes the following components: +- *Tyk Gateway*: An open source Enterprise API Gateway, supporting REST, GraphQL, TCP and gRPC protocols. +- *Tyk Pump*: An analytics purger that moves the data generated by your Tyk gateways to any back-end storage. + +Furthermore, it has all the required modifications to easily connect to Tyk Cloud or Multi Data Center (MDCB) control plane. + +[Supported MDCB versions](/tyk-cloud#tyk-cloud-mdcb-supported-versions) + +By default, this chart installs following components as subcharts on a [Kubernetes](https://kubernetes.io/) cluster using the [Helm](https://helm.sh/) package manager. + +| Component | Enabled by Default? | Flag | +| :--------- | :------------------ | :---- | +|Tyk Gateway |true | n/a | +|Tyk Pump |true | global.components.pump | + +To enable or disable each component, change the corresponding enabled flag. + +Also, you can set the version of each component through `image.tag`. You can find the list of version tags available from [Docker hub](https://hub.docker.com/u/tykio). + +For a quick start guide, please see [deploy hybrid gateway](/tyk-cloud#deploy-hybrid-gateways). + +## Prerequisites + +- [Kubernetes 1.19+](https://kubernetes.io/docs/setup/) +- [Helm 3+](https://helm.sh/docs/intro/install/) +- [Redis](/apim/open-source/installation#configure-legacy-tyk-headless-helm-chart) should already be installed or accessible by the gateway. +- Connection details to remote control plane. See below for how to obtain them from Tyk Cloud or Tyk Control Plane chart. + +## Obtain Remote Control Plane Connection details from Tyk Cloud + +For Tyk Cloud users who want to deploy hybrid data planes, you can easily obtain your remote control plane connection details on Tyk Cloud. + +1. Go to Deployment tab and create a Hybrid data plane configuration. You can also select from an existing one. +2. Copy Key, Org ID, and Data Planes Connection String (MDCB) as `global.remoteControlPlane`'s `userApiKey`, `orgId`, and `connectionString` respectively. + +<img src="/img/tyk-charts/tyk-cloud-deployment.png" alt="tyk-cloud-deployment" /> + +## Obtain Remote Control Plane Connection Details from tyk-control-plane Chart + +For Tyk Self-Managed MDCB users who want to deploy data planes, you can obtain MDCB connection details from the notes +of tyk-control-plane installation output, as listed below. + +``` expandable +=== Tyk Control Plane Details === +Before a worker gateway that is deployed in data plane can connect to MDCB, it is important to set remote control plane options. +If the worker gateway will be deployed via Helm, tyk-data-plane chart helps to facilitate this process. + +1- First obtain required connection details from Tyk MDCB: + + export GROUP_ID=your_group_id # You can use any name for your group. + export USER_API_KEY=$(kubectl get secret --namespace tyk tyk-operator-conf -o jsonpath="{.data.TYK_AUTH}" | base64 --decode) + export ORG_ID=$(kubectl get secret --namespace tyk tyk-operator-conf -o jsonpath="{.data.TYK_ORG}" | base64 --decode) + + +2- Create a Kubernetes Secret based on credentials in data plane's namespace, e.g. `tyk-dp`. + + kubectl create namespace tyk-dp + + kubectl create secret generic tyk-data-plane-details \ + --from-literal "orgId=$ORG_ID" \ + --from-literal "userApiKey=$USER_API_KEY" \ + --from-literal "groupID=$GROUP_ID" \ + --namespace tyk-dp + +3- Refer to this Kubernetes secret (tyk-data-plane-details) while installing worker gateways through +`global.remoteControlPlane.useSecretName` in tyk-data-plane chart. + +4- Set `global.remoteControlPlane.connectionString`, `global.remoteControlPlane.useSSL` and +`global.remoteControlPlane.sslInsecureSkipVerify` in tyk-data-plane chart to access MDCB service. + +If data plane is deployed in the same cluster, it can be accessed via this connection string: + export MDCB_CONNECTIONSTRING="mdcb-svc-tyk-control-plane-tyk-mdcb.tyk.svc:9091" + +If data plane is not deployed in the same cluster as control plane, get the connection string according +to how MDCB service is exposed. +``` + +1. Follow installation output to export USER_API_KEY, ORG_ID, and MDCB_CONNECTIONSTRING. The values can be used to set `global.remoteControlPlane`'s `userApiKey`, `orgId`, and `connectionString` respectively. + +2. Also verify that the SSL connection configuration is set correctly: +```yaml expandable +global: + remoteControlPlane: + # enable/disable ssl + useSSL: false + # Disables SSL certificate verification + sslInsecureSkipVerify: true +``` + + +## Tyk Data Plane Chart Installations +### Installing the Chart + +To install the chart from the Helm repository in namespace `tyk-dp` with the release name `tyk-data-plane`, issue the following commands: + +```bash + helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ + helm repo update + helm show values tyk-helm/tyk-data-plane > values.yaml +``` + +For further documentation relating to *helm* command usage, please refer to the [helm docs](https://helm.sh/docs/helm/). + +At a minimum, modify values.yaml for the following settings: +1. [Set Redis connection details](#set-redis-connection-details-required) + + +Consult the [Configuration](#configuration) section for the available configuration options and modify your local `values.yaml` file accordingly. Then install the chart by issuing the following command below: + +```bash +helm install tyk-data-plane tyk-helm/tyk-data-plane -n tyk-dp --create-namespace -f values.yaml +``` + +### Uninstalling the Chart + +```bash +helm uninstall tyk-data-plane -n tyk-dp +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +### Upgrading Chart + +```bash +helm upgrade tyk-data-plane tyk-helm/tyk-data-plane -n tyk-dp +``` + +## Configuration + +To list all configurable options with detailed comments, issue the following command: + +```bash +helm show values tyk-helm/tyk-data-plane > values.yaml +``` + +You can update any value in your local `values.yaml` file and use `-f [filename]` flag to override default values during installation. +Alternatively, you can use `--set` flag to set it in Tyk installation. + +To configure Tyk components, users can utilize both config files and [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/). Notably, environment variables take precedence over config files. To maintain simplicity and consistency, the Tyk Helm Charts deploy components with an empty config file while setting container environment variables based on user-defined [values](https://helm.sh/docs/chart_best_practices/values/). This approach ensures seamless integration with Kubernetes practices, allowing for efficient management of configurations. For a comprehensive overview of available configurations, please refer to the [configuration documentation](/tyk-oss-gateway/configuration). + + +### Setting Environment Variables +Should any environment variables not be set by the Helm Chart, users can easily add them under the `extraEnvs` section within the charts for further customization. Values set under `extraEnvs` would take precedence over all configurations. + +Example of setting extra environment variable to gateway: +```yaml +tyk-gateway: + gateway: + extraEnvs: + - name: TYK_GW_LOGLEVEL + value: debug +``` + +An example is listed below for setting extra [environment variable using ConfigMap data](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#define-container-environment-variables-using-configmap-data), using gateway: +```yaml expandable +tyk-gateway: + gateway: + extraEnvs: + - name: CONFIG_USERNAME + valueFrom: + configMapKeyRef: + name: backend-user + key: backend-username +``` + +An example is listed below for setting extra [environment variable using secret data](https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#define-container-environment-variables-using-secret-data), using gateway: +```yaml expandable +tyk-gateway: + gateway: + extraEnvs: + - name: SECRET_USERNAME + valueFrom: + secretKeyRef: + name: backend-user + key: backend-username +``` + +In the above example, an extra environment variable `SECRET_USERNAME` will be added to the Gateway container, with a value of `backend-username` associated with the secret `backend-user`. It is useful if you want to access secret data from [Tyk Gateway configuration file (tyk.conf) or API definitions](/tyk-configuration-reference/kv-store). + +### Set Redis Connection Details (Required) + +Tyk uses Redis for distributed rate-limiting and token storage. You may use the [Bitnami chart](https://github.com/bitnami/charts/tree/main/bitnami/redis) to install or Tyk's `simple-redis` chart for POC purpose. + +Set the following values after installing Redis: + +| Name | Description | +| :------ | :------------- | +| `global.redis.addrs` | Redis addresses | +| `global.redis.pass` | Redis password in plain text | +| `global.redis.passSecret.name` | If global.redis.pass is not provided, you can store it in a secret and provide the secret name here | +| `global.redis.passSecret.keyName` | key name to retrieve Redis password from the secret | + +***Recommended: via *Bitnami* chart*** + +For Redis you can use these rather excellent charts provided by [Bitnami](https://github.com/bitnami/charts/tree/main/bitnami/redis). +Copy the following commands to add it: + +```bash +helm upgrade tyk-redis oci://registry-1.docker.io/bitnamicharts/redis -n tyk --create-namespace --install --version 19.0.2 +``` + + +<Note> +Please make sure you are installing Redis versions that are supported by Tyk. Please refer to Tyk docs to get list of [supported versions](/tyk-self-managed#redis). +</Note> + + +Follow the notes from the installation output to get connection details and password. + +``` + Redis(TM) can be accessed on the following DNS names from within your cluster: + + tyk-redis-master.tyk.svc.cluster.local for read/write operations (port 6379) + tyk-redis-replicas.tyk.svc.cluster.local for read-only operations (port 6379) + + export REDIS_PASSWORD=$(kubectl get secret --namespace tyk tyk-redis -o jsonpath="{.data.redis-password}" | base64 --decode) +``` + +The Redis address as set by Bitnami is `tyk-redis-master.tyk.svc.cluster.local:6379` + +You can reference the password secret generated by Bitnami chart by `--set global.redis.passSecret.name=tyk-redis` and `--set global.redis.passSecret.keyName=redis-password`, or just set `--set global.redis.pass=$REDIS_PASSWORD`. + +```bash +helm install --set global.redis.passSecret.name=tyk-redis --set global.redis.passSecret.keyName=redis-password tyk-data-plane tyk-helm/tyk-data-plane +``` + +or + +```bash +export REDIS_PASSWORD=$(kubectl get secret --namespace tyk tyk-redis -o jsonpath="{.data.redis-password}" | base64 --decode) + +helm install --set global.redis.pass=$REDIS_PASSWORD tyk-data-plane tyk-helm/tyk-data-plane +``` + +***Evaluation only: via *simple-redis* chart*** + +Another option for Redis, to get started quickly, is to use our [simple-redis](https://artifacthub.io/packages/helm/tyk-helm/simple-redis) chart. + + +<Warning> +Please note that these provided charts must never be used in production or for anything +but a quick start evaluation only. Use [Bitnami Redis](https://github.com/bitnami/charts/tree/main/bitnami/redis) or [Official Redis installation guides](https://redis.io/docs/install/) in any other case. +We provide this chart, so you can quickly deploy *Tyk gateway*, but it is not meant for long term storage of data. +</Warning> + + +```bash +helm install redis tyk-helm/simple-redis -n tyk +``` + +The Tyk Helm Chart can connect to `simple-redis` in the same namespace by default. You do not need to set Redis address and password in `values.yaml`. + +### Protect Confidential Fields with Kubernetes Secrets + +In the `values.yaml` file, some fields are considered confidential, such as `APISecret`, connection strings, etc. +Declaring values for such fields as plain text might not be desired for all use cases. Instead, for certain fields, Kubernetes secrets can be referenced, and the chart will [define container environment variables using secret data](https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#define-container-environment-variables-using-secret-data). + +This section describes how to use Kubernetes secrets to declare confidential fields. + +***APISecret*** + +The `global.secrets.APISecret` field configures a [header value](/tyk-oss-gateway/configuration#secret) used in every interaction with Tyk Gateway API. + +It can be configured via `global.secrets.APISecret` as a plain text or Kubernetes secret which includes `APISecret` key in it. Then, this secret must be referenced via `global.secrets.useSecretName`. + +```yaml +global: + secrets: + APISecret: CHANGEME + useSecretName: "mysecret" # where mysecret includes `APISecret` key with the desired value. +``` + +***Remote Control Plane Configuration*** + +All configurations regarding remote control plane (`orgId`, `userApiKey`, and `groupID`) can be set via +Kubernetes secret. + +Instead of explicitly setting them in the values file, just create a Kubernetes secret including `orgId`, `userApiKey` and `groupID` keys and refer to it in `global.remoteControlPlane.useSecretName`. + +1. Create a secret that contains `orgId`, `userApiKey` and `groupID` keys in it: + +```bash +kubectl create secret generic foo-secret -n tyk --from-literal=orgId=[ORGID] --from-literal=userApiKey=[APIKEY]--from-literal=groupID=[GROUPID] +``` + +2. Refer to it in `global.remoteControlPlane.useSecretName`. + +```yaml +global: + remoteControlPlane: + useSecretName: "foo-secret" +``` + +***Redis Password*** + +The Redis password can also be provided via a secret. Store Redis password in Kubernetes secret and refer to this secret +via `global.redis.passSecret.name` and `global.redis.passSecret.keyName` field, as follows: + +```yaml +global: + redis: + passSecret: + name: "yourSecret" + keyName: "redisPassKey" +``` + +### Tyk MDCB Synchroniser (Optional) + +If control plane MDCB has enabled [Synchroniser feature](/api-management/mdcb#synchroniser-feature-with-mdcb), the following fields should be set accordingly: + +```yaml +global: + mdcbSynchronizer: + enabled: true + keySpaceSyncInterval: 10 +``` + +### Gateway Configurations + +Configure below inside `tyk-gateway` section. + +#### Update Tyk Gateway Version +Set version of gateway at `tyk-gateway.gateway.image.tag`. You can find the list of version tags available from [Docker hub](https://hub.docker.com/r/tykio/tyk-gateway/tags). Please check [Tyk Release notes](/developer-support/release-notes/gateway) carefully while upgrading or downgrading. + +#### Enabling TLS + +*Enable TLS* + +We have provided an easy way to enable TLS via the `global.tls.gateway` flag. Setting this value to true will +automatically enable TLS using the certificate provided under tyk-gateway/certs/. + +*Configure TLS secret* + +If you want to use your own key/cert pair, please follow the following steps: +1. Create a TLS secret using your cert and key pair. +2. Set `global.tls.gateway` to true. +3. Set `tyk-gateway.gateway.tls.useDefaultTykCertificate` to false. +4. Set `tyk-gateway.gateway.tls.secretName` to the name of the newly created secret. + +*Add Custom CA Certificates* + +To add your custom Certificate Authority(CA) to your containers, you can mount your CA certificate directly into /etc/ssl/certs folder. + +```yaml expandable + extraVolumes: + - name: self-signed-ca + secret: + secretName: self-signed-ca-secret + extraVolumeMounts: + - name: self-signed-ca + mountPath: "/etc/ssl/certs/myCA.pem" + subPath: myCA.pem +``` + +#### Enabling gateway autoscaling +You can enable autoscaling of the gateway by `--set tyk-gateway.gateway.autoscaling.enabled=true`. By default, it will enable the `Horizontal Pod Autoscaler` resource with target average CPU utilization at 60%, scaling between 1 and 3 instances. To customize those values you can modify the `tyk-gateway` section of `values.yaml` as shown below: + +```yaml expandable +tyk-gateway: + gateway: + autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 30 +``` + +Built-in rules include `tyk-gateway.gateway.autoscaling.averageCpuUtilization` for CPU utilization (set by default at 60%) and `tyk-gateway.gateway.autoscaling.averageMemoryUtilization` for memory (disabled by default). In addition to that you can define rules for custom metrics using `tyk-gateway.gateway.autoscaling.autoscalingTemplate` list: + +```yaml expandable +tyk-gateway: + gateway: + autoscaling: + autoscalingTemplate: + - type: Pods + pods: + metric: + name: nginx_ingress_controller_nginx_process_requests_total + target: + type: AverageValue + averageValue: 10000m +``` + +#### Accessing Gateway + +*Service port* + +Default service port of gateway is 8080. You can change this at `global.servicePorts.gateway`. + +*Ingress* + +An Ingress resource is created if `tyk-gateway.gateway.ingress.enabled` is set to true. + +```yaml expandable + ingress: + # if enabled, creates an ingress resource for the gateway + enabled: true + + # specify ingress controller class name + className: "" + + # annotations for ingress + annotations: {} + + # ingress rules + hosts: + - host: tyk-gw.local + paths: + - path: / + pathType: ImplementationSpecific + + # tls configuration for ingress + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + tls: [] +``` + +*Control Port* + +Set `tyk-gateway.gateway.control.enabled` to true will allow you to run the [Gateway API](/tyk-gateway-api) on a separate port and protect it behind a firewall if needed. + +#### Sharding + +Configure the gateways to load APIs with specific tags only by enabling `tyk-gateway.gateway.sharding.enabled`, and set `tags` to comma separated lists of matching tags. + +```yaml expandable + # Sharding gateway allows you to selectively load APIs to specific gateways. + # If enabled make sure you have at least one gateway that is not sharded. + # Also be sure to match API segmentation tags with the tags selected below. + sharding: + enabled: true + tags: "edge,dc1,product" +``` + +#### OpenTelemetry +To enable OpenTelemetry for Gateway set `gateway.opentelemetry.enabled` flag to true. It is disabled by default. + +You can also configure connection settings for it's exporter. By default `grpc` exporter is enabled on `localhost:4317` endpoint. + + To enable TLS settings for the exporter, you can set `gateway.opentelemetry.tls.enabled` to true. + +#### Liveness and readiness probes +Gateway liveness probes can be customised via `gateway.livenessProbe` field. All fields from PodLivenessProbe object can be added here. If set to empty or nil, the default health check on /health will be performed. + +Gateway readiness probes can be customised via `gateway.readinessProbe` field. All fields from PodReadinessProbe object can be added here. If set to empty or nil, the default health check on /health will be performed. + +For further details for configuring Tyk Gateway, please consult the [Tyk Gateway Configuration Options](/tyk-oss-gateway/configuration) guide. + +### Pump Configurations + +To enable Pump, set `global.components.pump` to true, and configure as detailed below inside `tyk-pump` section. + +| Pump | Configuration | +| :--------------------------- | :------------------------------------------------------------------------------------------------------------ | +| Prometheus Pump (Default) | Add `prometheus` to `tyk-pump.pump.backend`, and add connection details for prometheus under `tyk-pump.pump.prometheusPump`. | +| Hybrid Pump (Default) | Add `hybrid` to `tyk-pump.pump.backend`, and add remoteControlPlane details under `global.remoteControlPlane`. | +| Other Pumps | Add the required environment variables in `tyk-pump.pump.extraEnvs` | + +#### Prometheus Pump +Add `prometheus` to `tyk-pump.pump.backend`, and add connection details for prometheus under `tyk-pump.pump.prometheusPump`. + +We also support monitoring using Prometheus Operator. All you have to do is set `tyk-pump.pump.prometheusPump.prometheusOperator.enabled` to true. +This will create a *PodMonitor* resource for your Pump instance. + +#### Hybrid Pump +Add `hybrid` to `tyk-pump.pump.backend`, and add remoteControlPlane details under `global.remoteControlPlane`. + +```yaml expandable + # Set remoteControlPlane connection details if you want to configure hybrid pump. + remoteControlPlane: + # connection string used to connect to an MDCB deployment. For Tyk Cloud users, you can get it from Tyk Cloud Console and retrieve the MDCB connection string. + connectionString: "" + # orgID of your dashboard user + orgId: "" + # API key of your dashboard user + userApiKey: "" + # needed in case you want to have multiple data-planes connected to the same redis instance + groupID: "" + # enable/disable ssl + useSSL: true + # Disables SSL certificate verification + sslInsecureSkipVerify: true +``` + +```yaml expandable + # hybridPump configures Tyk Pump to forward Tyk metrics to a Tyk Control Plane. + # Please add "hybrid" to .Values.pump.backend in order to enable Hybrid Pump. + hybridPump: + # Specify the frequency of the aggregation in minutes or simply turn it on by setting it to true + enableAggregateAnalytics: true + # Hybrid pump RPC calls timeout in seconds. + callTimeout: 10 + # Hybrid pump connection pool size. + poolSize: 5 +``` + +#### Other Pumps +To setup other backends for Pump, refer to this [document](https://github.com/TykTechnologies/tyk-pump/blob/master/README.md#pumps--back-ends-supported) and add the required environment variables in `tyk-pump.pump.extraEnvs` + diff --git a/product-stack/tyk-charts/tyk-oss-chart.mdx b/product-stack/tyk-charts/tyk-oss-chart.mdx new file mode 100644 index 00000000..cb257133 --- /dev/null +++ b/product-stack/tyk-charts/tyk-oss-chart.mdx @@ -0,0 +1,520 @@ +--- +title: "Tyk Open Source Chart" +'og:description': "Explains the Tyk Open Source Chart" +sidebarTitle: "Tyk Open Source Chart" +tags: ['Tyk Open Source chart', 'Open Source Chart', 'helm charts', 'helm', 'charts', 'kubernetes', 'k8s'] +--- + +`tyk-oss` provides the default deployment of the Tyk Open Source stack. + +It includes the following components: +- *Tyk Gateway*: An open source Enterprise API Gateway, supporting REST, GraphQL, TCP and gRPC protocols. +- *Tyk Pump*: An analytics purger that moves the data generated by your Tyk gateways to any back-end storage. + +## What components are deployed with Tyk OSS Chart? + +By default, this chart installs following components as subcharts on a [Kubernetes](https://kubernetes.io/) cluster using the [Helm](https://helm.sh/) package manager. + +| Component | Enabled by Default | Flag | +| :-------------- | :-------------------- | :---------------------------- | +| Tyk Gateway | true | n/a | +| Tyk Pump | false | global.components.pump | +| Tyk Operator | false | global.components.operator | + +To enable or disable each component, change the corresponding enabled flag. + +Also, you can set the version of each component through `image.tag`. You could find the list of version tags available from [Docker hub](https://hub.docker.com/u/tykio). + +For quick start guide, please see [Quick Start with Tyk OSS Helm Chart](/apim/open-source/installation#quick-start-with-helm-chart). + +## Prerequisites + +* [Kubernetes 1.19+](https://kubernetes.io/docs/setup/) +* [Helm 3+](https://helm.sh/docs/intro/install/) +* [Redis](/tyk-configuration-reference/redis-cluster-sentinel) should already be installed or accessible by the gateway. + +## Tyk OSS Installations +### Installing the Chart + +To install the chart from the Helm repository in namespace `tyk` with the release name `tyk-oss`: + +First, add our Helm repo and get default values: + +```bash + helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ + helm repo update + helm show values tyk-helm/tyk-oss > values.yaml +``` + +For further documentation relating to *helm* command usage, please refer to the [helm docs](https://helm.sh/docs/helm/). + +See [Configuration](#configuration) section for the available config options and modify your local `values.yaml` file accordingly. Then install the chart: + +```bash +helm install tyk-oss tyk-helm/tyk-oss -n tyk --create-namespace -f values.yaml +``` + +### Uninstalling the Chart + +```bash +helm uninstall tyk-oss -n tyk +``` + +This removes all the Kubernetes components associated with the chart and deletes the release. + +### Upgrading Chart + +```bash +helm upgrade tyk-oss tyk-helm/tyk-oss -n tyk -f values.yaml +``` + +## Configuration + +To list all configurable options with detailed comments: + +```bash +helm show values tyk-helm/tyk-oss > values.yaml +``` + +You can update any value in your local `values.yaml` file and use `-f [filename]` flag to override default values during installation. +Alternatively, you can use `--set` flag to set it in Tyk installation. + +To configure Tyk components, users can utilize both config files and [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/). Notably, environment variables take precedence over config files. To maintain simplicity and consistency, the Tyk Helm Charts deploy components with an empty config file while setting container environment variables based on user-defined [values](https://helm.sh/docs/chart_best_practices/values/). This approach ensures seamless integration with Kubernetes practices, allowing for efficient management of configurations. For a comprehensive overview of available configurations, please refer to the [configuration documentation](/tyk-oss-gateway/configuration). + +### Setting Environment Variables +Should any environment variables not be set by the Helm Chart, users can easily add them under the `extraEnvs` section within the charts for further customization. Values set under `extraEnvs` would take precedence over all configurations. + +Example of setting extra environment variable to gateway: +```yaml +tyk-gateway: + gateway: + extraEnvs: + - name: TYK_GW_LOGLEVEL + value: debug +``` + +An example is listed below for setting extra [environment variable using ConfigMap data](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#define-container-environment-variables-using-configmap-data), using gateway: +```yaml expandable +tyk-gateway: + gateway: + extraEnvs: + - name: CONFIG_USERNAME + valueFrom: + configMapKeyRef: + name: backend-user + key: backend-username +``` + +An example is listed below for setting extra [environment variable using secret data](https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#define-container-environment-variables-using-secret-data), using gateway: +```yaml expandable +tyk-gateway: + gateway: + extraEnvs: + - name: SECRET_USERNAME + valueFrom: + secretKeyRef: + name: backend-user + key: backend-username +``` + +In the above example, an extra environment variable `SECRET_USERNAME` will be added to the Gateway container, with a value of `backend-username` associated with the secret `backend-user`. It is useful if you want to access secret data from [Tyk Gateway configuration file (tyk.conf) or API definitions](/tyk-configuration-reference/kv-store). + +### Set Redis Connection Details (Required) + +Tyk uses Redis for distributed rate-limiting and token storage. You may use the [Bitnami chart](https://github.com/bitnami/charts/tree/main/bitnami/redis) to install or Tyk's `simple-redis` chart for POC purpose. + +Set the following values after installing Redis: + +| Name | Description | +| :------ | :------------- | +| `global.redis.addrs` | Redis addresses | +| `global.redis.pass` | Redis password in plain text | +| `global.redis.passSecret.name` | If global.redis.pass is not provided, you can store it in a secret and provide the secret name here | +| `global.redis.passSecret.keyName` | key name to retrieve Redis password from the secret | + +***Recommended: via *Bitnami* chart*** + +For Redis you can use these rather excellent charts provided by [Bitnami](https://github.com/bitnami/charts/tree/main/bitnami/redis). +Copy the following commands to add it: + +```bash +helm upgrade tyk-redis oci://registry-1.docker.io/bitnamicharts/redis -n tyk --create-namespace --install --version 19.0.2 +``` + + +<Note> +Please make sure you are installing Redis versions that are supported by Tyk. Please refer to Tyk docs to get list of [supported versions](/tyk-self-managed#redis). +</Note> + + +Follow the notes from the installation output to get connection details and password. + +``` + Redis(TM) can be accessed on the following DNS names from within your cluster: + + tyk-redis-master.tyk.svc.cluster.local for read/write operations (port 6379) + tyk-redis-replicas.tyk.svc.cluster.local for read-only operations (port 6379) + + export REDIS_PASSWORD=$(kubectl get secret --namespace tyk tyk-redis -o jsonpath="{.data.redis-password}" | base64 --decode) +``` + +The Redis address as set by Bitnami is `tyk-redis-master.tyk.svc.cluster.local:6379` + +You can reference the password secret generated by Bitnami chart by `--set global.redis.passSecret.name=tyk-redis` and `--set global.redis.passSecret.keyName=redis-password`, or just set `global.redis.pass=$REDIS_PASSWORD` + +***Evaluation only: via *simple-redis* chart*** + +Another option for Redis, to get started quickly, is to use our [simple-redis](https://artifacthub.io/packages/helm/tyk-helm/simple-redis) chart. + + +<Warning> +Please note that these provided charts must never be used in production or for anything +but a quick start evaluation only. Use [Bitnami Redis](https://github.com/bitnami/charts/tree/main/bitnami/redis) or [Official Redis installation guides](https://redis.io/docs/install/) in any other case. +We provide this chart, so you can quickly deploy *Tyk gateway*, but it is not meant for long term storage of data. +</Warning> + + +```bash +helm install redis tyk-helm/simple-redis -n tyk +``` + +The Tyk Helm Chart can connect to `simple-redis` in the same namespace by default. You do not need to set Redis address and password in `values.yaml`. + +### Protect Confidential Fields with Kubernetes Secrets + +In the `values.yaml` file, some fields are considered confidential, such as `APISecret`, connection strings, etc. +Declaring values for such fields as plain text might not be desired for all use cases. Instead, for certain fields, Kubernetes secrets can be referenced and the chart will [define container environment variables using Secret data](https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#define-container-environment-variables-using-secret-data). + +This section describes how to use Kubernetes secrets to declare confidential fields. + +***APISecret*** + +The `global.secrets.APISecret` field configures a [header value](/tyk-oss-gateway/configuration#secret) used in every interaction with Tyk Gateway API. + +It can be configured via `global.secrets.APISecret` as a plain text or Kubernetes secret which includes `APISecret` key in it. Then, this secret must be referenced via `global.secrets.useSecretName`. + +```yaml +global: + secrets: + APISecret: CHANGEME + useSecretName: "mysecret" # where mysecret includes `APISecret` key with the desired value. +``` + +***Redis Password*** + +Redis password can also be provided via a secret. Store Redis password in Kubernetes secret and refer to this secret +via `global.redis.passSecret.name` and `global.redis.passSecret.keyName` field, as follows: + +```yaml +global: + redis: + passSecret: + name: "yourSecret" + keyName: "redisPassKey" +``` + +**_Tyk Operator License_** + +It can be configured via `global.license.operator` as a plain text or Kubernetes secret which includes `OperatorLicense` key in it. Then, this secret must be referenced via `global.secrets.useSecretName`. + +Note: If you are using `global.secrets.useSecretName`, you must configure the operator license in the referenced Kubernetes secret. `global.license.operator` will not be used in this case. + +### Create a Kubernetes Secret for Tyk Operator + +When `operatorSecret.enabled` is set to `true`, `tyk-oss` chart will create a Kubernetes Secret named `tyk-operator-conf` in the same namespace. It can be used by Tyk Operator to connect to Gateway to manage Tyk API resources. + +```yaml expandable +# operatorSecret controls if a secret needed to connect to Operator will be created +operatorSecret: + # enabled if set to true creates secret + enabled: true + # OSS doesn't have concept of OrgID. But we need to support some features (eg. basic auth key) in OSS + # You can set it to any arbitary value + orgID: "orgid" +``` + +### Gateway Configurations + +Configure below inside `tyk-gateway` section. + +#### Update Tyk Gateway Version +Set version of gateway at `tyk-gateway.gateway.image.tag`. You can find the list of version tags available from [Docker hub](https://hub.docker.com/u/tykio). Please check [Tyk Release notes](/developer-support/release-notes/gateway) carefully while upgrading or downgrading. + +#### Enabling TLS + +*Enable TLS* + +We have provided an easy way to enable TLS via the `global.tls.gateway` flag. Setting this value to true will +automatically enable TLS using the certificate provided under tyk-gateway/certs/. + +*Configure TLS secret* + +If you want to use your own key/cert pair, please follow the following steps: +1. Create a TLS secret using your cert and key pair. +2. Set `global.tls.gateway` to true. +3. Set `tyk-gateway.gateway.tls.useDefaultTykCertificate` to false. +4. Set `tyk-gateway.gateway.tls.secretName` to the name of the newly created secret. + +*Add Custom Certificates* + +To add your custom Certificate Authority(CA) to your containers, you can mount your CA certificate directly into /etc/ssl/certs folder. + +```yaml expandable + extraVolumes: + - name: self-signed-ca + secret: + secretName: self-signed-ca-secret + extraVolumeMounts: + - name: self-signed-ca + mountPath: "/etc/ssl/certs/myCA.pem" + subPath: myCA.pem +``` + +#### Enabling gateway autoscaling +You can enable autoscaling of the gateway by `--set tyk-gateway.gateway.autoscaling.enabled=true`. By default, it will enable the `Horizontal Pod Autoscaler` resource with target average CPU utilization at 60%, scaling between 1 and 3 instances. To customize those values you can modify the tyk-gateway section of `values.yaml`, as shown below: + +```yaml expandable +tyk-gateway: + gateway: + autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 30 +``` + +Built-in rules include `tyk-gateway.gateway.autoscaling.averageCpuUtilization` for CPU utilization (set by default at 60%) and `tyk-gateway.gateway.autoscaling.averageMemoryUtilization` for memory (disabled by default). In addition to that you can define rules for custom metrics using `tyk-gateway.gateway.autoscaling.autoscalingTemplate` list: + +```yaml expandable +tyk-gateway: + gateway: + autoscaling: + autoscalingTemplate: + - type: Pods + pods: + metric: + name: nginx_ingress_controller_nginx_process_requests_total + target: + type: AverageValue + averageValue: 10000m +``` + +#### Accessing Gateway + +*Service port* + +Default service port of gateway is 8080. You can change this at `global.servicePorts.gateway`. + +*Ingress* + +An Ingress resource is created if `tyk-gateway.gateway.ingress.enabled` is set to true. + +```yaml expandable +tyk-gateway: + gateway: + ingress: + # if enabled, creates an ingress resource for the gateway + enabled: true + + # specify ingress controller class name + className: "" + + # annotations for ingress + annotations: {} + + # ingress rules + hosts: + - host: tyk-gw.local + paths: + - path: / + pathType: ImplementationSpecific + + # tls configuration for ingress + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + tls: [] +``` + +*Control Port* + +Set `tyk-gateway.gateway.control.enabled` to true will allow you to run the [Gateway API](/tyk-gateway-api) on a separate port and protect it behind a firewall if needed. + +#### Mounting APIs, Policies, and Middleware + +By default, the Gateway stores API configurations at `/mnt/tyk-gateway/apps` inside the Gateway container. There are a a few challenges: +- Multiple gateways do not share app configs +- The configuration is not persistent. i.e. it will not be retained whenever a pod restarts. + +The same applies to security policies and middleware too, which are stored at `/mnt/tyk-gateway/policies` and `/mnt/tyk-gateway/middleware` respectively. + +This can be solved by instantiating a Persistent Volume as shared storage for the gateway instances. As each gateway is reloaded, they would get the API configurations from the same storage, solving the synchronisation issue between gateways. Also, the storage is persistent and can be designed to be resilient to cluster failure, thus your API configurations can be maintained after pod restart. + +You can configure persistent volume for APIs, Policies, and middleware using `extraVolumes` and `extraVolumeMounts`: + +```yaml expandable + extraVolumes: + - name: tyk-app-storage + persistentVolumeClaim: + claimName: tyk-app-claim + - name: tyk-policies-storage + persistentVolumeClaim: + claimName: tyk-policies-claim + - name: tyk-middleware-storage + persistentVolumeClaim: + claimName: tyk-middleware-claim + + extraVolumeMounts: + - name: tyk-app-storage + mountPath: /mnt/tyk-gateway/apps + - name: tyk-policies-storage + mountPath: /mnt/tyk-gateway/policies + - name: tyk-middleware-storage + mountPath: /mnt/tyk-gateway/middleware +``` + +For further details for configuring Tyk Gateway, consult the [Tyk Gateway Configuration Options](/tyk-oss-gateway/configuration) guide. + +#### OpenTelemetry +To enable OpenTelemetry for Gateway set `gateway.opentelemetry.enabled` flag to true. It is disabled by default. + +You can also configure connection settings for it's exporter. By default `grpc` exporter is enabled on `localhost:4317` endpoint. + + To enable TLS settings for the exporter, you can set `gateway.opentelemetry.tls.enabled` to true. + +#### Liveness and readiness probes +Gateway liveness probes can be customised via `gateway.livenessProbe` field. All fields from PodLivenessProbe object can be added here. If set to empty or nil, the default health check on /health will be performed. + +Gateway readiness probes can be customised via `gateway.readinessProbe` field. All fields from PodReadinessProbe object can be added here. If set to empty or nil, the default health check on /health will be performed. + +### Pump Configurations + +To enable Pump, set `global.components.pump` to true, and configure below inside `tyk-pump` section. + +| Pump | Configuration | +| :--------------------------- | :------------------------------------------------------------------------------------------------------------ | +| Prometheus Pump (Default) | Add `prometheus` to `tyk-pump.pump.backend`, and add connection details for prometheus under `tyk-pump.pump.prometheusPump`. | +| Mongo Pump | Add `mongo` to `tyk-pump.pump.backend`, and add connection details for mongo under `global.mongo`. | +| SQL Pump | Add `postgres` to `tyk-pump.pump.backend`, and add connection details for postgres under `global.postgres`. | +| Uptime Pump | Set `tyk-pump.pump.uptimePumpBackend` to `'mongo'` or `'postgres'` | +| Other Pumps | Add the required environment variables in `tyk-pump.pump.extraEnvs` | + +#### Prometheus Pump +Add `prometheus` to `tyk-pump.pump.backend`, and add connection details for Prometheus under `tyk-pump.pump.prometheusPump`. + +We also support monitoring using Prometheus Operator. All you have to do is set `tyk-pump.pump.prometheusPump.prometheusOperator.enabled` to true. +This will create a *PodMonitor* resource for your Pump instance. + +See [Configure Tyk Pump to expose analytics data to Prometheus](/api-management/tyk-pump#setup-prometheus-pump) for a step-by-step guide on setting up Prometheus Pump on Kubernetes. + +#### Mongo Pump +If you are using the MongoDB pumps in the tyk-oss installation you will require MongoDB installed for that as well. + +To install MongoDB you can use these rather excellent charts provided by Bitnami: + +```bash +helm install tyk-mongo bitnami/mongodb --set "replicaSet.enabled=true" -n tyk --version 15.1.3 +``` + + +<Note> +Bitnami MongoDB image is not supported on darwin/arm64 architecture. +</Note> + + + +<Note> +Please make sure you are installing MongoDB versions that are supported by Tyk. Please refer to Tyk docs to get list of [supported versions](/api-management/dashboard-configuration#supported-database). +</Note> + + +Follow notes from the installation output to get connection details and update them in the `values.yaml` file. + +*Important Note regarding MongoDB:* This helm chart enables the PodDisruptionBudget for MongoDB with an arbiter replica-count of 1. If you intend to perform system maintenance on the node where the MongoDB pod is running and this maintenance requires for the node to be drained, this action will be prevented due the replica count being 1. Increase the replica count in the helm chart deployment to a minimum of 2 to remedy this issue. + +Add the following configuration under the `global` section in `values.yaml`: + +```yaml expandable +global: + # Set mongo connection details if you want to configure mongo pump. + mongo: + # The mongoURL value will allow you to set your MongoDB address. + # Default value: mongodb://mongo.{{ .Release.Namespace }}.svc:27017/tyk_analytics + # mongoURL: mongodb://mongo.tyk.svc:27017/tyk_analytics + # If your MongoDB has a password you can add the username and password to the url + # mongoURL: mongodb://root:pass@tyk-mongo-mongodb.tyk.svc:27017/tyk_analytics?authSource=admin + mongoURL: <MongoDB address> + + # mongo-go driver is supported for Tyk 5.0.2+. + # We recommend using the `mongo-go` driver if you are using MongoDB 4.4.x+. + # For MongoDB versions prior to 4.4, please use the `mgo` driver. + # Since Tyk 5.3 the default driver is mongo-go. + driver: mongo-go + + # Enables SSL for MongoDB connection. MongoDB instance will have to support that. + # Default value: false + # useSSL: false +``` + +#### SQL Pump +If you are using the SQL Pumps in the tyk-oss installation you will require PostgreSQL installed for that as well. + +To install PostgreSQL you can use these rather excellent charts provided by Bitnami: + +```bash +helm install tyk-postgres bitnami/postgresql --set "auth.database=tyk_analytics" -n tyk --version 12.12.10 +``` + + +<Note> +Please make sure you are installing PostgreSQL versions that are supported by Tyk. Please refer to Tyk docs to get list of [supported versions](/api-management/dashboard-configuration#supported-database). +</Note> + + +Follow the notes from the installation output to get connection details and update them in `values.yaml` file. + +Add the following configuration under the `global` section in `values.yaml`: + +```yaml expandable +global: + # Postgres connection string parameters. + postgres: + # host corresponds to the host name of postgres + host: tyk-postgres-postgresql.tyk.svc + # port corresponds to the port of postgres + port: 5432 + # user corresponds to the user of postgres + user: postgres + # password corresponds to the password of the given postgres user in selected database + password: + # database corresponds to the database to be used in postgres + database: tyk_analytics + # sslmode corresponds to if postgres runs in sslmode (https) + sslmode: disable + # Connection string can also be set using a secret. Provide the name of the secret and key below. + # connectionStringSecret: + # name: "" + # keyName: "" +``` + +#### Uptime Pump +Uptime Pump can be configured by setting `tyk-pump.pump.uptimePumpBackend` in values.yaml file. It supports the following values +1. mongo: Used to set Mongo Pump for uptime analytics. Mongo Pump should be enabled. +2. postgres: Used to set Postgres Pump for uptime analytics. Postgres Pump should be enabled. +3. empty: Used to disable uptime analytics. + +#### Other Pumps +To setup other backends for pump, refer to this [document](https://github.com/TykTechnologies/tyk-pump/blob/master/README.md#pumps--back-ends-supported) and add the required environment variables in `tyk-pump.pump.extraEnvs` + +### Tyk Operator Configurations + +Tyk Operator is a licensed component that requires a valid key for operation. +Please refer to the [Tyk Operator Installation Guide](/api-management/automations/operator#install-and-configure-tyk-operator) +for detailed information on the installation and upgrade processes. + +Prior to installing Tyk Operator, ensure that a valid license key is provided by setting `global.license.operator` field in values.yaml file. You can set license key via a Kubernetes secret using `global.secrets.useSecretName` field. The secret should contain a key called `OperatorLicense`. + +In order to enable installing Tyk Operator along-side Tyk OSS installation, please set `global.components.operator` to `true`. + +All other configurations related to Tyk Operator are available under `tyk-operator` section of `values.yaml` file. + +> Tyk Operator needs a cert-manager to be installed. Ensure that cert-manager is installed as described in the official documentation: [Installing Tyk Operator](/api-management/automations/operator#install-and-configure-tyk-operator). diff --git a/product-stack/tyk-charts/tyk-stack-chart.mdx b/product-stack/tyk-charts/tyk-stack-chart.mdx new file mode 100644 index 00000000..b4d7a4e3 --- /dev/null +++ b/product-stack/tyk-charts/tyk-stack-chart.mdx @@ -0,0 +1,950 @@ +--- +title: "Tyk Stack Chart" +'og:description': "Install and configure Tyk Stack Chart" +sidebarTitle: "Tyk Stack Chart" +tags: ['Tyk Stack chart', 'Tyk Stack Chart', 'helm charts', 'helm', 'charts', 'kubernetes', 'k8s'] +--- + +The `tyk-stack` chart provides the default deployment of Tyk Self Managed on a Kubernetes cluster. It will deploy all required Tyk components with the settings provided in the values.yaml file. + +## What components are deployed with Tyk Stack Chart? + +It includes: +- Tyk Gateway, an Open Source Enterprise API Gateway (supporting REST, GraphQL, TCP and gRPC protocols). +- Tyk Pump, an analytics purger that moves the data generated by your Tyk nodes to any back-end. Furthermore, it has all the required modifications to easily connect to Tyk Cloud or Multi Data Center (MDCB) control plane. +- Tyk Dashboard, a license based component that provides a graphical management interface and analytics platform for Tyk. +- Tyk Developer Portal, a full-fledged CMS-like system for API providers to publish, monetize and drive the adoption of APIs. + +By default, this chart installs the following components as sub-charts on a [Kubernetes](https://kubernetes.io/) cluster using the [Helm](https://helm.sh/) package manager. + +| Component | Enabled by Default | Flag | +| :--------------------------------- | :-------------------- | :----------------------------- | +| Tyk Gateway | true | n/a | +| Tyk Dashboard | true | n/a | +| Tyk Pump | false | global.components.pump | +| Tyk Enterprise Developer Portal | false | global.components.devPortal | +| Tyk Operator | false | global.components.operator | + +To enable or disable each component, change the corresponding enabled flag. + +Also, you can set the version of each component through `image.tag`. You can find the list of version tags available from [Docker hub](https://hub.docker.com/u/tykio). + +For quick start guide, please see [Quick Start with Helm Chart and PostgreSQL](/tyk-self-managed/install#install-tyk-stack-with-helm-chart-postgresql) or [Quick Start with Helm Chart and MongoDB](/tyk-self-managed/install#install-tyk-stack-with-helm-chart-mongodb). + +## Prerequisites + +* [Kubernetes 1.19+](https://kubernetes.io/docs/setup/) +* [Helm 3+](https://helm.sh/docs/intro/install/) +* [Redis](/tyk-configuration-reference/redis-cluster-sentinel) should already be installed or accessible by the gateway and dashboard. +* [MongoDB](https://www.mongodb.com) or [PostgreSQL](https://www.postgresql.org) should already be installed or accessible by dashboard. Please consult the list of [supported versions](/api-management/dashboard-configuration#supported-database) that are compatible with Tyk. + + + + <Note> + If you want to enable Tyk Developer Portal, please use PostgreSQL. MongoDB is not supported in Developer Portal. + </Note> + + +## Tyk Stack Installations +### Installing The Chart + +To install the chart from Helm repository in namespace `tyk` with the release name `tyk-stack`: +```bash +helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ +helm repo update +helm show values tyk-helm/tyk-stack > values.yaml +``` + +For further documentation relating to *helm* command usage, please refer to the [helm docs](https://helm.sh/docs/helm/). + +At a minimum, modify values.yaml for the following settings: +1. [Set Redis connection details](#set-redis-connection-details-required) +2. [Set Mongo or PostgresSQL connection details](#set-mongo-or-postgressql-connection-details-required) +3. [Dashboard License](#tyk-dashboard-license-required) + +If you would like to use Developer Portal, an additional license is required: + +4. [Developer Portal License](#tyk-developer-portal-license-required) + +Then just run: +```bash +helm install tyk-stack tyk-helm/tyk-stack -n tyk --create-namespace -f values.yaml +``` + +### Uninstalling The Chart + +```bash +helm uninstall tyk-stack -n tyk +``` +This removes all the Kubernetes components associated with the chart and deletes the release. + +### Upgrading Chart + +```bash +helm upgrade tyk-stack tyk-helm/tyk-stack -n tyk -f values.yaml +``` + +## Configuration + +To list all configurable options with detailed comments: + +```bash +helm show values tyk-helm/tyk-stack > values.yaml +``` + +You can update any value in your local `values.yaml` file and use `-f [filename]` flag to override default values during installation. +Alternatively, you can use `--set` flag to set it in Tyk installation. See [Using Helm](https://helm.sh/docs/intro/using_helm/) for examples. + +To configure Tyk components, users can utilize both config files and [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/). Notably, environment variables take precedence over config files. To maintain simplicity and consistency, the Tyk Helm Charts deploy components with an empty config file while setting container environment variables based on user-defined [values](https://helm.sh/docs/chart_best_practices/values/). This approach ensures seamless integration with Kubernetes practices, allowing for efficient management of configurations. For a comprehensive overview of available configurations, please refer to the [configuration documentation](/tyk-oss-gateway/configuration). + +### Bootstrapping + +By default, the chart executes a bootstrapping job immediately after installation. This process ensures the presence of a valid dashboard license and initializes key components such as tyk-dashboard, tyk-portal, and tyk-operator, enabling them for immediate use. + +The bootstrapping job uses three distinct applications acting as Helm chart hooks: + +| **Bootstrapping Component** | **Description** | +| :----------------------------- | :----------------- | +| `bootstrap-pre-install` | - Runs as a pre-install hook. <br /> - Validates the Tyk Dashboard license key to ensure proper installation prerequisites. | +| `bootstrap-post-install` | - Executes post-installation. <br /> - Sets up an organization and admin user in the Tyk Dashboard. <br /> - Creates Kubernetes secrets required by Tyk Operator and Tyk Enterprise Portal. <br /> **Note**: If an existing organization and admin user are found in the database, the bootstrapping job will not set up a new organization and user. The Kubernetes secrets will not contain the expected Org ID or API key. Please update the Secret with existing credentials in this case. | +| `bootstrap-pre-delete` | - Functions as a pre-delete hook. <br /> - Cleans up resources, ensuring no residual configurations remain post-uninstallation. | + +Key Notes on Bootstrapping: + +- Bootstrapping is triggered **only during** a `helm install` and does not run during a `helm upgrade`. +- If `global.components.bootstrap` is set to `false`, only the dashboard license check will be performed. + +Handling Bootstrapping Failures: + +- If the bootstrapping process fails, check the logs from the bootstrap pods to diagnose the issue. +- Once reviewed, you can safely delete the bootstrap pods. +- To re-trigger the bootstrapping process after a failure, you must run `helm uninstall` and start the installation process anew. + +### Setting Environment Variables +Should any environment variables not be set by the Helm Chart, users can easily add them under the `extraEnvs` section within the charts for further customization. Values set under `extraEnvs` would take precedence over all configurations. + +Example of setting extra environment variable to gateway: +```yaml +tyk-gateway: + gateway: + extraEnvs: + - name: TYK_GW_LOGLEVEL + value: debug +``` + +An example is listed below for setting extra [environment variable using ConfigMap data](https://kubernetes.io/docs/tasks/configure-pod-container/configure-pod-configmap/#define-container-environment-variables-using-configmap-data), using gateway: +```yaml expandable +tyk-gateway: + gateway: + extraEnvs: + - name: CONFIG_USERNAME + valueFrom: + configMapKeyRef: + name: backend-user + key: backend-username +``` + +An example is listed below for setting extra [environment variable using secret data](https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#define-container-environment-variables-using-secret-data), using gateway: +```yaml expandable +tyk-gateway: + gateway: + extraEnvs: + - name: SECRET_USERNAME + valueFrom: + secretKeyRef: + name: backend-user + key: backend-username +``` + +In the above example, an extra environment variable `SECRET_USERNAME` will be added to the Gateway container, with a value of `backend-username` associated with the secret `backend-user`. It is useful if you want to access secret data from [Tyk Gateway configuration file (tyk.conf) or API definitions](/tyk-configuration-reference/kv-store). + +### Set Redis Connection Details (Required) + +Tyk uses Redis for distributed rate-limiting and token storage. You may use the Bitnami chart to install or Tyk's `simple-redis` chart for POC purpose. + +Set the following values after installing Redis: + +| Name | Description | +| :------ | :------------- | +| `global.redis.addrs` | Redis addresses | +| `global.redis.pass` | Redis password in plain text | +| `global.redis.passSecret.name` | If global.redis.pass is not provided, you can store it in a secret and provide the secret name here | +| `global.redis.passSecret.keyName` | key name to retrieve redis password from the secret | + +***Recommended: via *Bitnami* chart*** + +For Redis you can use these rather excellent charts provided by [Bitnami](https://github.com/bitnami/charts/tree/main/bitnami/redis). +Copy the following commands to add it: + +```bash +helm upgrade tyk-redis oci://registry-1.docker.io/bitnamicharts/redis -n tyk --create-namespace --install --version 19.0.2 +``` + + +<Note> +Please make sure you are installing Redis versions that are supported by Tyk. Please refer to Tyk docs to get list of [supported versions](/tyk-self-managed#redis). +</Note> + + +Follow the notes from the installation output to get connection details and password. + +``` + Redis(TM) can be accessed on the following DNS names from within your cluster: + + tyk-redis-master.tyk.svc.cluster.local for read/write operations (port 6379) + tyk-redis-replicas.tyk.svc.cluster.local for read-only operations (port 6379) + + export REDIS_PASSWORD=$(kubectl get secret --namespace tyk tyk-redis -o jsonpath="{.data.redis-password}" | base64 --decode) +``` + +The Redis address as set by Bitnami is `tyk-redis-master.tyk.svc.cluster.local:6379` + +You can reference the password secret generated by Bitnami chart by `--set global.redis.passSecret.name=tyk-redis` and `--set global.redis.passSecret.keyName=redis-password`, or just set `global.redis.pass=$REDIS_PASSWORD` + +***Evaluation only: via *simple-redis* chart*** + +Another option for Redis, to get started quickly, is to use our [simple-redis](https://artifacthub.io/packages/helm/tyk-helm/simple-redis) chart. + + +<Warning> +Please note that these provided charts must never be used in production or for anything +but a quick start evaluation only. Use [Bitnami Redis](https://github.com/bitnami/charts/tree/main/bitnami/redis) or [Official Redis installation guides](https://redis.io/docs/install/) in any other case. +We provide this chart, so you can quickly deploy *Tyk gateway*, but it is not meant for long term storage of data. +</Warning> + + +```bash +helm install redis tyk-helm/simple-redis -n tyk +``` + +The Tyk Helm Chart can connect to `simple-redis` in the same namespace by default. You do not need to set Redis address and password in `values.yaml`. + +### Set Mongo or PostgresSQL Connection Details (Required) +If you have already installed MongoDB or PostgreSQL, you can set the connection details in the `global.mongo` and `global.postgres` sections of the values.yaml file respectively. + +If not, you can use these rather excellent charts provided by Bitnami to install MongoDB or PostgreSQL: + +**Mongo Installation** + +```bash +helm repo add bitnami https://charts.bitnami.com/bitnami +helm install tyk-mongo bitnami/mongodb --set "replicaSet.enabled=true" -n tyk --version 15.1.3 +``` + + +<Note> +Bitnami MongoDB image is not supported on darwin/arm64 architecture. +</Note> + + +Then follow notes from the installation output to get connection details and update them in `values.yaml` file. + + +<Note> +Please make sure you are installing MongoDB versions that are supported by Tyk. Please refer to Tyk docs to get list of [supported versions](/api-management/dashboard-configuration#supported-database). +</Note> + + + +<Note> +Important Note regarding MongoDB: + +This helm chart enables the `PodDisruptionBudget` for MongoDB with an arbiter replica-count of 1. If you intend to perform system maintenance on the node where the MongoDB pod is running and this maintenance requires the node to be drained, then this action will be prevented due to the the replica count being 1. + +Increase the replica count in the helm chart deployment to a minimum of 2 to remedy this issue. +</Note> + + +Configure `global.mongo.mongoURL` and `global.storageType` as below. You should replace password in the connection string with the MONGODB_ROOT_PASSWORD you obtain from the installation output notes. + +```yaml expandable +global: + # Set mongo connection details if you want to configure mongo pump. + mongo: + # The mongoURL value will allow you to set your MongoDB address. + # Default value: mongodb://mongo.{{ .Release.Namespace }}.svc:27017/tyk_analytics + # mongoURL: mongodb://mongo.tyk.svc:27017/tyk_analytics + + # If your MongoDB has a password you can add the username and password to the url + mongoURL: mongodb://root:pass@tyk-mongo-mongodb.tyk.svc:27017/tyk_analytics?authSource=admin + + # mongo-go driver is supported for Tyk 5.0.2+. + # We recommend using the mongo-go driver if you are using MongoDB 4.4.x+. + # For MongoDB versions prior to 4.4, please use the mgo driver. + # Since Tyk 5.3 the default driver is mongo-go. + driver: mongo-go + + # Connection URL can also be set using a secret. Provide the name of the secret and key below. + # connectionURLSecret: + # name: "" + # keyName: "" + + # Enables SSL for MongoDB connection. MongoDB instance will have to support that. + # Default value: false + useSSL: false + + # Choose the storageType for Tyk. [ "mongo", "postgres" ] + storageType: &globalStorageType mongo +``` + +**PostgresSQL Installation** +```bash +helm repo add bitnami https://charts.bitnami.com/bitnami +helm install tyk-postgres bitnami/postgresql --set "auth.database=tyk_analytics" -n tyk --version 12.12.10 +``` + +Follow the notes from the installation output to get connection details. + + +<Note> +Please make sure you are installing PostgreSQL versions that are supported by Tyk. Please refer to Tyk docs to get list of [supported versions](/api-management/dashboard-configuration#supported-database). +</Note> + + +```yaml expandable +global: + # PostgreSQL connection string parameters. + postgres: + # host corresponds to the host name of postgreSQL + host: tyk-postgres-postgresql.tyk.svc + # port corresponds to the port of postgreSQL + port: 5432 + # user corresponds to the user of postgreSQL + user: postgres + # password corresponds to the password of the given postgres user in selected database + password: + # database corresponds to the database to be used in postgreSQL + database: tyk_analytics + # sslmode corresponds to if postgreSQL runs in sslmode (https) + sslmode: disable + # Connection string can also be set using a secret. Provide the name of the secret and key below. + # connectionStringSecret: + # name: "" + # keyName: "" +``` + +### Protect Confidential Fields with Kubernetes Secrets + +In the `values.yaml` file, some fields are considered confidential, such as `APISecret`, connection strings, etc. +Declaring values for such fields as plain text might not be desired. Instead, for certain fields, +Kubernetes secrets can be referenced and the chart will [define container environment variables using secret data](https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#define-container-environment-variables-using-secret-data). + +This section describes how to use Kubernetes secrets to declare confidential fields. + +***Tyk Dashboard and Developer Portal Admin*** + +If Tyk Dashboard bootstrapping is enabled, the admin user will be created according to the `global.adminUser` field. + +All admin credentials can also be set through Kubernetes secret. + + +<Note> +Once `global.adminUser.useSecretName` is declared, it takes precedence over `global.adminUser.firstName`, +`global.adminUser.lastName`, `global.adminUser.email` and `global.adminUser.password`. + +If `global.adminUser.useSecretName` is in use, please add all keys mentioned below to the secret. +</Note> + + +***Tyk Dashboard Admin First Name*** + +It can be configured via `global.adminUser.firstName` as a plain text or Kubernetes secret which includes `adminUserFirstName` key in it. Then, this secret must be referenced via `global.adminUser.useSecretName`. + + +***Tyk Dashboard Admin Last Name*** + +It can be configured via `global.adminUser.lastName` as a plain text or Kubernetes secret which includes `adminUserLastName` key in it. Then, this secret must be referenced via `global.adminUser.useSecretName`. + +***Tyk Dashboard and Developer Portal Admin Email*** + +It can be configured via `global.adminUser.email` as a plain text or Kubernetes secret which includes `adminUserEmail` key in it. Then, this secret must be referenced via `global.adminUser.useSecretName`. + + +***Tyk Dashboard and Developer Portal Admin Password*** + +It can be configured via `global.adminUser.password` as a plain text or Kubernetes secret which includes `adminUserPassword` key in it. Then, this secret must be referenced via `global.adminUser.useSecretName`. + +***APISecret*** + +The `global.secrets.APISecret` field configures a [header value](/tyk-oss-gateway/configuration#secret) used in every interaction with Tyk Gateway API. + +It can be configured via `global.secrets.APISecret` as a plain text or Kubernetes secret which includes `APISecret` key in it. Then, this secret must be referenced via `global.secrets.useSecretName`. + +```yaml +global: + secrets: + APISecret: CHANGEME + useSecretName: "mysecret" # where mysecret includes `APISecret` key with the desired value. +``` + +***AdminSecret*** + +The `global.secrets.AdminSecret` field sets a [secret](/tyk-dashboard/configuration#admin_secret) for Admin API. + +It can be configured via `global.secrets.AdminSecret` as a plain text or Kubernetes secret which includes `AdminSecret` key in it. Then, this secret must be referenced via `global.secrets.useSecretName`. + +```yaml +global: + secrets: + useSecretName: "mysecret" # where mysecret includes `AdminSecret` key with the desired value. +``` + + +<Note> +Once `global.secrets.useSecretName` is declared, it takes precedence over `global.secrets.APISecret` and `global.secrets.AdminSecret`. +</Note> + + +***Dashboard License*** + +In order to refer Tyk Dashboard license using a Kubernetes secret, please use `global.secrets.useSecretName`, where the secret should contain a key called `DashLicense`. + +***Tyk Developer Portal License*** + +In order to refer Tyk Developer Portal license using a Kubernetes secret, please use +`tyk-dev-portal.useSecretName`, where the secret should contain a key called `DevPortalLicense`. + +***Tyk Developer Portal Admin Password*** + +In order to refer Tyk Developer Portal's admin password using a Kubernetes secret, +please use `global.adminUser.useSecretName`, where the secret should contain a key called `adminUserPassword`. + +***Tyk Developer Portal Storage Connection String*** + +In order to refer Tyk Developer Portal connection string to the selected database using a Kubernetes secret, please use `tyk-dev-portal.useSecretName`, where the secret should contain a key called +`DevPortalStorageConnectionString`. + + +<Note> +If `tyk-dev-portal.useSecretName` is in use, please add all keys mentioned to the secret. +</Note> + + +***Tyk Developer Portal AWS S3 Access Key ID*** + +In order to refer to the Tyk Developer Portal AWS S3 Access Key ID using a Kubernetes secret, please use `tyk-dev-portal.useSecretName`, where the secret should contain a key called `DevPortalAwsAccessKeyId`. + + +<Note> +If `tyk-dev-portal.useSecretName` is in use, please add all keys mentioned to the secret. +</Note> + + +***Tyk Developer Portal AWS S3 Secret Access Key*** + +In order to refer Tyk Developer Portal connection string to the selected database through Kubernetes secret, +please use `tyk-dev-portal.useSecretName`, where the secret should contain a key called +`DevPortalAwsSecretAccessKey`. + + +<Note> +If `tyk-dev-portal.useSecretName` is in use, please add all keys mentioned to the secret. +</Note> + + +***Redis Password*** + +Redis password can also be provided via a secret. Store the Redis password in a Kubernetes secret and refer to this secret as follows: + +```yaml +global: + redis: + passSecret: + name: "yourSecret" + keyName: "redisPassKey" +``` + +***MongoDB or PostgreSQL connection strings*** + +Storage connection strings can also be provided via a secret. Store the connection string in Kubernetes secret and refer to this secret via `global.{mongo,postgres}.connectionURLSecret.name` and `global.{mongo,postgres}.connectionURLSecret.keyName` field, as follows: + +- MongoDB: +```yaml +global: + mongo: + connectionURLSecret: + name: "yourSecret" + keyName: "mongoConnectionURLkey" +``` + +- PostgreSQL: +```yaml +global: + postgres: + connectionStringSecret: + name: "yourSecret" + keyName: "postgreConnectionURLkey" +``` + +***Tyk Operator License*** + +It can be configured via `global.license.operator` as a plain text or Kubernetes secret which includes `OperatorLicense` key in it. Then, this secret must be referenced via `global.secrets.useSecretName`. + +Note: If you are using `global.secrets.useSecretName`, you must configure the operator license in the referenced Kubernetes secret. `global.license.operator` will not be used in this case. + +### Gateway Configurations + +This section explains how to configure the `tyk-gateway` section for updating the Gateway version, enabling TLS, enabling autoscaling etc. + +#### Update Tyk Gateway Version +Set version of gateway at `tyk-gateway.gateway.image.tag`. You can find the list of version tags available from [Docker hub](https://hub.docker.com/r/tykio/tyk-gateway/tags). Please check [Tyk Release notes](/developer-support/release-notes/gateway) carefully while upgrading or downgrading. + +#### Enabling TLS + +*Enable TLS* + +We have provided an easy way to enable TLS via the `global.tls.gateway` flag. Setting this value to true will +automatically enable TLS using the certificate provided under tyk-gateway/certs/. + +*Configure TLS secret* + +If you want to use your own key/cert pair, please follow the following steps: +1. Create a TLS secret using your cert and key pair. +2. Set `global.tls.gateway` to true. +3. Set `tyk-gateway.gateway.tls.useDefaultTykCertificate` to false. +4. Set `tyk-gateway.gateway.tls.secretName` to the name of the newly created secret. + +*Add Custom Certificates* + +To add your custom Certificate Authority(CA) to your containers, you can mount your CA certificate directly into /etc/ssl/certs folder. + +```yaml expandable + extraVolumes: + - name: self-signed-ca + secret: + secretName: self-signed-ca-secret + extraVolumeMounts: + - name: self-signed-ca + mountPath: "/etc/ssl/certs/myCA.pem" + subPath: myCA.pem +``` + +#### Enabling gateway autoscaling +You can enable autoscaling of the gateway by `--set tyk-gateway.gateway.autoscaling.enabled=true`. By default, it will enable the `Horizontal Pod Autoscaler` resource with target average CPU utilization at 60%, scaling between 1 and 3 instances. To customize those values you can modify the tyk-gateway section of the `values.yaml` file as shown below: + +```yaml expandable +tyk-gateway: + gateway: + autoscaling: + enabled: true + minReplicas: 3 + maxReplicas: 30 +``` + +Built-in rules include `tyk-gateway.gateway.autoscaling.averageCpuUtilization` for CPU utilization (set by default at 60%) and `tyk-gateway.gateway.autoscaling.averageMemoryUtilization` for memory (disabled by default). In addition to that you can define rules for custom metrics using `tyk-gateway.gateway.autoscaling.autoscalingTemplate` list: + +```yaml expandable +tyk-gateway: + gateway: + autoscaling: + autoscalingTemplate: + - type: Pods + pods: + metric: + name: nginx_ingress_controller_nginx_process_requests_total + target: + type: AverageValue + averageValue: 10000m +``` + +#### Accessing Gateway + +*Service port* + +Default service port of gateway is 8080. You can change this at `global.servicePorts.gateway`. + +*Ingress* + +An Ingress resource is created if `tyk-gateway.gateway.ingress.enabled` is set to true. + +```yaml expandable + ingress: + # if enabled, creates an ingress resource for the gateway + enabled: true + + # specify ingress controller class name + className: "" + + # annotations for ingress + annotations: {} + + # ingress rules + hosts: + - host: tyk-gw.local + paths: + - path: / + pathType: ImplementationSpecific + + # tls configuration for ingress + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + tls: [] +``` + +*Control Port* + +Set `tyk-gateway.gateway.control.enabled` to true will allow you to run the [Gateway API](/tyk-gateway-api) on a separate port and protect it behind a firewall if needed. + +#### Sharding + +Configure the gateways to load APIs with specific tags only by enabling `tyk-gateway.gateway.sharding.enabled`, and set `tags` to comma separated lists of matching tags. + +```yaml expandable + # Sharding gateway allows you to selectively load APIs to specific gateways. + # If enabled make sure you have at least one gateway that is not sharded. + # Also be sure to match API segmentation tags with the tags selected below. + sharding: + enabled: true + tags: "edge,dc1,product" +``` + +#### Deploy additional gateway groups + +The `tyk-stack` chart manages one Gateway deployment in the same namespace as Tyk Dashboard. You can flexibly deploy additional gateways using `tyk-gateway` component chart. With gateway sharding, it is useful for: +- Deploying Gateways in different networks, +- Deploying Gateways with different resources and autoscaling parameters, +- Allow different teams to manage their own Gateway instances in their own namespace. + +Here is an example configuration for `tyk-gateway` `values.yaml`. +```yaml expandable +global: + redis: + addrs: + - tyk-redis-master.tyk-stack.svc:6379 # New Gateway groups should connect to the same Redis + pass: "xxxxxxx" + +gateway: + # If this option is set to true, it will enable polling the Tyk Dashboard service for API definitions + useDashboardAppConfig: + enabled: true + # Set it to the URL to your Dashboard instance (or a load balanced instance) + # The URL needs to be formatted as: http://dashboard_host:port + # It is used to set TYK_GW_DBAPPCONFOPTIONS_CONNECTIONSTRING + dashboardConnectionString: "http://dashboard-svc-tyk-tyk-dashboard.tyk-stack.svc:3000" + + # This option is required if Policy source is set to Tyk Dashboard (`service`). + # Set this to the URL of your Tyk Dashboard installation. + # The URL needs to be formatted as: http://dashboard_host:port. + # It is used to set TYK_GW_POLICIES_POLICYCONNECTIONSTRING + policyConnectionString: "http://dashboard-svc-tyk-tyk-dashboard.tyk-stack.svc:3000" + + ... + + # Sharding gateway allows you to selectively load APIs to specific gateways. + # If enabled make sure you have at least one gateway that is not sharded. + # Also be sure to match API segmentation tags with the tags selected below. + sharding: + enabled: true + tags: "gw-dmz" + + ... + + # analyticsEnabled property is used to enable/disable analytics. + # If set to empty or nil, analytics will be enabled/disabled based on `global.components.pump`. + analyticsEnabled: "true" + + # used to decide whether to send the results back directly to Tyk without a hybrid pump + # if you want to send analytics to control plane instead of pump, change analyticsConfigType to "rpc" + analyticsConfigType: "" +``` + +Run the following command to deploy additional Gateways in namespace `another-namespace`. +```bash +helm install another-gateway tyk-helm/tyk-gateway --namespace another-namespace -f values.yaml +``` + +#### OpenTelemetry +To enable OpenTelemetry for Gateway set `gateway.opentelemetry.enabled` flag to true. It is disabled by default. + +You can also configure connection settings for it's exporter. By default `grpc` exporter is enabled on `localhost:4317` endpoint. + + To enable TLS settings for the exporter, you can set `gateway.opentelemetry.tls.enabled` to true. + +<h4 id="gateway-probes">Liveness and readiness probes</h4> +Gateway liveness probes can be customised via `gateway.livenessProbe` field. All fields from PodLivenessProbe object can be added here. If set to empty or nil, the default health check on /health will be performed. + +Gateway readiness probes can be customised via `gateway.readinessProbe` field. All fields from PodReadinessProbe object can be added here. If set to empty or nil, the default health check on /health will be performed. + +### Pump Configurations + +To enable Pump, set `global.components.pump` to true, and configure below inside `tyk-pump` section. + +| Pump | Configuration | +| :--------------------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------- | +| Prometheus Pump (Default) | Add the value `prometheus` to the `tyk-pump.pump.backend` entry, and add connection details for Prometheus under `tyk-pump.pump.prometheusPump`. | +| Mongo Pump | Add `mongo` to `tyk-pump.pump.backend`, and add connection details for mongo under `global.mongo`. | +| Mongo Selective Pump | Add `mongo-selective` to `tyk-pump.pump.backend`, and add connection details for mongo under `global.mongo`. | +| Mongo Aggregate Pump | Add `mongo-aggregate` to `tyk-pump.pump.backend`, and add connection details for mongo under `global.mongo`. | +| Postgres Pump | Add `postgres` to `tyk-pump.pump.backend`, and add connection details for postgres under `global.postgres`. | +| Postgres Aggregate Pump | Add `postgres-aggregate` to `tyk-pump.pump.backend`, and add connection details for postgres under `global.postgres`. | +| Uptime Pump | Set `tyk-pump.pump.uptimePumpBackend` to `mongo` or `postgres` or `""` | +| Other Pumps | Add the required environment variables in `tyk-pump.pump.extraEnvs` | + + + +<Note> +For additional information on Tyk Pump configurations, refer to the +[Setup Dashboard Analytics](/api-management/tyk-pump#setup-dashboard-analytics) documentation. + +To explore the list of supported backends for Tyk Pump, please visit [Pump Backends](/api-management/tyk-pump#external-data-stores). +</Note> + + +#### Prometheus Pump +Add `prometheus` to `tyk-pump.pump.backend`, and add connection details for Prometheus under `tyk-pump.pump.prometheusPump`. + +We also support monitoring using Prometheus Operator. All you have to do is set `tyk-pump.pump.prometheusPump.prometheusOperator.enabled` to true. + +This will create a _PodMonitor_ resource for your Pump instance. + +```yaml expandable + # prometheusPump configures Tyk Pump to expose Prometheus metrics. + # Please add "prometheus" to .Values.pump.backend in order to enable Prometheus Pump. + prometheusPump: + # host represents the host without port, where Tyk Pump serve the metrics for Prometheus. + host: "" + # port represents the port where Tyk Pump serve the metrics for Prometheus. + port: 9090 + # path represents the path to the Prometheus collection. For example /metrics. + path: /metrics + # customMetrics allows defining custom Prometheus metrics for Tyk Pump. + # It accepts a string that represents a JSON object. For instance, + # + # customMetrics: '[{"name":"tyk_http_requests_total","description":"Total of API requests","metric_type":"counter","labels":["response_code","api_name","method","api_key","alias","path"]}, { "name":"tyk_http_latency", "description":"Latency of API requests", "metric_type":"histogram", "labels":["type","response_code","api_name","method","api_key","alias","path"] }]' + customMetrics: "" + # If you are using prometheus Operator, set the fields in the section below. + prometheusOperator: + # enabled determines whether the Prometheus Operator is in use or not. By default, + # it is disabled. + # Tyk Pump can be monitored with PodMonitor Custom Resource of Prometheus Operator. + # If enabled, PodMonitor resource is created based on .Values.pump.prometheusPump.prometheusOperator.podMonitorSelector + # for Tyk Pump. + enabled: false + # podMonitorSelector represents a podMonitorSelector of your Prometheus resource. So that + # your Prometheus resource can select PodMonitor objects based on selector defined here. + # Please set this field to the podMonitorSelector field of your monitoring.coreos.com/v1 + # Prometheus resource's spec. + # + # You can check the podMonitorSelector via: + # kubectl describe prometheuses.monitoring.coreos.com <PROMETHEUS_POD> + podMonitorSelector: + release: prometheus-stack +``` + +#### Mongo pump +To enable Mongo pump, add `mongo` to `tyk-pump.pump.backend`, and add connection details for mongo under `global.mongo`. See [Mongo Installation](#set-mongo-or-postgressql-connection-details-required) section above. + +By default, it will enable Mongo Aggregate, Mongo Graph Pump and Mongo Selective Pump. + + +#### SQL pump +To enable SQL pump, add `postgres` to `tyk-pump.pump.backend`, and add connection details for postgres under `global.postgres`. See [PostgresSQL Installation](#set-mongo-or-postgressql-connection-details-required) section above. + +By default, it will enable Postgres Aggregate, Postgres Graph Aggregate, SQL Pump and SQL graph pump. + +#### Uptime Pump +Uptime Pump can be configured by setting `pump.uptimePumpBackend` in values.yaml file. It supports the following values +1. mongo: Used to set Mongo Pump for uptime analytics. Mongo Pump should be enabled. +2. postgres: Used to set Postgres Pump for uptime analytics. Postgres Pump should be enabled. +3. empty: Used to disable uptime analytics. + +```yaml + # uptimePumpBackend configures uptime Tyk Pump. ["", "mongo", "postgres"]. + # Set it to "" for disabling uptime Tyk Pump. By default, uptime pump is disabled. + uptimePumpBackend: "" +``` + +#### Other Pumps +To setup other backends for pump, refer to this [document](https://github.com/TykTechnologies/tyk-pump/blob/master/README.md#pumps--back-ends-supported) and add the required environment variables in `pump.extraEnvs` + + +### Tyk Dashboard Configurations + +#### Tyk Dashboard License (Required) + +Tyk Dashboard License is required. It can be set up in `global.license.dashboard` or through secret `global.secrets.useSecretName`. The secret should contain a key called *DashLicense*. + +```yaml expandable +global: + license: + # The license key needed by Tyk Dashboard to work. + # + # NOTE: If you do not want to store license as a plain text in the file, you can use a Kubernetes secret + # that stores the dashboard license. Please see `.global.secrets.useSecretName`. + dashboard: "" +``` + +#### Enabling Dashboard TLS + +Assuming that TLS certificates for the Tyk Dashboard are available in the Kubernetes Secret `tyk-dashboard-tls`, +follow these steps to enable TLS: + +1. Set `global.tls.dashboard` to `true`. +2. Set `tyk-dashboard.dashboard.tls.secretName` to the name of the Kubernetes secret containing TLS certificates for the Tyk Dashboard, in this case, `tyk-dashboard-tls`. +3. Define certificate configurations in `tyk-dashboard.dashboard.tls.certificates`, which generates `TYK_DB_HTTPSERVEROPTIONS_CERTIFICATES` for the Tyk Dashboard. + +Optional Steps, if needed: +- Modify the secret mount path on the Tyk Dashboard Pod via `tyk-dashboard.dashboard.tls.certificatesMountPath`. +- If necessary, either enable `insecureSkipVerify` via `tyk-dashboard.dashboard.tls.certificates`, or mount CA information through `tyk-dashboard.dashboard.extraVolumes` and `tyk-dashboard.dashboard.extraVolumeMounts`. +- If the `tyk-bootstrap` chart is used to bootstrap the Tyk Dashboard, ensure that it has certificates to send requests to the Tyk Dashboard or enable `insecureSkipVerify` in the `tyk-bootstrap` chart. +- If the Tyk Gateway connects to the Tyk Dashboard, confirm that the Tyk Gateway has appropriate certificates for connecting to the Tyk Dashboard + +#### Audit Log Configurations + +You can manage audit logging for Tyk Dashboard via `auditLogs`: + +- `auditLogs.enabled`: Enables or disables audit logging. It sets corresponding Dashboard environment variable `TYK_DB_AUDIT_ENABLED`. Disabled by default. +- `auditLogs.type`: Specifies the storage type for audit logs (`db` or `file`). It sets corresponding Dashboard environment variable `TYK_DB_AUDIT_STORETYPE`. Set to `file` by default. +- `auditLogs.format`: Defines the format of audit log files (`json` or `text`). It sets corresponding Dashboard environment variable `TYK_DB_AUDIT_FORMAT`. Set to `text` by default. +- `auditLogs.path`: Sets the path to the audit log file. It sets corresponding Dashboard environment variable `TYK_DB_AUDIT_PATH`. Set to "" by default. +- `auditLogs.enableDetailedRecording`: Enables detailed logging, including HTTP requests (headers only) and full HTTP responses. It sets corresponding Dashboard environment variable `TYK_DB_AUDIT_DETAILEDRECORDING`. Disabled by default. + +<h4 id="opa-configurations">OPA Configurations</h4> +You can manage OPA (Open Agent Policy) for Tyk Dashboard via `opa`: + +- `opa.enabled`: Enables OPA support. It sets corresponding Dashboard environment `TYK_DB_SECURITY_OPENPOLICY_ENABLED`. Disabled by default. +- `opa.debug`: Activates OPA debug mode for detailed logging of policy execution. It sets corresponding Dashboard environment `TYK_DB_SECURITY_OPENPOLICY_DEBUG`. Disabled by default. +- `opa.api`: Enables OPA API mode to manage policies via the Dashboard API. It sets corresponding Dashboard environment `TYK_DB_SECURITY_OPENPOLICY_ENABLEAPI`. Disabled by default. +- `opa.allowAdminPasswordReset`: Required if OPA is enabled with its default policies. It sets corresponding Dashboard environment `TYK_DB_SECURITY_ALLOWADMINRESETPASSWORD`. Enabled by default. + +### Tyk Bootstrap Configurations + +To enable [bootstrapping](#bootstrapping), set `global.components.bootstrap` to `true`. It would run [tyk-k8s-bootstrap](https://github.com/TykTechnologies/tyk-k8s-bootstrap) to bootstrap `tyk-stack` and to create Kubernetes secrets that can be utilized in Tyk Operator and Tyk Developer Portal. + + +<Note> +During bootstrapping, admin user needs to reset its password. It may be denied by Dashboard OPA rules if [OPA](/api-management/dashboard-configuration#extend-permissions-using-open-policy-agent-opa) was enabled. Please disable OPA during the initial bootstrapping or set Dashboard configuration [TYK_DB_SECURITY_ALLOWADMINRESETPASSWORD](/tyk-dashboard/configuration#securityallow_admin_reset_password) to true. +</Note> + + +#### Bootstrapped Environments + +If Tyk is already bootstrapped, the application will bypass the creation of the Tyk Organization and Admin User, proceeding directly with the creation of Kubernetes Secrets. + +Given that the Kubernetes Secrets require values for `TYK_AUTH` and `TYK_ORG`, it is essential to provide these values through the respective environment variables, called `TYK_K8SBOOTSTRAP_TYK_ADMIN_AUTH` for `TYK_AUTH` and `TYK_K8SBOOTSTRAP_TYK_ORG_ID` for `TYK_ORG`. + +Ensure that these environment variables are set appropriately to `postInstall` hook for bootstrapped environments. + +### Tyk Developer Portal Configurations + +To enable Tyk Developer Portal, set `global.components.devPortal` to true, and configure below inside `tyk-dev-portal` section. + +#### Tyk Developer Portal License (Required) + +Tyk Developer Portal License is required. It can be set up in `tyk-dev-portal.license` or through secret `global.secrets.useSecretName`. The secret should contain a key called `DevPortalLicense`. + +```yaml +tyk-dev-portal: + # Tyk Developer Portal license. + license: "" +``` + +#### Tyk Developer Portal Database + + +<Note> +Tyk no longer supports SQLite as of Tyk 5.7.0. To avoid disruption, please transition to [PostgreSQL](/planning-for-production/database-settings#postgresql), [MongoDB](/planning-for-production/database-settings#mongodb), or one of the listed compatible alternatives. +</Note> + + +By default, Tyk Developer Portal use `sqlite3` to store portal metadata. If you want to use other SQL Database, please modify the section below. + +```yaml expandable +tyk-dev-portal: + database: + # This selects the SQL dialect to be used + # The supported values are mysql, postgres and sqlite3 + dialect: "sqlite3" + connectionString: "db/portal.db" + enableLogs: false + maxRetries: 3 + retryDelay: 5000 +``` + +#### Storage Settings + +Tyk Developer Portal supports different storage options for storing the portal's CMS assets such as images, theme files and Open API Specification files. Please see the [Developer Portal storage settings](/product-stack/tyk-enterprise-developer-portal/deploy/configuration#portal-settings) page for all the available options. + +If you use the file system as storage, please set `tyk-dev-portal.storage.type` to `fs`, and configure `tyk-dev-portal.storage.persistence` to mount an existing persistent volume to Tyk Developer Portal. + +If you use [AWS S3](https://aws.amazon.com/s3/) as storage, please set `tyk-dev-portal.storage.type` to `s3`, and configure `tyk-dev-portal.storage.s3` section with credentials to access AWS S3 bucket. + +If you use database as storage, please set `tyk-dev-portal.storage.type` to `db`, and configure the `tyk-dev-portal.database` section with database connection details. + +```yaml expandable +tyk-dev-portal: + # Sensitive configuration of Portal could be set using k8s secret + # You can set following fields: + # - DevPortalLicense - Sets LicenseKey for Developer Portal + # - DevPortalStorageConnectionString - Sets connectionString for Developer Portal + # - DevPortalAwsAccessKeyId - Sets AWS S3 Access Key ID + # - DevPortalAwsSecretAccessKey - Sets AWS S3 Secret Access Key + useSecretName: "" + # The hostname to bind the Developer Portal to. + hostName: tyk-dev-portal.org + # Developer Portal license. + license: "" + # Developer portal can be deployed as StatefulSet or as Deployment + kind: StatefulSet + storage: + # User can set the storage type for portal. + # Supported types: fs, s3, db + type: "db" + # Configuration values for using s3 as storage for Tyk Developer Portal + # In case you want to provide the key ID and access key via secrets please + # refer to the existing secret inside the helm chart or the + # .Values.useSecretName field + s3: + awsAccessKeyid: your-access-key + awsSecretAccessKey: your-secret-key + region: sa-east-1 + endpoint: https://s3.sa-east-1.amazonaws.com + bucket: your-portal-bucket + acl: private + presign_urls: true + persistence: + mountExistingPVC: "" + storageClass: "" + accessModes: + - ReadWriteOnce + size: 8Gi + annotations: {} + labels: {} + selector: {} + database: + # This selects the SQL dialect to be used + # The supported values are mysql, postgres and sqlite3 + dialect: "sqlite3" + connectionString: "db/portal.db" + enableLogs: false + maxRetries: 3 + retryDelay: 5000 +``` + +#### Other Configurations + +Other [Developer Portal configurations](/product-stack/tyk-enterprise-developer-portal/deploy/configuration) can be set by using environment variables with `extraEnvs` fields, e.g.: + +```yaml +tyk-dev-portal: + extraEnvs: + - name: PORTAL_LOG_LEVEL + value: debug +``` + +### Tyk Operator Configurations + +Tyk Operator is a licensed component that requires a valid key for operation. +Please refer to the [Tyk Operator Installation Guide](/api-management/automations/operator#install-and-configure-tyk-operator) +for detailed information on the installation and upgrade processes. + +Prior to installing Tyk Operator, ensure that a valid license key is provided by setting `global.license.operator` field in values.yaml file. You can set license key via a Kubernetes secret using `global.secrets.useSecretName` field. The secret should contain a key called `OperatorLicense`. + +In order to enable installing Tyk Operator along-side Tyk Stack installation, please set `global.components.operator` +to `true`. + +All other configurations related to Tyk Operator are available under `tyk-operator` section of `values.yaml` file. + +> Tyk Operator needs a cert-manager to be installed. Ensure that cert-manager is installed as described in the +> official documentation: [Installing Tyk Operator](/api-management/automations/operator#install-and-configure-tyk-operator). diff --git a/product-stack/tyk-enterprise-developer-portal/api-documentation/list-of-endpoints/portal-1.13.0-list-of-endpoints.mdx b/product-stack/tyk-enterprise-developer-portal/api-documentation/list-of-endpoints/portal-1.13.0-list-of-endpoints.mdx new file mode 100644 index 00000000..b424df06 --- /dev/null +++ b/product-stack/tyk-enterprise-developer-portal/api-documentation/list-of-endpoints/portal-1.13.0-list-of-endpoints.mdx @@ -0,0 +1,275 @@ +--- +title: "List of endpoints exposed by Tyk Enterprise Developer Portal v1.13.0" +'og:description': "Internal APIs exposed by Tyk Developer Portal" +tags: ['Tyk Developer Portal', 'Enterprise Portal', 'Endpoints', 'Firewall', 'Integration', 'Portal 1.13.0'] +order: 3 +sidebarTitle: "Enterprise Developer Portal Admin" +--- + + +<Note> +**Tyk Enterprise Developer Portal** + +If you are interested in getting access, contact us at [support@tyk.io](<mailto:support@tyk.io?subject=Tyk Enterprise Portal Beta>) +</Note> + + +## Introduction + +In highly secure environments, it is often necessary to configure a firewall precisely and to allow only the endpoints exposed for the portal, while banning all other routes. +For this purpose, we have prepared a list of all endpoints exposed by the Tyk Enterprise Developer Portal v1.13.0. + + +<Note> +Please note that this list only refers to v1.13.0 of the portal. The list of endpoints for other version might be different. +</Note> + + + +## List of the portal endpoints + +### Admin APIs +| **Resource** | **Endpoint** | +| :------------------ | :-------------------------------------------------------------------------------------- | +| Providers | `/providers` | +| Providers | `/providers/{provider_id}` | +| Providers | `/providers/{provider_id}/synchronize` | +| Users | `/users` | +| Users | `/users/{user_id}` | +| Users | `/users/{user_id}/custom-attributes` | +| Users | `/users/{user_id}/custom-attributes/{custom-attribute_id}` | +| Organizations | `/organisations` | +| Organizations | `/organisations/{organisation_id}` | +| Teams | `/organisations/{organisation_id}/teams` | +| Teams | `/organisations/{organisation_id}/teams/{team_id}` | +| Products | `/products` | +| Products | `/products/{product_id}` | +| Product Tags | `/products/{product_id}/tags` | +| Product Tags | `/products/{product_id}/tags/{tag_name}` | +| Product Client Types | `/products/{product_id}/client_types` | +| Product Client Types | `/products/{product_id}/client_types/{client_type_id}` | +| Product Logo Image | `/products/{product_id}/logo` | +| Product Preview Image | `/products/{product_id}/preview` | +| Product Docs | `/products/{product_id}/docs` | +| Product Docs | `/products/{product_id}/docs/{doc_id}` | +| Product Docs | `/products/{product_id}/docs/reorder` | +| API Doc | `/products/{product_id}/api-details` | +| API Doc | `/products/{product_id}/api-details/{api-id}` | +| API Doc | `/products/{product_id}/api-details/{api_id}/oas` | +| Product Spec | `/products/{product_id}/spec-details` | +| Product Spec | `/products/{product_id}/spec-details/{spec-id}` | +| Product Spec | `/products/{product_id}/spec-details/{spec-id}/oas` | +| Plans | `/plans` | +| Plans | `/plans/{plan_id}` | +| Catalogues | `/catalogues` | +| Catalogues | `/catalogues/{catalogue_id}` | +| Catalogues | `/catalogues/{catalogue_id}/audiences` | +| Catalogues | `/catalogues/{catalogue_id}/audiences/{audience_id}` | +| Access Requests | `/access_requests` | +| Access Requests | `/access_requests/{access_request_id}` | +| Access Requests | `/access_requests/{access_request_id}/approve` | +| Access Requests | `/access_requests/{access_request_id}/reject` | +| Applications | `/apps` | +| Applications | `/apps/{app_id}` | +| Applications | `/apps/{app_id}/access-requests` | +| Applications | `/apps/{app_id}/access-requests/{access-request_id}` | +| Credentials | `/apps/{app_id}/access-requests/{access-request_id}/credentials` | +| Credentials | `/apps/{app_id}/access-requests/{access-request_id}/credentials/{credential_id}` | +| Credentials | `/apps/{app_id}/access-requests/{access-request_id}/credentials/{credential_id}/rotate` | +| Applications | `/apps/{app_id}/provision` | +| Custom Credentials | `/apps/{app_id}/custom_credentials` | +| Custom Credentials | `/apps/{app_id}/custom_credentials/{credential_id}` | +| Config | `/configs` | +| Pages | `/pages` | +| Pages | `/pages/{page_id}` | +| Pages | `/pages/{page_id}/content-blocks` | +| Pages | `/pages/{page_id}/content-blocks/{content-block_id}` | +| Themes | `/themes` | +| Themes | `/themes/{theme_id}` | +| Themes | `/themes/{theme_id}/activate` | +| Themes | `/themes/{theme_id}/download` | +| Themes | `/themes/upload` | +| Custom Attributes | `/extended_attributes` | +| Custom Attributes | `/extended_attributes/{extended_attribute_id}` | +| Custom Attributes | `/extended_attributes/{extended_attribute_id}/custom-attributes` | +| Custom Attributes | `/extended_attributes/{extended_attribute_id}/custom-attributes/{custom_attribute_id}` | +| Custom Attributes | `/extended_attributes/{extended_attribute_id}/default-attributes` | +| Custom Attributes | `/extended_attributes/{extended_attribute_id}/default-attributes/{default_attribute_id}` | +| OAuth2.0 Providers | `/oauth-providers` | +| OAuth2.0 Providers | `/oauth-providers/{provider_id}` | +| OAuth2.0 Providers | `/oauth-providers/{provider_id}/client-types` | +| OAuth2.0 Providers | `/oauth-providers/{provider_id}/client-types/{client_type_id}` | +| Webhooks | `/webhooks` | +| Webhooks | `/webhooks/{webhook_id}` | +| Webhooks | `/webhooks/{webhook_id}/headers` | +| Webhooks | `/webhooks/{webhook_id}/headers/{header_id}` | +| Posts | `/posts` | +| Posts | `/posts/{post_id}` | +| Posts Categories | `/posts/{post_id}/categories` | +| Posts Categories | `/posts/{post_id}/categories/{category_id}` | +| Posts Tags | `/posts/{post_id}/tags` | +| Posts Tags | `/posts/{post_id}/tags/{tag_id}` | +| Tags | `/tags` | +| Tags | `/tags/{tag_id}` | +| SSO Profiles | `/sso_profiles` | +| SSO Profiles | `/sso_profiles/{sso_profile_id}` | + +### Admin dashboard endpoints +| **Resource** | **Endpoint** | +| :------------------ | :-------------------------------------------------------------- | +| Login | `/auth/password/login` | +| Logout | `/auth/password/logout` | +| Profile | `/admin/admin_profile` | +| Profile | `/admin/admin_profile/rotate` | +| Providers | `/admin/providers` | +| Providers | `/admin/providers/new` | +| Providers | `/admin/providers/{provider_id}` | +| Providers | `/admin/providers/{provider_id}/synchronize` | +| Products | `/admin/api_products` | +| Products | `/admin/api_products/new` | +| Products | `/admin/api_products/filter` | +| Products | `/admin/api_products/apis` | +| Products | `/admin/api_products/{api_product_id}` | +| Products | `/admin/api_products/{api_product_id}/posts/{post_id}/move_up` | +| Products | `/admin/api_products/{api_product_id}/posts/{post_id}/move_down` | +| Plans | `/admin/plans` | +| Plans | `/admin/plans/new` | +| Plans | `/admin/plans/{plan_id}` | +| Catalogues | `/admin/catalogues` | +| Catalogues | `/admin/catalogues/new` | +| Catalogues | `/admin/catalogues/{catalogue_id}` | +| Product Docs | `/admin/product_docs` | +| Product Docs | `/admin/product_docs/{product_doc_id}` | +| OAuth2.0 Providers | `/admin/oauth2-0-providers` | +| OAuth2.0 Providers | `/admin/oauth2-0-providers/new` | +| OAuth2.0 Providers | `/admin/oauth2-0-providers/{oauth2.0-provider_id}` | +| OAuth2.0 Providers | `/admin/oauth2-0-providers/ping` | +| OAuth2.0 Providers | `/admin/dcr_config_templates/filter` | +| Apps | `/admin/apps` | +| Apps | `/admin/apps/new` | +| Apps | `/admin/apps/{app_id}` | +| Access Requests | `/admin/access_requests` | +| Access Requests | `/admin/access_requests/{access_request_id}` | +| Access Requests | `/admin/access_requests/{access_request_id}/approve` | +| Access Requests | `/admin/access_requests/{access_request_id}/reject` | +| Users | `/admin/users` | +| Users | `/admin/admin_users` | +| Users | `/admin/users/new` | +| Users | `/admin/admin_users/new` | +| Users | `/admin/users/{user_id}` | +| Users | `/admin/admin_users/{user_id}` | +| Users | `/admin/users/{user_id}/activate` | +| Users | `/admin/admin_users/{user_id}/activate` | +| Users | `/admin/users/{user_id}/send_invite` | +| Users | `/admin/admin_users/{user_id}/send_invite` | +| Users | `/admin/users/{user_id}/deactivate` | +| Users | `/admin/admin_users/{user_id}/deactivate` | +| Organizations | `/admin/organisations` | +| Organizations | `/admin/organisations/new` | +| Organizations | `/admin/organisations/org:{organisation_id}` | +| Teams | `/admin/teams` | +| Teams | `/admin/teams/new` | +| Teams | `/admin/teams/filter` | +| Teams | `/admin/teams/{team_id}` | +| Invite Codes | `/admin/invite_codes` | +| Invite Codes | `/admin/invite_codes/new` | +| Invite Codes | `/admin/invite_codes/{invite_code_id}` | +| Themes | `/admin/themes` | +| Themes | `/admin/themes/new` | +| Themes | `/admin/themes/{theme_name}` | +| Themes | `/admin/themes/{theme_name}/activate` | +| Themes | `/files/download` | +| Pages | `/admin/pages` | +| Pages | `/admin/pages/new` | +| Pages | `/admin/pages/{page_id}` | +| Pages | `/admin/pages/{page_id}/to_draft` | +| Menus | `/admin/menus` | +| Menus | `/admin/menus/new` | +| Menus | `/admin/menus/{menu_id}` | +| Menus | `/admin/menus/{menu_id}/menu_items/{menu_item_id}/move_up` | +| Menus | `/admin/menus/{menu_id}/menu_items/{menu_item_id}/move_down` | +| Menu Items | `/admin/menu_items` | +| Menu Items | `/admin/menu_items/new` | +| Menu Items | `/admin/menu_items/{menu_item_id}` | +| Custom Attributes | `/admin/custom_attributes` | +| Custom Attributes | `/admin/custom_attributes/{custom_attribute_id}` | +| Posts | `/admin/posts` | +| Posts | `/admin/posts/new` | +| Posts | `/admin/posts/{post_id}` | +| Categories | `/admin/categories` | +| Categories | `/admin/categories/new` | +| Categories | `/admin/categories/{category_id}` | +| Tags | `/admin/tags` | +| Tags | `/admin/tags/new` | +| Tags | `/admin/tags/{tag_id}` | +| Settings | `/admin/general` | +| About | `/admin/about` | +| About | `/admin/about/clear-cache` | +| Webhooks | `/admin/webhooks` | +| Webhooks | `/admin/webhooks/new` | +| Webhooks | `/admin/webhooks/{webhook_id}` | +| Webhooks | `/admin/webhooks/ping` | +| SSO Profiles | `/admin/sso_profiles` | +| SSO Profiles | `/admin/sso_profiles/new` | +| SSO Profiles | `/admin/sso_profiles/{sso_profile_id}` | +| SSO Profiles | `/admin/sso_profiles/raw-editor` | +| SSO Profiles | `/admin/sso_profiles/{sso_profile_id}/raw-editor` | + +### Live portal endpoints +| **Resource** | **Endpoint** | +| :----------------- | :------------------------------------------------------------- | +| Auth | `/auth/password/login` | +| Auth | `/auth/password/register` | +| Auth | `/auth/password/logout` | +| Auth | `/auth/password/new` | +| Auth | `/auth/password/edit` | +| Auth | `/auth/password/recover` | +| Auth | `/auth/password/update` | +| Auth | `/api/sso` | +| Auth | `/api/portal/developers` | +| Auth | `/api/portal/developers/ssokey/{id}` | +| Auth | `/api/portal/developers/{id}` | +| Auth | `/sso` | +| Auth | `/tib/auth/{id}/{provider}` | +| Auth | `/tib/auth/{id}/{provider}/callback` | +| Posts | `/blog` | +| Posts | `/blog/category/{category}` | +| Posts | `/blog/{year}/{month}/{day}/{path}` | +| Pages | `/about-us` | +| Pages | `/portal/catalogue-products` | +| Products | `/portal/catalogue-products/{product_name}` | +| Products | `/portal/catalogue-products/{product_name}/{api}/docs` | +| Products | `/portal/catalogue-products/{product_name}/{api}/docs/download` | +| OAS Spec Render | `/portal/catalogue-products/{product_name}/*` | +| Cart | `/portal/private/cart/provision` | +| Cart | `/portal/private/cart/remove/{catalogue_id}/{product_id}` | +| Apps | `/portal/private/dashboard` | +| Apps | `/portal/private/apps` | +| Apps | `/portal/private/apps/{app_id}` | +| Apps | `/portal/private/apps/{app_id}/cert/{cert_id}` | +| Credentials | `/portal/private/credentials` | +| Profile | `/portal/private/profile` | +| Organization Flow | `/portal/private/organisation` | +| Organization Flow | `/portal/private/users` | +| Organization Flow | `/portal/private/users/{user_id}` | +| Organization Flow | `/portal/private/users/{user_id}/edit` | +| Organization Flow | `/portal/private/users/invite` | +| Analytics | `/portal/private/analytics` | +| Analytics | `/portal/private/analytics/api/chart/overview` | +| Analytics | `/portal/private/analytics/api/chart/traffic` | +| Analytics | `/portal/private/analytics/api/chart/error` | +| Analytics | `/portal/private/analytics/api/chart/latency` | + +### Assets endpoints +| **Resource** | **Endpoint** | +| :-------------- | :------------------------------- | +| Images | `/admin/assets/images/*` | +| Images | `/assets/images/*` | +| Fonts | `/admin/assets/fonts/*` | +| Fonts | `/assets/fonts/*` | +| Javascripts | `/admin/assets/javascripts/*` | +| Javascripts | `/assets/javascripts/*` | +| Stylesheets | `/admin/assets/stylesheets/*` | +| Stylesheets | `/assets/stylesheets/*` | +| Vendors | `/assets/vendor/bootstrap/*` | +| Vendors | `/assets/vendor/jquery/*` | diff --git a/product-stack/tyk-enterprise-developer-portal/api-documentation/tyk-edp-api.mdx b/product-stack/tyk-enterprise-developer-portal/api-documentation/tyk-edp-api.mdx new file mode 100644 index 00000000..b7e57b18 --- /dev/null +++ b/product-stack/tyk-enterprise-developer-portal/api-documentation/tyk-edp-api.mdx @@ -0,0 +1,6 @@ +--- +title: "Tyk Enterprise Developer Portal API" +sidebarTitle: "Enterprise Developer Portal" +order: 3 +--- + diff --git a/product-stack/tyk-enterprise-developer-portal/deploy/configuration.mdx b/product-stack/tyk-enterprise-developer-portal/deploy/configuration.mdx new file mode 100644 index 00000000..5eba84f8 --- /dev/null +++ b/product-stack/tyk-enterprise-developer-portal/deploy/configuration.mdx @@ -0,0 +1,664 @@ +--- +title: "Environment Variables and Configuration" +'og:description': "Configuration reference for the Tyk Enterprise Developer Portal" +tags: ['Configure Tyk Enterprise Developer Portal', 'Tyk Enterprise Developer Portal'] +sidebarTitle: "Enterprise Developer Portal" +--- + +To configure the Tyk Enterprise Developer Portal, you can use either a config file or environment variables. +The table below provides a reference to all options available to you when configuring the portal. +## Portal settings +This section explains the general portal settings, including which port it will be listening on, how often it should synchronize API Products and plans with the Tyk Dashboard, and so on. +Most of these settings are optional, except for the PORTAL_LICENSEKEY. If you don't specify these settings, the default values will be used. +However, you can leverage the settings below to customize the deployment of your portal. + +### Sample storage setting section via config file +```json expandable +{ + "HostPort": 3001, + "RefreshInterval": 10, + "LicenseKey": "your-license-key", + "Theming": { + "Theme": "default", + "Path": "./themes" + }, + "ProductDocRenderer": "stoplight", + "LogLevel": "debug", + "LogFormat": "dev", + "TLSConfig": { + "Enable": true, + "InsecureSkipVerify": true, + "Certificates": [ + { + "Name": "localhost", + "CertFile": "portal.crt", + "KeyFile": "portal.key" + } + ], + "MinVersion": "772" + }, + "PortalAPISecret": "your-portal-api-secret" +} +``` + +### Sample storage setting section via environment variables +```.ini expandable +PORTAL_HOSTPORT=3001 +PORTAL_REFRESHINTERVAL=10 +PORTAL_LICENSEKEY=your-license-key +PORTAL_THEMING_THEME=default +PORTAL_THEMING_PATH=./themes +PORTAL_DOCRENDERER=stoplight +PORTAL_LOG_LEVEL=debug +PORTAL_LOG_FORMAT=dev +PORTAL_TLS_ENABLE=true +PORTAL_TLS_INSECURE_SKIP_VERIFY=true +PORTAL_TLS_CERTIFICATES = '[{"Name": "localhost","CertFile": "portal.crt","KeyFile": "portal.key"}]' +PORTAL_API_SECRET=your-portal-api-secret +``` + +### PORTAL_HOSTPORT +**Config file:** HostPort <br/> +**Type:** `int` <br/> +**Description**: The port on which the portal will run inside the container. Not required. If it is not specified, the default value is 3001. + +### PORTAL_REFRESHINTERVAL +**Config file:** RefreshInterval <br/> +**Type:** `int` <br/> +**Description**: How the portal will synchronise API Products and plans with the Tyk Dashboard. The value is specified in minutes. +Not required. If it is not specified, the default value is 10. + +### PORTAL_LICENSEKEY +**Config file:** LicenseKey <br/> +**Type:** `string` <br/> +**Description**: A license key that Tyk provides. Required to start the portal. + +### PORTAL_THEMING_THEME +**Config file:** Theming.Theme <br/> +**Type:** `string` <br/> +**Description**: The name of a theme the portal should use after the start-up. You can change this later via the Themes UI. +It's not required to specify, as the portal comes with only one theme named `default`; therefore, PORTAL_THEMING_THEME defaults to `default`. +However, if you have already created [a theme](/portal/customization#) and want the portal to use when it starts for the first time, then you can use this setting to achieve that. + +### PORTAL_THEMING_PATH +**Config file:** Theming.Path <br/> +**Type:** `string` <br/> +**Description**: Defines a folder where themes are located. Depending on the storage type that you use, you can specify either a relative or an absolute path: +- If you use the `fs` storage type, you can specify both a relative path (e.g., `./themes`) and an absolute path (e.g., `/themes`) +- If you use the `s3` or `db` storage type, however, you can only use an absolute path (e.g., `/themes`). + +The default value for this variable is `./themes`, so it's important to redefine it if you plan to use the `s3` or `db` storage types. + +### PORTAL_THEMING_DISABLE_UPLOAD +**Config file:** Theming.DisableUpload <br/> +**Type:** `boolean` <br/> +**Description**: Disables uploading theme via the UI. The default value is `false`. + +### PORTAL_MAX_UPLOAD_SIZE +**Config file:** MaxUploadSize <br/> +**Type:** `int` <br/> +**Description**: Defines the maximum size in bytes of a theme file that can be uploaded via the UI. The default value is 33554432 bytes (32 mb). + +Please note that the size of individual files should not exceed 5 MB. If the size of any individual file in a theme exceeds 5 MB, the theme will not be uploaded, even if the total size of all files is less than `PORTAL_MAX_UPLOAD_SIZE`. + +### PORTAL_DOCRENDERER +**Config file:** ProductDocRenderer <br/> +**Type:** `string` <br/> +**Options:** +- `stoplight` to use Stoplight as a documentation renderer; +- `redoc` to use Redoc as a documentation renderer. + +**Description**: Use this setting to specify which OAS documentation renderer to use to render the OpenAPI Specification. Not required. If it is not specified, the default value is `stoplight`. + +### PORTAL_DCR_LOG_ENABLED +**Config file:** DCRLogEnabled <br/> +**Type:** `boolean` <br/> +**Description**: When enabled, the portal will print raw responses from the OAuth2.0 Identity Provider for the DCR flow. +Raw responses from the Identity Providers may contain sensitive information; therefore, we recommend enabling this option only for debugging purposes. Available options are: +- `true` for enabling the detailed logs; +- `false` for disabling the detailed logs. +The default value is `false`. + +## Audit log settings +This section explains how to configure the audit log in the portal. When the audit log is enabled, each admin's action will leave a trace in the *portal.log* file located in the directory specified by the `PORTAL_AUDIT_LOG_ENABLE` setting. + +### PORTAL_AUDIT_LOG_ENABLE +**Config file:** AuditLog.Enable <br/> +**Type:** `boolean` <br/> +**Description**: Enables the audit log capability. The default value is `false`. + +### PORTAL_AUDIT_LOG_PATH +**Config file:** AuditLog.Path <br/> +**Type:** `string` <br/> +**Description**: Path to a directory with the audit log file. When audit log is enabled, the portal will create a file called `portal.log` in that directory. All admin actions will be reflected in that file. + +## Session management +This section explains how to configure session management for the portal. Using the settings below, you can configure: +- Name of the portal's session cookie. +- Various aspects of cookie security, including: should it be sent using a TLS-encrypted connection,and is it accessible by JavaScript API on the client-side? +- Cookie encryption key. +- Cookie lifetime. + +### PORTAL_SESSION_NAME +**Config file:** Session.Name <br/> +**Type:** `string` <br/> +**Description**: Name of the portal's cookie. The default value is `portal-session`. + +### PORTAL_SESSION_SECURE +**Config file:** Session.Secure <br/> +**Type:** `boolean` <br/> +**Description**: Controls whether the portal adds the `Secure` attribute to the `Set-Cookie` header in all responses from the portal's backend, except for the admin APIs. It's important to note that if the connection between the portal and the browser is not secured with TLS, the browser will ignore the `Secure` attribute. +We recommend enabling TLS and setting this attribute to `true` for all production environments. The default value is `false`. + +### PORTAL_SESSION_HTTPONLY +**Config file:** Session.HttpOnly <br/> +**Type:** `boolean` <br/> +**Description**: Controls whether the portal adds the `HttpOnly` attribute to the `Set-Cookie` header in all responses from the portal's backend, except for the admin APIs. This cookie attribute controls if the cookie is only accessible at the server and not by JavaScript on the client side. +This is a security measure to prevent XSS attacks. + +We recommend setting it to `true` in production environments. The default value is `true`. + +### PORTAL_SESSION_SAMESITE +**Config file:** Session.SameSite <br/> +**Type:** `string` <br/> +**Description**: Controls the value of the `SameSite` attribute for the portal’s cookie. The portal adds the `SameSite` attribute with the value specified in `PORTAL_SESSION_SAMESITE` to the `Set-Cookie` header in all responses from the portal's backend, except for the admin APIs. +Available options are: +- `None`; +- `Lax`; +- `Strict`. + +The default value is `Strict`. If the value specified in the `PORTAL_SESSION_SAMESITE` setting does not match any of the above-mentioned options, it defaults to `Strict`. + +### PORTAL_SESSION_KEY +**Config file:** Session.Key <br/> +**Type:** `string` <br/> +**Description**: The cookie encryption key. The default value is a random 32-bytes string. + +### PORTAL_SESSION_LIFETIME +**Config file:** Session.LifeTime <br/> +**Type:** `int` <br/> +**Description**: The lifetime of the portal's session in seconds. The default value is 604800 seconds (7 days). + +## PORTAL_SESSION_IDLE_TIMEOUT +**Config file:** Session.IdleTimeout <br/> +**Type:** `int` <br/> +**Description**: The duration in seconds before a portal session is considered idle. A session is deemed idle when there is no user activity, such as clicks, navigation, or input. Once the idle timeout is reached, the session will expire, requiring the user to log in again. The default value is 3600 seconds (1 hour). + +### PORTAL_ENABLE_HTTP_PROFILER +**Config file:** EnableHttpProfiler <br/> +**Type:** `boolean` <br/> +**Description**: Enables debugging of the portal by exposing the Golang profiling information at `/debug/pprof/`. The default value is `false`. + + +<Note> +**Profiling** + +We recommend using the profiler only in non-production environments. Be sure to disable it in production by setting `PORTAL_ENABLE_HTTP_PROFILER` to `false`. +</Note> + + +### PORTAL_LOG_LEVEL +**Config file:** LogLevel <br/> +**Type:** `string` <br/> +**Description**: Defines the log level, available options are: +- debug +- info +- warn +- error +- dpanic +- panic +- fatal + +### PORTAL_LOG_FORMAT +**Config file:** LogFormat <br/> +**Type:** `string` <br/> +**Description**: Defines the log format, available options are: +- `dev` for verbose human-readable output +- `prod` for output in json format. + +### PORTAL_TLS_ENABLE +**Config file:** TLSConfig.Enable <br/> +**Type:** `boolean` <br/> +**Description**: Enables TLS. The default value is `false`. + +### PORTAL_TLS_INSECURE_SKIP_VERIFY +**Config file:** TLSConfig.InsecureSkipVerify <br/> +**Type:** `boolean` <br/> +**Description**: Skip verification of self-signed certificates. + +### PORTAL_TLS_MIN_VERSION +**Config file:** TLSConfig.MinVersion <br/> +**Type:** `string` <br/> +**Description**: Minimum TLS version. Defaults to 769 (TLS 1.0). + +Values for TLS Versions: + +| TLS Version | Value to Use | +| :--------------- | :---------------- | +| 1.0 | 769 | +| 1.1 | 770 | +| 1.2 | 771 | +| 1.3 | 772 | + +### PORTAL_TLS_MAX_VERSION +**Config file:** TLSConfig.MaxVersion <br/> +**Type:** `string` <br/> +**Description**: Maximum TLS version. Defaults to 772 (TLS 1.3). + +Values for TLS Versions: + +| TLS Version | Value to Use | +| :--------------- | :---------------- | +| 1.0 | 769 | +| 1.1 | 770 | +| 1.2 | 771 | +| 1.3 | 772 | + +### PORTAL_TLS_CIPHER_SUITES +**Config file:** TLSConfig.CipherSuites <br/> +**Type:** `[]string` <br/> +**Description**: Array of allowed cipher suites as defined at https://golang.org/pkg/crypto/tls/#pkg-constants. + +### PORTAL_TLS_CERTIFICATES +**Config file:** TLSConfig.Certificates <br/> +**Type:** `json` <br/> +**Description**: JSON (or JSON-formatted string in case of environment variable) containing a list of certificates. Each certificate is defined by three properties: +- Name +- CertFile +- KeyFile + +### PORTAL_API_SECRET +**Config file:** PortalAPISecret <br/> +**Type:** `string` <br/> +**Description**: API secret for enabling [Single Sign-on (SSO) flow](/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/enable-sso) with the Tyk Identity Broker. +You can specify any string value in this setting. Omit this setting if you don't require SSO. + +## Response Headers Configuration +This section explains how to configure custom HTTP response headers that will be added to all responses from the Portal. + +### PORTAL_RESPONSE_HEADERS +**Config file:** ResponseHeaders <br/> +**Type:** `[]{Key: string, Value: string}` <br/> +**Description**: Configures custom HTTP response headers that will be added to all responses from the Portal. The value must be a JSON array of objects containing Key and Value fields. + +**Example configuration via environment variable:** +```bash +export PORTAL_RESPONSE_HEADERS='[{"Key":"X-Frame-Options", "Value":"DENY"}, {"Key":"Content-Security-Policy", "Value":"default-src '\''self'\''"}]' +``` + +**Example configuration via config file:** +```json expandable +{ + "ResponseHeaders": [ + { + "Key": "X-Frame-Options", + "Value": "DENY" + }, + { + "Key": "Content-Security-Policy", + "Value": "default-src 'self'" + } + ] +} +``` + +**Common use cases include:** +- Security headers (`X-Frame-Options`, `Content-Security-Policy`) +- CORS headers +- Cache control headers +- Custom application headers + +If the JSON format is invalid, the Portal will return an error message indicating the correct format: +``` +Invalid value for PORTAL_RESPONSE_HEADERS. Valid Format: '[{"Key": "header-key", "Value": "value-for-given-key"}]' +``` + +## Storage settings +Using variables from this section, you can configure storage for the portal's CMS assets, such as themes, images, and Open API Specification files. The portal supports two types of storage: +- S3 volume; +- And filesystem. + +### Sample storage setting section via config file +```json expandable +{ + "Storage": "s3", + "S3": { + "AccessKey": "your-access-key", + "SecretKey": "your-secret-key", + "Region": "sa-east-1", + "Endpoint": "https://s3.sa-east-1.amazonaws.com", + "Bucket": "your-portal-bucket", + "ACL": "private", + "PresignURLs": true + } +} +``` + +### Sample storage setting section via environment variables +```.ini expandable +PORTAL_STORAGE=s3 +PORTAL_S3_AWS_ACCESS_KEY_ID=your-access-key +PORTAL_S3_AWS_SECRET_ACCESS_KEY=your-secret-key +PORTAL_S3_REGION=sa-east-1 +PORTAL_S3_ENDPOINT=your-portal-bucket +PORTAL_S3_BUCKET=https://s3.sa-east-1.amazonaws.com +PORTAL_S3_ACL=private +PORTAL_S3_PRESIGN_URLS=true +``` + +### PORTAL_STORAGE +**Config file:** Storage <br/> +**Type:** `string` <br/> +**Options:** +- `fs` to use file system storage type; +- `db` to use the portal's main database. If the `db` is selected as a storage type, the portal application will create appropriate structure in the database that +- `s3` to use S3 volume for storing the portal assets. + +**Description**: Defines which type of storage to use for the portal's CMS assets. Not required. If it is not specified, the default value is `fs`. + +### PORTAL_S3_AWS_ACCESS_KEY_ID +**Config file:** S3.AccessKey <br/> +**Type:** `string` <br/> +**Description**: Access key for your S3 bucket. This option is only required for the `s3` storage type and will be ignored for the `fs` and `db` storage types. + +### PORTAL_S3_AWS_SECRET_ACCESS_KEY +**Config file:** S3.SecretKey <br/> +**Type:** `string` <br/> +**Description**: Secret access key for your S3 bucket. This option is only required for the `s3` storage type and will be ignored for the `fs` and `db` storage types. + +### PORTAL_S3_REGION +**Config file:** S3.Region <br/> +**Type:** `string` <br/> +**Description**: AWS region where the S3 bucket is hosted. E.g., `sa-east-1`. This option is only required for the `s3` storage type and will be ignored for the `fs` and `db` storage types. + +### PORTAL_S3_ENDPOINT +**Config file:** S3.Endpoint <br/> +**Type:** `string` <br/> +**Description**: URL to object storage service. E.g., `https://s3.sa-east-1.amazonaws.com` or `https://play.min.io`. This option is only required for the `s3` storage type and will be ignored for the `fs` and `db` storage types. + +### PORTAL_S3_BUCKET +**Config file:** S3.Bucket <br/> +**Type:** `string` <br/> +**Description**: Name of the S3 bucket. Required only for the `s3` storage type. This option is only required for the `s3` storage type and will be ignored for the `fs` and `db` storage types. + +### PORTAL_S3_ACL +**Config file:** S3.ACL <br/> +**Type:** `string` <br/> +**Description**: ACL permissions are set on the bucket, with options including `private`, `public-read`, `public-read-write`, and `authenticated-read`. +If the bucket uses a policy to set permissions, you should leave the ACL value empty. This option is only required for the `s3` storage type and will be ignored for the `fs` and `db` storage types. + +### PORTAL_S3_PRESIGN_URLS +**Config file:** S3.PresignURLs <br/> +**Type:** `string` <br/> +**Description**: The PresignURLs option instructs the client to retrieve presigned URLs for the objects. +This is particularly useful if the bucket is private and you need to access the object directly, such as when displaying an image on a web page. +This option is only required for the `s3` storage type and will be ignored for the `fs` and `db` storage types. + +### PORTAL_ASSETS_CACHE_DISABLE +**Config file:** AssetsCache.Disable <br/> +**Type:** `boolean` <br/> +**Description**: If the storage type is set to `db`, an in-memory cache will be used for the themes storage. This configuration disables the assets cache. The default value is `false`. + +## TLS configuration +This section explains the TLS configuration settings to enable connection to the portal's UI over HTTPS. + +### PORTAL_TLS_ENABLE +**Config file:** TLSConfig.Enable <br/> +**Type:** `boolean` <br/> +**Description**: Enables or disables connection over HTTPS. When TLS is enabled, the portal will expect a TLS certificate to be provided via *PORTAL_TLS_CERTIFICATES*. +When TLS is enabled and no certificates are provided, the portal won't start. The default value is `false`. + +### PORTAL_TLS_CERTIFICATES +**Config file:** TLSConfig.Certificates <br/> +**Type:** `string` <br/> +**Description**: A JSON-formatted string that provides the hostname, in addition to the paths to a TLS certificate and key file: +- `Name`: The hostname of the portal. This should match the hostname of the certificate file. +- `CertFile`: The path to a TLS certificate file in the CRT format for the specified hostname. +- `KeyFile`: The path to a TLS key file for the specified hostname. +Example: +```json +[{"Name": "tyk.io","CertFile": "portal.crt","KeyFile": "portal.key"}] +``` + + +## Database connection settings +This section provides a reference for the database connection settings used in the portal. +### Sample database connection setting section via config file +```json expandable +{ + "Database": { + "Dialect": "mysql", + "ConnectionString": "admin:secr3t@(localhost:3308)/portal?charset=utf8&parseTime=True&loc=Local", + "EnableLogs": true, + "MaxRetries": 3, + "RetryDelay": 2000, + "MaxOpenConnections": 20, + "MaxIdleConnections": 2, + "ConnectionMaxLifetime": 180000 + + } +} +``` + +### Sample database connection setting section via environment variables +```.ini expandable +PORTAL_DATABASE_DIALECT="mysql" +PORTAL_DATABASE_CONNECTIONSTRING="admin:secr3t@(localhost:3308)/portal?charset=utf8&parseTime=True&loc=Local" +PORTAL_DATABASE_ENABLELOGS=true +PORTAL_DATABASE_MAXRETRIES=3 +PORTAL_DATABASE_RETRYDELAY=5000 +PORTAL_DATABASE_MAX_OPEN_CONNECTIONS=20 +PORTAL_DATABASE_MAX_IDLE_CONNECTIONS=2 +PORTAL_DATABASE_CONNECTION_MAX_LIFETIME=180000 +``` + +### PORTAL_DATABASE_DIALECT +**Config file:** Database.Dialect <br/> +**Type:** `string` <br/> +**Description**: A database will be used to store the portal data. Available dialects are: +- `mysql` +- `postgres` +- `sqlite3` + +### PORTAL_DATABASE_CONNECTIONSTRING +**Config file:** Database.ConnectionString <br/> +**Type:** `string` <br/> +**Description**: Connection string to the selected database. This setting must be present if the `PORTAL_DATABASE_DIALECT` is specified. + +### PORTAL_DATABASE_ENABLELOGS +**Config file:** Database.EnableLogs <br/> +**Type:** `boolean` <br/> +**Description**: Enables logging connection to the database. We recommend disabling this in production environments. + +### PORTAL_DATABASE_MAXRETRIES +**Config file:** Database.MaxRetries <br/> +**Type:** `int` <br/> +**Description**: Defines how many times the portal will retry to connect to the database. Optional, the default value is 3. + +### PORTAL_DATABASE_RETRYDELAY +**Config file:** Database.RetryDelay <br/> +**Type:** `int` <br/> +**Description**: Defines the delay between connect attempts (in milliseconds). Optional. Default value: 5000. + +### PORTAL_DATABASE_MAX_OPEN_CONNECTIONS +**Config file:** Database.MaxOpenConnections <br/> +**Type:** `int` <br/> +**Description**: Defines the maximum number of concurrent connections that the database can handle from the application. When the number of open connections reaches this limit, new requests will wait until a connection becomes available. Optional. Default value: unlimited. + + +### PORTAL_DATABASE_MAX_IDLE_CONNECTIONS +**Config file:** Database.MaxIdleConnections <br/> +**Type:** `int` <br/> +**Description**: Defines the maximum number of idle connections in the database connection pool. Idle connections are open but not currently being used. Keeping some idle connections can improve performance by reducing the time it takes to establish a new connection when demand increases. Optional. Default value: 2. + +### PORTAL_DATABASE_CONNECTION_MAX_LIFETIME +**Config file:** Database.ConnectionMaxLifetime <br/> +**Type:** `int` <br/> +**Description**: Defines the maximum lifetime of a connection in milliseconds. This setting is optional. If not specified, the default value is 1800000 milliseconds (30 minutes). If set to `0`, the connection lifetime is unlimited, meaning connections are reused indefinitely unless closed due to errors or manually by the application. + + +## CORS settings +This section explains how to configure [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS) for the portal. + +### PORTAL_CORS_ENABLE +**Config file:** CORS.Enable <br/> +**Type:** `boolean` <br/> +**Description**: Enables or disables the CORS settings for the portal. When disabled, no CORS settings are applied. +In other words, any cross-origin request will be denied. When enabled, the below defined CORS settings are applied. The default value is `false`. + +### PORTAL_CORS_ALLOWED_ORIGINS +**Config file:** CORS.AllowedOrigins <br/> +**Type:** `[string]` <br/> +**Description**: A list of origin domains to allow access from. Wildcards are also supported, e.g. [`*.foo.com`] will allow access from any domain that ends with *.foo.com*. +By default, no origins are allowed. To apply this setting, an array of the allowed origins. + +To configure using a configuration file: +```json +{ + "CORS": { + "AllowedOrigins": ["*.foo.com","*.bar.com"] + } +} +``` +To configure using an environment variable: +```console +PORTAL_CORS_ALLOWED_ORIGINS=*.foo.com,*.bar.com +``` + +### PORTAL_CORS_ALLOWED_HEADERS +**Config file:** CORS.AllowedHeaders <br/> +**Type:** `[string]` <br/> +**Description**: Headers that are allowed within a request. To apply this setting, specify an array of the allowed headers. By default, no headers are allowed. + +To configure using a configuration file: +```json +{ + "CORS": { + "AllowedHeaders": ["X-Method-Override","X-API-Key"] + } +} +``` +To configure using an environment variable: +```console +PORTAL_CORS_ALLOWED_HEADERS=X-Method-Override,X-API-Key +``` + +### PORTAL_CORS_ALLOWED_METHODS +**Config file:** CORS.AllowedMethods <br/> +**Type:** `[string]` <br/> +**Description**: A list of methods that are allowed access. To apply this setting, specify an array of the allowed methods. By default, `GET` and `POST` methods are allowed. + +To configure using a configuration file: +```json +{ + "CORS": { + "AllowedMethods": ["GET", "POST", "HEAD"] + } +} +``` +To configure using an environment variable: +```console +PORTAL_CORS_ALLOWED_METHODS=GET,POST,HEAD +``` + +### PORTAL_CORS_MAX_AGE +**Config file:** CORS.MaxAge <br/> +**Type:** `int` <br/> +**Description**: Indicates how long the results of a preflight request can be cached. The default value is `0`, which stands for no max age. + +### PORTAL_DISABLE_CSRF_CHECK +**Config file:** DisableCSRFCheck <br/> +**Type:** `bool` <br/> +**Description**: When set to `true`, disables CSRF protection for all routes. By default, CSRF protection is enabled to prevent cross-site request forgery attacks. Only disable this in development environments or when you have alternative security measures in place. + +### PORTAL_CORS_ALLOW_CREDENTIALS +**Config file:** CORS.AllowCredentials <br/> +**Type:** `boolean` <br/> +**Description**: Indicates whether the request can include user credentials like cookies, HTTP authentication, or client-side SSL certificates. The default is `false`. + +### PORTAL_TIB_ENABLED +**Config file:** TIB.Enable <br/> +**Type:** `boolean` <br/> +**Description**: Enables or disables the Tyk Identity Broker (TIB) integration. When disabled, it will not appear in the UI. The default value is `false`. + +### PORTAL_NOTIFICATIONS_JOB_FREQUENCY +**Config file:** NotificationsJobFrequency <br/> +**Type:** `int` <br/> +**Description**: Defines the frequency of the notifications job that fetches notifications from the portal's database in minutes. The default value is `30` minutes. + + +## Sample config file +```json expandable +{ + "HostPort": 3001, + "RefreshInterval": 10, + "LicenseKey": "your-license-key", + "Theming": { + "Theme": "default", + "Path": "./themes" + }, + "ProductDocRenderer": "stoplight", + "LogLevel": "debug", + "LogFormat": "dev", + "TLSConfig": { + "Enable": true, + "InsecureSkipVerify": true, + "Certificates": [ + { + "Name": "localhost", + "CertFile": "portal.crt", + "KeyFile": "portal.key" + } + ] + }, + "PortalAPISecret": "your-portal-api-secret", + "Storage": "s3", + "S3": { + "AccessKey": "your-access-key", + "SecretKey": "your-secret-key", + "Region": "sa-east-1", + "Endpoint": "https://s3.sa-east-1.amazonaws.com", + "Bucket": "your-portal-bucket", + "ACL": "private", + "PresignURLs": true + }, + "Database": { + "Dialect": "mysql", + "ConnectionString": "admin:secr3t@(localhost:3308)/portal?charset=utf8&parseTime=True&loc=Local", + "EnableLogs": true, + "MaxRetries": 3, + "RetryDelay": 2000 + }, + "TIB": { + "Enable": true + } +} +``` +## Sample .env file +```ini expandable +PORTAL_HOSTPORT=3001 +PORTAL_REFRESHINTERVAL=10 +PORTAL_LICENSEKEY=your-license-key +PORTAL_THEMING_THEME=default +PORTAL_THEMING_PATH=./themes +PORTAL_DOCRENDERER=stoplight +PORTAL_LOG_LEVEL=debug +PORTAL_LOG_FORMAT=dev +PORTAL_TLS_ENABLE=true +PORTAL_TLS_INSECURE_SKIP_VERIFY=true +PORTAL_TLS_CERTIFICATES = '[{"Name": "localhost","CertFile": "portal.crt","KeyFile": "portal.key"}]' +PORTAL_API_SECRET=your-portal-api-secret +PORTAL_STORAGE=s3 +PORTAL_S3_AWS_ACCESS_KEY_ID=your-access-key +PORTAL_S3_AWS_SECRET_ACCESS_KEY=your-secret-key +PORTAL_S3_REGION=sa-east-1 +PORTAL_S3_ENDPOINT=your-portal-bucket +PORTAL_S3_BUCKET=https://s3.sa-east-1.amazonaws.com +PORTAL_S3_ACL=private +PORTAL_S3_PRESIGN_URLS=true +PORTAL_DATABASE_DIALECT="mysql" +PORTAL_DATABASE_CONNECTIONSTRING="admin:secr3t@(localhost:3308)/portal?charset=utf8&parseTime=True&loc=Local" +PORTAL_DATABASE_ENABLELOGS=true +PORTAL_DATABASE_MAXRETRIES=3 +PORTAL_TIB_ENABLED=true +``` diff --git a/product-stack/tyk-enterprise-developer-portal/getting-started/setup-email-notifications.mdx b/product-stack/tyk-enterprise-developer-portal/getting-started/setup-email-notifications.mdx new file mode 100644 index 00000000..1118a29a --- /dev/null +++ b/product-stack/tyk-enterprise-developer-portal/getting-started/setup-email-notifications.mdx @@ -0,0 +1,60 @@ +--- +title: "Setup email notifications" +'og:description': "Learn how to set up email notifications in the Tyk Enterprise Developer Portal." +tags: ['Tyk Developer Portal', 'Enterprise Portal', 'Email', 'Notifications'] +sidebarTitle: "Email Settings" +--- + +## Email Configuration + +Configuring the emailing settings is necessary for the portal to send notifications to admin users and API consumers. +Once the configuration is finished, the portal will send emails upon the following events: +* Password reset; +* New access request; +* Access request approved; +* Access request rejected; +* Pending user registration request; +* Invitation to a user to register in the portal; +* User account is activated; +* User account is deactivated; +* New organization registration request is created; +* Organization registration request is accepted; +* Organization registration request is rejected. + + +**Prerequisites** + +Before setting up the emailing configuration, you need your email server up and running. +To complete the email setup, you will need the following information about your SMTP server: +* Address of your SMTP server; +* A port on which it accepts connections; +* Username and password to connect to your SMTP server. + +## Portal Admin User Notifications + +To start with, you need to configure an email address where the portal will send notifications for admin users: new API Product access requests, new organization registration requests, and so on. +For that, you need to navigate to the General section in the Setting menu, scroll down to the Portal admin notification address, and specify the admin email address in the Portal admin email field. +<img src="/img/dashboard/portal-management/enterprise-portal/admin_email_settings.png" alt="Portal admin notification address settings" /> + +## Outbound Mailing + +### The default from email + +To enable the portal to send notifications to admin users and API Consumers, you need to specify the outbound email address in the Default Email From field. +No notifications will be sent until the Default Email From field is specified. +<img src="/img/dashboard/portal-management/enterprise-portal/default_from_email_settings.png" alt="Default from email settings" /> + +### Email Subjects + +Once the default from email is configured, you can specify subjects for notifications. +If you don’t, the default subjects will be used for email notifications. +<img src="/img/dashboard/portal-management/enterprise-portal/email_subjects_settings.png" alt="Email subject settings" /> + +### SMTP Server Settings + +Once the default from email, the admin notification email, and the subjects for outbound emails are configured, you need to configure settings for the SMTP server. +To do so, navigate to the SMTP setting section in the Settings/General menu and specify: +* Your SMTP server host and port; +* The SMTP username and password if authentication is configured for your SMTP server. +<img src="/img/dashboard/portal-management/enterprise-portal/smtp_settings.png" alt="SMTP settings" /> + diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 00000000..09110151 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,109 @@ +# Site Content Analysis Scripts + +This directory contains scripts for analyzing the content quality of the Tyk documentation site. + +## Browser Site Analyzer + +The `browser_site_analyzer.py` script uses a headless browser to analyze documentation pages for content rendering issues, specifically designed for Next.js/Mintlify sites. + +### Features + +- **Browser-based analysis**: Uses Puppeteer to properly render JavaScript-generated content +- **Smart content detection**: Waits for `.mdx-content` divs to load with meaningful content +- **Configurable timing**: Adjustable wait times and timeouts for different site speeds +- **Comprehensive reporting**: Detailed analysis with statistics and issue identification +- **Production-ready**: Designed to work with https://tyk.mintlify.app + +### Local Usage + +```bash +# Install dependencies +pip install pyppeteer beautifulsoup4 + +# Run analysis on production site +python scripts/browser_site_analyzer.py \ + --base-url https://tyk.mintlify.app \ + --docs-json docs.json \ + --wait-time 3 \ + --timeout 30 + +# Run analysis on local development server +python scripts/browser_site_analyzer.py \ + --base-url http://localhost:3000 \ + --docs-json docs.json \ + --wait-time 1 \ + --timeout 20 +``` + +### Parameters + +- `--base-url`: Base URL of the website to analyze (default: http://localhost:3000) +- `--docs-json`: Path to docs.json file (required) +- `--output-dir`: Directory to save downloaded HTML files (default: site_download) +- `--report-file`: Output file for detailed JSON report (default: browser_analysis_report.json) +- `--wait-time`: Time to wait after page load in seconds (default: 4) +- `--timeout`: Timeout for page operations in seconds (default: 30) + +### GitHub Action + +The analysis can also be run automatically via GitHub Actions: + +1. Go to the **Actions** tab in the repository +2. Select **"Site Content Analysis"** workflow +3. Click **"Run workflow"** +4. Optionally adjust timing parameters: + - **Wait time**: 2-4 seconds (recommended for production) + - **Timeout**: 30-45 seconds +5. View results in the action summary + +### Output + +The script generates: + +- **Console output**: Real-time progress and summary statistics +- **HTML files**: Downloaded pages for manual inspection (in `site_analysis_output/`) +- **JSON report**: Detailed analysis results (`site_analysis_report.json`) +- **GitHub summary**: Formatted results in GitHub Actions (when run via workflow) + +### Content Quality Criteria + +Pages are flagged as having issues if they have: +- No `.mdx-content` div found +- Empty `.mdx-content` div +- Less than 15 meaningful words (>3 characters, alphabetic) +- Less than 100 total characters in content + +### Timing Recommendations + +**For localhost development:** +- `--wait-time 1` (fast local rendering) +- `--timeout 20` (local network is fast) + +**For production sites:** +- `--wait-time 3` (allow for CDN/network delays) +- `--timeout 30` (account for slower responses) + +**For slow or heavily loaded sites:** +- `--wait-time 4-6` (extra time for content loading) +- `--timeout 45-60` (generous timeout for reliability) + +### Troubleshooting + +**"No URLs found in docs.json"** +- Verify the docs.json file path is correct +- Check that docs.json contains a valid navigation structure + +**Browser timeout errors** +- Increase `--timeout` value +- Check if the target site is accessible +- Verify network connectivity + +**Empty content detected** +- Increase `--wait-time` to allow more time for content loading +- Check if the site uses client-side rendering that needs more time +- Manually verify the page loads correctly in a browser + +**Package installation issues** +- Run: `pip install pyppeteer beautifulsoup4` +- On first run, pyppeteer will download Chromium automatically +- Ensure you have sufficient disk space for Chromium download diff --git a/scripts/browser_site_analyzer.py b/scripts/browser_site_analyzer.py new file mode 100644 index 00000000..2b98939f --- /dev/null +++ b/scripts/browser_site_analyzer.py @@ -0,0 +1,495 @@ +#!/usr/bin/env python3 +""" +Browser-Based Site Content Analyzer + +This script uses a headless browser (Puppeteer) to analyze documentation sites +by visiting each page, waiting for content to render, and examining the final HTML. +Perfect for Next.js sites with client-side rendering or dynamic content. + +USAGE: + pip install pyppeteer beautifulsoup4 + python browser_site_analyzer.py --base-url http://localhost:3000 --docs-json mintlify-poc/docs.json + +FEATURES: + - Uses headless browser for accurate content rendering + - Waits for .mdx-content divs to load with actual content + - Configurable wait times and timeouts + - Handles JavaScript-rendered content properly + - Same analysis logic as curl version but more reliable + - Optional screenshots on failures +""" + +import json +import os +import asyncio +import argparse +import time +from pathlib import Path +from urllib.parse import urlparse, urljoin +from bs4 import BeautifulSoup +import re + +try: + from pyppeteer import launch + from pyppeteer.errors import TimeoutError as PuppeteerTimeoutError +except ImportError: + print("❌ pyppeteer not installed. Run: pip install pyppeteer") + exit(1) + +class BrowserSiteAnalyzer: + def __init__(self, base_url, output_dir="site_download", wait_time=4, timeout=30): + self.base_url = base_url.rstrip('/') + self.output_dir = Path(output_dir) + self.output_dir.mkdir(exist_ok=True) + self.wait_time = wait_time # Time to wait after page load + self.timeout = timeout * 1000 # Convert to milliseconds for Puppeteer + self.browser = None + self.results = { + 'summary': { + 'total_pages_visited': 0, + 'total_pages_analyzed': 0, + 'pages_with_empty_content': 0, + 'pages_with_sufficient_content': 0, + 'browser_failures': 0 + }, + 'empty_pages': [], + 'healthy_pages': [], + 'browser_failures': [] + } + + def get_urls_from_docs_json(self, docs_json_path): + """Extract URLs from docs.json navigation structure.""" + urls = [] + + try: + print(f"Loading navigation from: {docs_json_path}") + with open(docs_json_path, 'r') as f: + docs_data = json.load(f) + + def extract_urls_recursive(nav_item, base_url): + """Recursively extract URLs from navigation structure.""" + extracted = [] + + if isinstance(nav_item, dict): + # Check if this item has a page URL + if 'page' in nav_item: + page_url = nav_item['page'] + if not page_url.startswith('http'): + page_url = f"{base_url}/{page_url.lstrip('/')}" + extracted.append(page_url) + + # Check for nested navigation + if 'pages' in nav_item: + for sub_item in nav_item['pages']: + extracted.extend(extract_urls_recursive(sub_item, base_url)) + + elif isinstance(nav_item, str): + # Direct page reference + if not nav_item.startswith('http'): + nav_item = f"{base_url}/{nav_item.lstrip('/')}" + extracted.append(nav_item) + + return extracted + + # Extract URLs from navigation + if 'navigation' in docs_data: + navigation = docs_data['navigation'] + + # Handle different navigation structures + if 'tabs' in navigation: + # Mintlify structure with tabs + for tab in navigation['tabs']: + if 'pages' in tab: + for page in tab['pages']: + urls.extend(extract_urls_recursive(page, self.base_url)) + if 'groups' in tab: + for group in tab['groups']: + if 'pages' in group: + for page in group['pages']: + urls.extend(extract_urls_recursive(page, self.base_url)) + elif isinstance(navigation, list): + # Direct list of navigation items + for nav_item in navigation: + urls.extend(extract_urls_recursive(nav_item, self.base_url)) + else: + # Other navigation structures + urls.extend(extract_urls_recursive(navigation, self.base_url)) + + # Also add the home page + urls.insert(0, self.base_url) + + # Remove duplicates while preserving order + seen = set() + unique_urls = [] + for url in urls: + if url not in seen: + seen.add(url) + unique_urls.append(url) + + print(f"Found {len(unique_urls)} URLs in docs.json") + return unique_urls + + except Exception as e: + print(f"Failed to parse docs.json: {e}") + return [] + + def url_to_filename(self, url): + """Convert URL to a safe filename.""" + parsed = urlparse(url) + path = parsed.path.strip('/') + + if not path: + return "index.html" + + # Replace path separators with underscores + filename = path.replace('/', '_') + + # Add .html extension if not present + if not filename.endswith('.html'): + filename += '.html' + + # Remove any unsafe characters + filename = re.sub(r'[<>:"/\\|?*]', '_', filename) + + return filename + + async def init_browser(self): + """Initialize the browser instance.""" + print("πŸš€ Launching headless browser...") + self.browser = await launch( + headless=True, + args=[ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + '--disable-gpu', + '--no-first-run', + '--no-default-browser-check', + '--disable-default-apps' + ] + ) + print("βœ… Browser launched successfully") + + async def close_browser(self): + """Close the browser instance.""" + if self.browser: + await self.browser.close() + print("πŸ”’ Browser closed") + + async def download_page_with_browser(self, url): + """Download and render a single page using browser.""" + filename = self.url_to_filename(url) + filepath = self.output_dir / filename + + print(f"🌐 Visiting: {url} -> {filename}") + + page = None + try: + # Create new page + page = await self.browser.newPage() + + # Set viewport and user agent + await page.setViewport({'width': 1280, 'height': 720}) + await page.setUserAgent('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36') + + # Navigate to the page + print(f" πŸ“„ Loading page...") + response = await page.goto(url, {'waitUntil': 'networkidle0', 'timeout': self.timeout}) + + if not response or response.status >= 400: + error_msg = f"HTTP {response.status if response else 'No response'}" + print(f" ❌ Failed to load: {error_msg}") + return filepath, False, error_msg + + print(f" ⏳ Waiting for content to render...") + + # Wait for .mdx-content div to appear + try: + await page.waitForSelector('div[class*="mdx-content"]', {'timeout': self.timeout}) + print(f" βœ… Found .mdx-content div") + except PuppeteerTimeoutError: + print(f" ⚠️ No .mdx-content div found within timeout") + + # Additional wait for content to load + if self.wait_time > 0: + print(f" ⏱️ Waiting {self.wait_time}s for content to fully load...") + await asyncio.sleep(self.wait_time) + + # Try to wait for meaningful content in .mdx-content div + try: + await page.waitForFunction( + '''() => { + const mdxDiv = document.querySelector('div[class*="mdx-content"]'); + if (!mdxDiv) return false; + const text = mdxDiv.innerText.trim(); + return text.length > 50; // Wait for at least 50 characters + }''', + {'timeout': 5000} # 5 second timeout for content + ) + print(f" βœ… Content loaded successfully") + except PuppeteerTimeoutError: + print(f" ⚠️ Content may still be loading...") + + # Get the final HTML content + html_content = await page.content() + + # Save HTML to file + with open(filepath, 'w', encoding='utf-8') as f: + f.write(html_content) + + print(f" βœ… Page saved: {filename}") + return filepath, True, None + + except PuppeteerTimeoutError as e: + error_msg = f"Browser timeout: {str(e)}" + print(f" ❌ Timeout: {error_msg}") + return filepath, False, error_msg + + except Exception as e: + error_msg = f"Browser error: {str(e)}" + print(f" ❌ Error: {error_msg}") + return filepath, False, error_msg + + finally: + if page: + await page.close() + + def analyze_html_content(self, filepath, url): + """Analyze downloaded HTML file for content issues.""" + try: + with open(filepath, 'r', encoding='utf-8') as f: + html_content = f.read() + + soup = BeautifulSoup(html_content, 'html.parser') + + # Get page title + title_tag = soup.find('title') + page_title = title_tag.get_text().strip() if title_tag else "Unknown" + + # Look for .mdx-content div specifically + mdx_content_div = soup.find('div', class_=lambda x: x and 'mdx-content' in x) + + page_info = { + 'url': url, + 'filename': filepath.name, + 'title': page_title, + 'has_mdx_content_div': mdx_content_div is not None, + 'mdx_content_length': 0, + 'mdx_content_text': '', + 'total_body_length': 0, + 'meaningful_words': 0, + 'is_empty': False, + 'issues': [] + } + + # Analyze .mdx-content div + if mdx_content_div: + mdx_text = mdx_content_div.get_text().strip() + page_info['mdx_content_length'] = len(mdx_text) + page_info['mdx_content_text'] = mdx_text[:200] # First 200 chars for preview + + # Count meaningful words in mdx content + words = mdx_text.split() + meaningful_words = [w for w in words if len(w) > 3 and w.isalpha()] + page_info['meaningful_words'] = len(meaningful_words) + + # Check if content is empty or insufficient + if len(mdx_text) == 0: + page_info['is_empty'] = True + page_info['issues'].append("MDX content div is completely empty") + elif len(meaningful_words) < 15: + page_info['is_empty'] = True + page_info['issues'].append(f"MDX content has only {len(meaningful_words)} meaningful words (need 15+)") + elif len(mdx_text) < 100: + page_info['is_empty'] = True + page_info['issues'].append(f"MDX content is very short ({len(mdx_text)} characters)") + else: + page_info['is_empty'] = True + page_info['issues'].append("No .mdx-content div found") + + # Also analyze total body content for comparison + body = soup.find('body') + if body: + body_text = body.get_text().strip() + page_info['total_body_length'] = len(body_text) + + return page_info + + except Exception as e: + return { + 'url': url, + 'filename': filepath.name, + 'title': 'Error', + 'has_mdx_content_div': False, + 'mdx_content_length': 0, + 'mdx_content_text': '', + 'total_body_length': 0, + 'meaningful_words': 0, + 'is_empty': True, + 'issues': [f"Error analyzing HTML: {str(e)}"] + } + + async def visit_and_analyze_site(self, urls): + """Visit all pages with browser and analyze their content.""" + print(f"\nπŸš€ Starting browser analysis of {len(urls)} pages...") + print(f"πŸ“ Saving files to: {self.output_dir}") + print(f"⏱️ Wait time per page: {self.wait_time}s") + print(f"⏰ Timeout per page: {self.timeout/1000}s") + + # Initialize browser + await self.init_browser() + + try: + for i, url in enumerate(urls, 1): + print(f"\n[{i}/{len(urls)}] Processing: {url}") + + # Visit the page with browser + filepath, success, error = await self.download_page_with_browser(url) + + if success: + self.results['summary']['total_pages_visited'] += 1 + + # Analyze the downloaded HTML + page_info = self.analyze_html_content(filepath, url) + self.results['summary']['total_pages_analyzed'] += 1 + + # Categorize the results + if page_info['is_empty']: + self.results['empty_pages'].append(page_info) + self.results['summary']['pages_with_empty_content'] += 1 + print(f" ❌ Empty content: {page_info['issues']}") + else: + self.results['healthy_pages'].append(page_info) + self.results['summary']['pages_with_sufficient_content'] += 1 + print(f" βœ… Good content: {page_info['meaningful_words']} words, {page_info['mdx_content_length']} chars") + else: + self.results['browser_failures'].append({ + 'url': url, + 'error': error + }) + self.results['summary']['browser_failures'] += 1 + + finally: + # Close browser + await self.close_browser() + + return self.results + + def print_summary_report(self): + """Print a comprehensive summary of the analysis.""" + summary = self.results['summary'] + + print("\n" + "="*80) + print("BROWSER-BASED SITE CONTENT ANALYSIS SUMMARY") + print("="*80) + print(f"Total pages visited: {summary['total_pages_visited']}") + print(f"Total pages analyzed: {summary['total_pages_analyzed']}") + print(f"Pages with empty/insufficient content: {summary['pages_with_empty_content']}") + print(f"Pages with sufficient content: {summary['pages_with_sufficient_content']}") + print(f"Browser failures: {summary['browser_failures']}") + print() + + if self.results['empty_pages']: + print("🚨 PAGES WITH EMPTY OR INSUFFICIENT CONTENT:") + print() + + for page in self.results['empty_pages']: + print(f"❌ {page['url']}") + print(f" Title: {page['title']}") + print(f" File: {page['filename']}") + print(f" MDX content length: {page['mdx_content_length']} characters") + print(f" Meaningful words: {page['meaningful_words']}") + print(f" Has .mdx-content div: {page['has_mdx_content_div']}") + if page['mdx_content_text']: + print(f" Content preview: {page['mdx_content_text'][:100]}...") + print(f" Issues: {', '.join(page['issues'])}") + print() + + if self.results['browser_failures']: + print("🚨 BROWSER FAILURES:") + print() + + for failure in self.results['browser_failures']: + print(f"❌ {failure['url']}") + print(f" Error: {failure['error']}") + print() + + if summary['pages_with_sufficient_content'] > 0: + print("βœ… PAGES WITH GOOD CONTENT:") + print() + + for page in self.results['healthy_pages'][:5]: # Show first 5 + print(f"βœ… {page['url']}") + print(f" Title: {page['title']}") + print(f" MDX content: {page['mdx_content_length']} chars, {page['meaningful_words']} words") + if page['mdx_content_text']: + print(f" Preview: {page['mdx_content_text'][:100]}...") + print() + + if len(self.results['healthy_pages']) > 5: + print(f" ... and {len(self.results['healthy_pages']) - 5} more pages with good content") + print() + + print("="*80) + print(f"RESULT: {summary['pages_with_empty_content']} pages have content issues out of {summary['total_pages_analyzed']} analyzed") + print("="*80) + + def save_detailed_report(self, output_file="browser_analysis_report.json"): + """Save detailed results to JSON file.""" + with open(output_file, 'w') as f: + json.dump(self.results, f, indent=2) + print(f"\nπŸ“„ Detailed report saved to: {output_file}") + +async def main(): + """Main function to run the browser-based site analyzer.""" + parser = argparse.ArgumentParser(description='Browser-based site content analyzer') + parser.add_argument('--base-url', default='http://localhost:3000', + help='Base URL of the website to analyze') + parser.add_argument('--docs-json', required=True, + help='Path to docs.json file') + parser.add_argument('--output-dir', default='site_download', + help='Directory to save downloaded HTML files') + parser.add_argument('--report-file', default='browser_analysis_report.json', + help='Output file for detailed JSON report') + parser.add_argument('--wait-time', type=int, default=4, + help='Time to wait after page load (seconds)') + parser.add_argument('--timeout', type=int, default=30, + help='Timeout for page operations (seconds)') + + args = parser.parse_args() + + print("πŸ” Browser-Based Site Content Analyzer") + print(f"πŸ“ Base URL: {args.base_url}") + print(f"πŸ“‹ Docs JSON: {args.docs_json}") + print(f"πŸ“ Output directory: {args.output_dir}") + print(f"⏱️ Wait time: {args.wait_time}s") + print(f"⏰ Timeout: {args.timeout}s") + + # Initialize analyzer + analyzer = BrowserSiteAnalyzer( + args.base_url, + args.output_dir, + args.wait_time, + args.timeout + ) + + # Get URLs from docs.json + urls = analyzer.get_urls_from_docs_json(args.docs_json) + + if not urls: + print("❌ No URLs found in docs.json. Exiting.") + return + + # Visit and analyze all pages + results = await analyzer.visit_and_analyze_site(urls) + + # Print summary report + analyzer.print_summary_report() + + # Save detailed report + analyzer.save_detailed_report(args.report_file) + + print(f"\n🎯 Analysis complete! Check {args.output_dir}/ for downloaded HTML files.") + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/eod-report-generator/github.js b/scripts/eod-report-generator/github.js new file mode 100644 index 00000000..d0fd91c8 --- /dev/null +++ b/scripts/eod-report-generator/github.js @@ -0,0 +1,134 @@ +import { Octokit } from "@octokit/rest"; + +const octokit = new Octokit({ + auth: "" || process.env.GITHUB_TOKEN, +}); + +const org = "TykTechnologies"; + +export async function isUserInOrg(username) { + try { + await octokit.rest.orgs.checkMembershipForUser({ + org, + username, + }); + // console.log(`${username} is a member of ${org}`); + return true; + } catch (error) { + if (error.status === 404) { + console.log(`${username} is NOT a member of ${org}`); + return false; + } + console.error("Error checking membership:", error); + throw error; + } +} + +export async function getOpenNonDraftPRs(owner, repo) { + try { + const openPRs = await octokit.paginate(octokit.rest.pulls.list, { + owner, + repo, + state: "open", + per_page: 1, // This does not work. + }); + + console.log("Total Open PRs", openPRs.length); + + return openPRs.filter((pr) => pr.draft === false || pr.draft === undefined); + } catch (error) { + console.error("Error fetching open non-draft PRs:", error); + throw error; + } +} + +export async function getPROwner(owner, repo, pull_number) { + try { + const { data } = await octokit.rest.pulls.get({ + owner, + repo, + pull_number, + }); + + return data.user.login; + } catch (error) { + console.error("Error fetching PR owner:", error); + throw error; + } +} + +export async function getRequestedReviewers(owner, repo, pullNumber) { + try { + const response = await octokit.rest.pulls.listRequestedReviewers({ + owner, + repo, + pull_number: pullNumber, + }); + + let reviewers = [] + for (const user of response.data.users) { + const isReviewerFromTykOrg = await isUserInOrg(reviewerName) + reviewers.push({ + name: user.login, + status: "not_reviewed", + isUserFromTykOrg: isReviewerFromTykOrg + }) + } + + return reviewers + } catch (error) { + console.error("Error fetching requested reviewers:", error); + } +} + +export async function getPRReviews(owner, repo, pull_number) { + try { + const reviews = await octokit.paginate(octokit.rest.pulls.listReviews, { + owner, + repo, + pull_number, + per_page: 100, + }); + + return reviews; + } catch (error) { + console.error("Error fetching PR reviews:", error); + throw error; + } +} + +export async function getMergedPRsAfterDate(owner, repo, startDate, endDate) { + try { + // Convert the provided merge date to a Date object for comparison + const sd = new Date(startDate); + const ed = new Date(endDate); + + // Fetch all closed PRs (this includes merged PRs) + const allClosedPRs = await octokit.request(`GET /repos/${owner}/${repo}/pulls?state=closed&per_page=100`, { + owner: owner, + repo: repo, + headers: { + 'X-GitHub-Api-Version': '2022-11-28' + } + }) + + console.log("Total Closed PRs:", allClosedPRs.data.length); + + // Filter merged PRs and only return those merged after the provided mergeDate + const mergedPRs = allClosedPRs.data.filter((pr) => { + + if (pr.merged_at) { + const mergedAtDate = new Date(pr.merged_at); + // console.log("MergedAt Log", pr.title, mergedAtDate.toISOString(), ed.toISOString(), sd.toISOString(), mergedAtDate >= sd, mergedAtDate <= ed); + return mergedAtDate >= sd && mergedAtDate <= ed; + } + return false; // Exclude PRs that are closed but not merged + }); + + console.log(`Total Closed PRs after date ${startDate}`, mergedPRs.length); + return mergedPRs; + } catch (error) { + console.error("Error fetching merged PRs after the specified date:", error); + throw error; + } +} \ No newline at end of file diff --git a/scripts/eod-report-generator/helper.js b/scripts/eod-report-generator/helper.js new file mode 100644 index 00000000..86d65e68 --- /dev/null +++ b/scripts/eod-report-generator/helper.js @@ -0,0 +1,98 @@ +const githubSlackUserNameMapping = { + "sharadregoti": "@sharad", + "Eopayemi": "@Elizabeth", + "sedkis": "@Sedky", + "andrei-tyk": "@Andy", + "caroltyk": "@carol", + "letzya": "@yaara", + "sredxny": "@sredny", + "excieve": "@artem", + "jeffy-mathew": "@Jeff", + "DavidRollins": "@David Rollins", + "Arturas-Salivonas": "@Arturas", + "buger": "@leo", + "tbuchaillot": "@tomas", + "carlostyk": "@Carlos", + "kofoworola": "@Kofo", + "jay-deshmukh": "@Jay", + "yurisasuke": "@keith", + "buraksezer": "@buraksezer", + "titpetric": "@Tit Petric", + "umit": "@umit", + "andyo-tyk": "@Andy", + "munkiat": "@MK", + "MFCaballero": "@Flor", + "edsonmichaque": "@Edson", + "JRWu": "@Jia Wu", + "JoanCamosTyk": "@Joan", + "LLe27": "@Long Le", + "olamilekan000": "@Ola", + "brianoh1979": "@brian", + "scott-tyk-web": "@scott" +} + +const githubStatusToMessageMapping = { + "approved": "approved the pr", + "changes_requested": "requested changes in the pr", + "not_reviewed": "not reviewed the pr", +} + +const getSlackUserName = (pr) => { + if (!pr.isUserFromTykOrg) { + return "ExternalUser" + } + return githubSlackUserNameMapping[pr.prOwner] || githubSlackUserNameMapping[pr.name] || 'undefined'; +} + +const peerStatusTemplate = (peer) => { + const slackUsername = getSlackUserName(peer) + const newStatusImprovedMessage = githubStatusToMessageMapping[peer.status] || peer.status + + return `${slackUsername} has ${newStatusImprovedMessage}` +} + +const thankYouTemplate = (pr) => { + const slackUsername = getSlackUserName(pr) + return `- [${pr.prName}](${pr.prUrl}) - Thanks ${slackUsername}.`; +} + +const inProgressTemplate = (pr) => { + // Check if pr.prOwner exists in the mapping and replace it, otherwise use pr.prOwner or 'undefined' + const slackUsername = getSlackUserName(pr) + + const peerReviewMessage = pr.peerReviewers.map(peerStatusTemplate).join("\n"); + + const newStatusImprovedMessage = githubStatusToMessageMapping[pr.mainBossStatus] || pr.mainBossStatus + + let finalMessage = `- [${pr.prName}](${pr.prUrl}) - ${slackUsername} - I have ${newStatusImprovedMessage}.` + if (peerReviewMessage !== "") { + finalMessage += ` Regarding peer review, ${peerReviewMessage}`; + } + return finalMessage +} + +export const categorizeBulletPoints = (prs, isThankYou) => { + // Group the PRs by category + const grouped = prs.reduce((acc, pr) => { + const category = pr.category || 'Uncategorized'; // Default category if not present + if (!acc[category]) { + acc[category] = []; + } + acc[category].push(pr); + return acc; + }, {}); + + // Create the bullet points for each category + let result = ''; + for (const [category, prsInCategory] of Object.entries(grouped)) { + result += `*${category}*\n`; + let bulletPoints = ""; + if (isThankYou) { + bulletPoints = prsInCategory.map(thankYouTemplate).join("\n"); + } else { + bulletPoints = prsInCategory.map(inProgressTemplate).join("\n"); + } + result += bulletPoints + "\n\n"; // Add bullet points and spacing between categories + } + return result.trim(); // Trim extra newlines at the end +} diff --git a/scripts/eod-report-generator/index.js b/scripts/eod-report-generator/index.js new file mode 100644 index 00000000..bb056357 --- /dev/null +++ b/scripts/eod-report-generator/index.js @@ -0,0 +1,140 @@ +import { formatName } from "./sonet.js"; +import { getOpenNonDraftPRs, getMergedPRsAfterDate,getRequestedReviewers, getPROwner, getPRReviews, isUserInOrg } from "./github.js"; +import { categorizeBulletPoints } from "./helper.js"; + +const mainBoss = "" || process.env.MAIN_REVIEWER; + +let eod_report_result = []; + +const labelsToConsiderForGrouping = ["Release Notes", "future-release", "New IA Project"] + +async function analyzePR(owner, repo, pr, ignoreBuger) { + const pull_number = pr.number; + const prName = await formatName(pr.title); + const prUrl = pr.html_url; + const prOwner = await getPROwner(owner, repo, pull_number); + const isUserFromTykOrg = await isUserInOrg(prOwner) + const reviews = await getPRReviews(owner, repo, pull_number); + // const reviewers = await getRequestedReviewers(owner, repo, 5936) + + if (ignoreBuger && (prOwner === "buger")) { + return + } + + let mainBossStatus = "not_reviewed"; + const peerReviewers = []; + + // Get reviewers of a PR + for (const review of reviews) { + const reviewerName = review.user.login; + const reviewStatus = review.state.toLowerCase(); + const isReviewerFromTykOrg = await isUserInOrg(reviewerName) + + if (reviewStatus === "commented" || reviewStatus === "dismissed") { + continue + } + + if (reviewerName === mainBoss) { + mainBossStatus = reviewStatus; + } else { + // Check if existing reviewr, if yes update the status + const existingReviewer = peerReviewers.find(reviewer => reviewer.name === reviewerName); + + if (existingReviewer) { + existingReviewer.status = reviewStatus; + } else { + peerReviewers.push({ + name: reviewerName, + status: reviewStatus, + isUserFromTykOrg: isReviewerFromTykOrg + }); + } + } + } + + // This are reviewers who were requested + // for (const review of reviewers) { + // } + + // Check if part of category + const category = labelsToConsiderForGrouping.find(field => + pr.labels.some(label => label.name === field) + ) || "Docs"; + + eod_report_result.push({ + prName, + prUrl, + prOwner, + peerReviewers, + mainBossStatus, + // isMerged: false, + isUserFromTykOrg, + category + }); +} + + +// Example usage +(async () => { + const owner = "TykTechnologies"; + const repo = "tyk-docs"; + + const startDate = process.env.START_DATE || new Date().toISOString(); + const endDate = process.env.END_DATE || new Date().toISOString(); + console.log("Start Date:", startDate); // Example output: "2025-02-04T15:30:00Z" + console.log("End Date:", endDate); // Example output: "2025-02-04T15:30:00Z" + + try { + // ############################### + // Get Open Prs - In Making + // ############################### + console.log("============ Stats ============") + const openPrs = await getOpenNonDraftPRs(owner, repo); + console.log("Open PRs - Excluding Draft:", openPrs.length) + + // Populate "eod_report_result" variable with our JSON structure + for (const pr of openPrs) { + await analyzePR(owner, repo, pr, false); + } + + // console.log(JSON.stringify(eod_report_result, null, 2)); + const inMaking = categorizeBulletPoints(eod_report_result); + // console.log(inMaking); + + // ############################### + // Get Merged PRs + // ############################### + + const mergedPrs = await getMergedPRsAfterDate(owner, repo, startDate, endDate); + + eod_report_result = []; //Reset array + for (const pr of mergedPrs) { + await analyzePR(owner, repo, pr, true); + } + + // console.log(JSON.stringify(eod_report_result, null, 2)); + // return + let merged = categorizeBulletPoints(eod_report_result, true); + if (merged === "") { + merged = "No new pages published." + } + // console.log(merged); + + console.log("============ Final Print ============") + + console.log(` +Hi all, EOD status report for ${endDate}, is as follows: +:white_check_mark: New published pages :partywizard::partywizard::partywizard::partywizard: +================================ +${merged} + +In the making +================================ +${inMaking} +`) + + + } catch (error) { + console.error("Error generating EOD report:", error); + } +})(); diff --git a/scripts/eod-report-generator/package-lock.json b/scripts/eod-report-generator/package-lock.json new file mode 100644 index 00000000..faf9314d --- /dev/null +++ b/scripts/eod-report-generator/package-lock.json @@ -0,0 +1,1562 @@ +{ + "name": "eod-report-generator", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "eod-report-generator", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@anthropic-ai/sdk": "^0.32.1", + "@octokit/rest": "21.0.2", + "case": "^1.6.3", + "change-case": "^5.4.4", + "gray-matter": "^4.0.3", + "js-yaml": "^4.1.0", + "markdown-link-extractor": "^4.0.2", + "marked": "^14.1.2", + "openai": "^4.72.0", + "sharp": "0.32.6" + } + }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.32.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.32.1.tgz", + "integrity": "sha512-U9JwTrDvdQ9iWuABVsMLj8nJVwAyQz6QXvgLsVhryhCEPkLsbcP/MXxm+jYcAwLoV8ESbaTTjnD4kuAFa+Hyjg==", + "license": "MIT", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + } + }, + "node_modules/@octokit/auth-token": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-5.1.2.tgz", + "integrity": "sha512-JcQDsBdg49Yky2w2ld20IHAlwr8d/d8N6NiOXbtuoPCqzbsiJgF633mVUw3x4mo0H5ypataQIX7SFu3yy44Mpw==", + "license": "MIT", + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/core": { + "version": "6.1.3", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-6.1.3.tgz", + "integrity": "sha512-z+j7DixNnfpdToYsOutStDgeRzJSMnbj8T1C/oQjB6Aa+kRfNjs/Fn7W6c8bmlt6mfy3FkgeKBRnDjxQow5dow==", + "license": "MIT", + "dependencies": { + "@octokit/auth-token": "^5.0.0", + "@octokit/graphql": "^8.1.2", + "@octokit/request": "^9.1.4", + "@octokit/request-error": "^6.1.6", + "@octokit/types": "^13.6.2", + "before-after-hook": "^3.0.2", + "universal-user-agent": "^7.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/endpoint": { + "version": "10.1.3", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-10.1.3.tgz", + "integrity": "sha512-nBRBMpKPhQUxCsQQeW+rCJ/OPSMcj3g0nfHn01zGYZXuNDvvXudF/TYY6APj5THlurerpFN4a/dQAIAaM6BYhA==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^13.6.2", + "universal-user-agent": "^7.0.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/graphql": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-8.2.0.tgz", + "integrity": "sha512-gejfDywEml/45SqbWTWrhfwvLBrcGYhOn50sPOjIeVvH6i7D16/9xcFA8dAJNp2HMcd+g4vru41g4E2RBiZvfQ==", + "license": "MIT", + "dependencies": { + "@octokit/request": "^9.1.4", + "@octokit/types": "^13.8.0", + "universal-user-agent": "^7.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/openapi-types": { + "version": "23.0.1", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-23.0.1.tgz", + "integrity": "sha512-izFjMJ1sir0jn0ldEKhZ7xegCTj/ObmEDlEfpFrx4k/JyZSMRHbO3/rBwgE7f3m2DHt+RrNGIVw4wSmwnm3t/g==", + "license": "MIT" + }, + "node_modules/@octokit/plugin-paginate-rest": { + "version": "11.4.3", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-11.4.3.tgz", + "integrity": "sha512-tBXaAbXkqVJlRoA/zQVe9mUdb8rScmivqtpv3ovsC5xhje/a+NOCivs7eUhWBwCApJVsR4G5HMeaLbq7PxqZGA==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^13.7.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@octokit/core": ">=6" + } + }, + "node_modules/@octokit/plugin-request-log": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/@octokit/plugin-request-log/-/plugin-request-log-5.3.1.tgz", + "integrity": "sha512-n/lNeCtq+9ofhC15xzmJCNKP2BWTv8Ih2TTy+jatNCCq/gQP/V7rK3fjIfuz0pDWDALO/o/4QY4hyOF6TQQFUw==", + "license": "MIT", + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@octokit/core": ">=6" + } + }, + "node_modules/@octokit/plugin-rest-endpoint-methods": { + "version": "13.3.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-13.3.0.tgz", + "integrity": "sha512-LUm44shlmkp/6VC+qQgHl3W5vzUP99ZM54zH6BuqkJK4DqfFLhegANd+fM4YRLapTvPm4049iG7F3haANKMYvQ==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^13.7.0" + }, + "engines": { + "node": ">= 18" + }, + "peerDependencies": { + "@octokit/core": ">=6" + } + }, + "node_modules/@octokit/request": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-9.2.2.tgz", + "integrity": "sha512-dZl0ZHx6gOQGcffgm1/Sf6JfEpmh34v3Af2Uci02vzUYz6qEN6zepoRtmybWXIGXFIK8K9ylE3b+duCWqhArtg==", + "license": "MIT", + "dependencies": { + "@octokit/endpoint": "^10.1.3", + "@octokit/request-error": "^6.1.7", + "@octokit/types": "^13.6.2", + "fast-content-type-parse": "^2.0.0", + "universal-user-agent": "^7.0.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/request-error": { + "version": "6.1.7", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-6.1.7.tgz", + "integrity": "sha512-69NIppAwaauwZv6aOzb+VVLwt+0havz9GT5YplkeJv7fG7a40qpLt/yZKyiDxAhgz0EtgNdNcb96Z0u+Zyuy2g==", + "license": "MIT", + "dependencies": { + "@octokit/types": "^13.6.2" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/rest": { + "version": "21.0.2", + "resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-21.0.2.tgz", + "integrity": "sha512-+CiLisCoyWmYicH25y1cDfCrv41kRSvTq6pPWtRroRJzhsCZWZyCqGyI8foJT5LmScADSwRAnr/xo+eewL04wQ==", + "license": "MIT", + "dependencies": { + "@octokit/core": "^6.1.2", + "@octokit/plugin-paginate-rest": "^11.0.0", + "@octokit/plugin-request-log": "^5.3.1", + "@octokit/plugin-rest-endpoint-methods": "^13.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/types": { + "version": "13.8.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.8.0.tgz", + "integrity": "sha512-x7DjTIbEpEWXK99DMd01QfWy0hd5h4EN+Q7shkdKds3otGQP+oWE/y0A76i1OvH9fygo4ddvNf7ZvF0t78P98A==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^23.0.1" + } + }, + "node_modules/@types/node": { + "version": "18.19.75", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.75.tgz", + "integrity": "sha512-UIksWtThob6ZVSyxcOqCLOUNg/dyO1Qvx4McgeuhrEtHTLFTf7BBhEazaE4K806FGTPtzd/2sE90qn4fVr7cyw==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@types/node-fetch": { + "version": "2.6.12", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz", + "integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==", + "license": "MIT", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.0" + } + }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "license": "MIT", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/agentkeepalive": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", + "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", + "license": "MIT", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "license": "Python-2.0" + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, + "node_modules/b4a": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.7.tgz", + "integrity": "sha512-OnAYlL5b7LEkALw87fUVafQw5rVR9RjwGd4KUwNQ6DrrNmaVaUCgLipfVlzrPQ4tWOR9P0IXGNOx50jYCCdSJg==", + "license": "Apache-2.0" + }, + "node_modules/bare-events": { + "version": "2.5.4", + "resolved": "https://registry.npmjs.org/bare-events/-/bare-events-2.5.4.tgz", + "integrity": "sha512-+gFfDkR8pj4/TrWCGUGWmJIkBwuxPS5F+a5yWjOHQt2hHvNZd5YLzadjmDUtFmMM4y429bnKLa8bYBMHcYdnQA==", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/bare-fs": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/bare-fs/-/bare-fs-4.0.1.tgz", + "integrity": "sha512-ilQs4fm/l9eMfWY2dY0WCIUplSUp7U0CT1vrqMg1MUdeZl4fypu5UP0XcDBK5WBQPJAKP1b7XEodISmekH/CEg==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-events": "^2.0.0", + "bare-path": "^3.0.0", + "bare-stream": "^2.0.0" + }, + "engines": { + "bare": ">=1.7.0" + } + }, + "node_modules/bare-os": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/bare-os/-/bare-os-3.4.0.tgz", + "integrity": "sha512-9Ous7UlnKbe3fMi7Y+qh0DwAup6A1JkYgPnjvMDNOlmnxNRQvQ/7Nst+OnUQKzk0iAT0m9BisbDVp9gCv8+ETA==", + "license": "Apache-2.0", + "optional": true, + "engines": { + "bare": ">=1.6.0" + } + }, + "node_modules/bare-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bare-path/-/bare-path-3.0.0.tgz", + "integrity": "sha512-tyfW2cQcB5NN8Saijrhqn0Zh7AnFNsnczRcuWODH0eYAXBsJ5gVxAUuNr7tsHSC6IZ77cA0SitzT+s47kot8Mw==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-os": "^3.0.1" + } + }, + "node_modules/bare-stream": { + "version": "2.6.5", + "resolved": "https://registry.npmjs.org/bare-stream/-/bare-stream-2.6.5.tgz", + "integrity": "sha512-jSmxKJNJmHySi6hC42zlZnq00rga4jjxcgNZjY9N5WlOe/iOoGRtdwGsHzQv2RlH2KOYMwGUXhf2zXd32BA9RA==", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "streamx": "^2.21.0" + }, + "peerDependencies": { + "bare-buffer": "*", + "bare-events": "*" + }, + "peerDependenciesMeta": { + "bare-buffer": { + "optional": true + }, + "bare-events": { + "optional": true + } + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/before-after-hook": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-3.0.2.tgz", + "integrity": "sha512-Nik3Sc0ncrMK4UUdXQmAnRtzmNQTAAXmXIopizwZ1W1t8QmfJj+zL4OA2I7XPTPW5z5TDqv4hRo/JzouDJnX3A==", + "license": "Apache-2.0" + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC" + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/case": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/case/-/case-1.6.3.tgz", + "integrity": "sha512-mzDSXIPaFwVDvZAHqZ9VlbyF4yyXRuX6IvB06WvPYkqJVO24kX1PPhv9bfpKNFZyxYFmmgo03HUiD8iklmJYRQ==", + "license": "(MIT OR GPL-3.0-or-later)", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/change-case": { + "version": "5.4.4", + "resolved": "https://registry.npmjs.org/change-case/-/change-case-5.4.4.tgz", + "integrity": "sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w==", + "license": "MIT" + }, + "node_modules/cheerio": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0.tgz", + "integrity": "sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==", + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.1.0", + "encoding-sniffer": "^0.2.0", + "htmlparser2": "^9.1.0", + "parse5": "^7.1.2", + "parse5-htmlparser2-tree-adapter": "^7.0.0", + "parse5-parser-stream": "^7.1.2", + "undici": "^6.19.5", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=18.17" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, + "node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "license": "MIT", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/css-select": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/detect-libc": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/encoding-sniffer": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.0.tgz", + "integrity": "sha512-ju7Wq1kg04I3HtiYIOrUrdfdDvkyO9s5XM8QAj/bN61Yo/Vb4vgJxy5vi4Yxk01gWHbrofpPtpxM8bKger9jhg==", + "license": "MIT", + "dependencies": { + "iconv-lite": "^0.6.3", + "whatwg-encoding": "^3.1.1" + }, + "funding": { + "url": "https://github.com/fb55/encoding-sniffer?sponsor=1" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, + "node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "license": "MIT", + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fast-content-type-parse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-content-type-parse/-/fast-content-type-parse-2.0.1.tgz", + "integrity": "sha512-nGqtvLrj5w0naR6tDPfB4cUmYCqouzyQiz6C5y/LtcDllJdrcc6WaWW6iXyIIOErTa/XRybj28aasdn4LkVk6Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/fast-fifo/-/fast-fifo-1.3.2.tgz", + "integrity": "sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ==", + "license": "MIT" + }, + "node_modules/form-data": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.1.tgz", + "integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", + "license": "MIT" + }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "license": "MIT", + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "license": "MIT" + }, + "node_modules/gray-matter": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/gray-matter/-/gray-matter-4.0.3.tgz", + "integrity": "sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==", + "license": "MIT", + "dependencies": { + "js-yaml": "^3.13.1", + "kind-of": "^6.0.2", + "section-matter": "^1.0.0", + "strip-bom-string": "^1.0.0" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/gray-matter/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/gray-matter/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/html-link-extractor": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/html-link-extractor/-/html-link-extractor-1.0.5.tgz", + "integrity": "sha512-ADd49pudM157uWHwHQPUSX4ssMsvR/yHIswOR5CUfBdK9g9ZYGMhVSE6KZVHJ6kCkR0gH4htsfzU6zECDNVwyw==", + "license": "MIT", + "dependencies": { + "cheerio": "^1.0.0-rc.10" + } + }, + "node_modules/htmlparser2": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz", + "integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.1.0", + "entities": "^4.5.0" + } + }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.0.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, + "node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", + "license": "MIT" + }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/markdown-link-extractor": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/markdown-link-extractor/-/markdown-link-extractor-4.0.2.tgz", + "integrity": "sha512-5cUOu4Vwx1wenJgxaudsJ8xwLUMN7747yDJX3V/L7+gi3e4MsCm7w5nbrDQQy8nEfnl4r5NV3pDXMAjhGXYXAw==", + "license": "ISC", + "dependencies": { + "html-link-extractor": "^1.0.5", + "marked": "^12.0.1" + } + }, + "node_modules/markdown-link-extractor/node_modules/marked": { + "version": "12.0.2", + "resolved": "https://registry.npmjs.org/marked/-/marked-12.0.2.tgz", + "integrity": "sha512-qXUm7e/YKFoqFPYPa3Ukg9xlI5cyAtGmyEIzMfW//m6kXwCy2Ps9DYf5ioijFKQ8qyuscrHoY04iJGctu2Kg0Q==", + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/marked": { + "version": "14.1.4", + "resolved": "https://registry.npmjs.org/marked/-/marked-14.1.4.tgz", + "integrity": "sha512-vkVZ8ONmUdPnjCKc5uTRvmkRbx4EAi2OkTOXmfTDhZz3OFqMNBM1oTTWwTr4HY4uAEojhzPf+Fy8F1DWa3Sndg==", + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "license": "MIT" + }, + "node_modules/node-abi": { + "version": "3.74.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.74.0.tgz", + "integrity": "sha512-c5XK0MjkGBrQPGYG24GBADZud0NCbznxNx0ZkS+ebUTrmV1qTDxPxSL8zEAPURXSbLRWVexxmP4986BziahL5w==", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-addon-api": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", + "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==", + "license": "MIT" + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "license": "MIT", + "dependencies": { + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" + }, + "peerDependencies": { + "encoding": "^0.1.0" + }, + "peerDependenciesMeta": { + "encoding": { + "optional": true + } + } + }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/openai": { + "version": "4.82.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.82.0.tgz", + "integrity": "sha512-1bTxOVGZuVGsKKUWbh3BEwX1QxIXUftJv+9COhhGGVDTFwiaOd4gWsMynF2ewj1mg6by3/O+U8+EEHpWRdPaJg==", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + }, + "bin": { + "openai": "bin/cli" + }, + "peerDependencies": { + "ws": "^8.18.0", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/parse5": { + "version": "7.2.1", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.2.1.tgz", + "integrity": "sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ==", + "license": "MIT", + "dependencies": { + "entities": "^4.5.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", + "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-parser-stream": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz", + "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==", + "license": "MIT", + "dependencies": { + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/prebuild-install/node_modules/tar-fs": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.3.tgz", + "integrity": "sha512-090nwYJDmlhwFwEW3QQl+vaNnxsO2yVsd45eTKRBzSzu+hlb1w2K9inVq5b0ngXuLVqQ4ApvsUHHnu/zQNkWAg==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/prebuild-install/node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pump": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", + "integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/section-matter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz", + "integrity": "sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==", + "license": "MIT", + "dependencies": { + "extend-shallow": "^2.0.1", + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/semver": { + "version": "7.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz", + "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp": { + "version": "0.32.6", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.32.6.tgz", + "integrity": "sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.2", + "node-addon-api": "^6.1.0", + "prebuild-install": "^7.1.1", + "semver": "^7.5.4", + "simple-get": "^4.0.1", + "tar-fs": "^3.0.4", + "tunnel-agent": "^0.6.0" + }, + "engines": { + "node": ">=14.15.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "license": "BSD-3-Clause" + }, + "node_modules/streamx": { + "version": "2.22.0", + "resolved": "https://registry.npmjs.org/streamx/-/streamx-2.22.0.tgz", + "integrity": "sha512-sLh1evHOzBy/iWRiR6d1zRcLao4gGZr3C1kzNz4fopCOKJb6xD9ub8Mpi9Mr1R6id5o43S+d93fI48UC5uM9aw==", + "license": "MIT", + "dependencies": { + "fast-fifo": "^1.3.2", + "text-decoder": "^1.1.0" + }, + "optionalDependencies": { + "bare-events": "^2.2.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/strip-bom-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-bom-string/-/strip-bom-string-1.0.0.tgz", + "integrity": "sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tar-fs": { + "version": "3.0.9", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.9.tgz", + "integrity": "sha512-XF4w9Xp+ZQgifKakjZYmFdkLoSWd34VGKcsTCwlNWM7QG3ZbaxnTsaBwnjFZqHRf/rROxaR8rXnbtwdvaDI+lA==", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^4.0.1", + "bare-path": "^3.0.0" + } + }, + "node_modules/tar-stream": { + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-3.1.7.tgz", + "integrity": "sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ==", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/text-decoder": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/text-decoder/-/text-decoder-1.2.3.tgz", + "integrity": "sha512-3/o9z3X0X0fTupwsYvR03pJ/DjWuqqrfwBgTQzdWDiQSm9KitAyz/9WqsT2JQW7KV2m+bC2ol/zqpW37NHxLaA==", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "license": "MIT" + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/undici": { + "version": "6.21.3", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.21.3.tgz", + "integrity": "sha512-gBLkYIlEnSp8pFbT64yFgGE6UIB9tAkhukC23PmMDCe5Nd+cRqKxSjw5y54MK2AZMgZfJWMaNE4nYUHgi1XEOw==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "license": "MIT" + }, + "node_modules/universal-user-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-7.0.2.tgz", + "integrity": "sha512-0JCqzSKnStlRRQfCdowvqy3cy0Dvtlb8xecj/H8JFZuCze4rwjPZQOgvFvn0Ws/usCHQFGpyr+pB9adaGwXn4Q==", + "license": "ISC" + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "license": "BSD-2-Clause" + }, + "node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "license": "MIT", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + } + } +} diff --git a/scripts/eod-report-generator/package.json b/scripts/eod-report-generator/package.json new file mode 100644 index 00000000..bda35321 --- /dev/null +++ b/scripts/eod-report-generator/package.json @@ -0,0 +1,25 @@ +{ + "name": "eod-report-generator", + "version": "1.0.0", + "main": "index.js", + "type": "module", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "description": "", + "dependencies": { + "@anthropic-ai/sdk": "^0.32.1", + "@octokit/rest": "21.0.2", + "case": "^1.6.3", + "change-case": "^5.4.4", + "gray-matter": "^4.0.3", + "js-yaml": "^4.1.0", + "markdown-link-extractor": "^4.0.2", + "marked": "^14.1.2", + "openai": "^4.72.0", + "sharp": "0.32.6" + } +} diff --git a/scripts/eod-report-generator/readme.md b/scripts/eod-report-generator/readme.md new file mode 100644 index 00000000..6e4569f1 --- /dev/null +++ b/scripts/eod-report-generator/readme.md @@ -0,0 +1 @@ +This script generates a PR review report. \ No newline at end of file diff --git a/scripts/eod-report-generator/sonet.js b/scripts/eod-report-generator/sonet.js new file mode 100644 index 00000000..5d7d2eec --- /dev/null +++ b/scripts/eod-report-generator/sonet.js @@ -0,0 +1,39 @@ +import Anthropic from '@anthropic-ai/sdk'; + +const anthropic = new Anthropic({ + apiKey: '' || process.env.ANTHROPIC_KEY, +}); + +export async function extractHeadings(combinedString) { + const msg = await anthropic.messages.create({ + model: "claude-3-5-sonnet-20241022", + max_tokens: 1024, + messages: [{ role: "user", content: ` +Below are some markdown headings. Extract the headings which is using acronyms or shortcodes like K8s for kubernetes in it. + +Exclude the following acronysms ["API"] + +The output should be an array of strings. No explanation required. + +${combinedString}` }], + }); + + return msg.content[0].text; +} + +export async function formatName(prName) { + const msg = await anthropic.messages.create({ + model: "claude-3-5-sonnet-20241022", + max_tokens: 1024, + messages: [{ role: "user", content: ` + Hello, Claude. + + Remove the Jira ticket reference from the below string. Don't include any explanation text. The output will be read by a machine. + + ${prName} + ` }], + }); + + return msg.content[0].text; +} + diff --git a/merge_docs_configs.py b/scripts/merge_docs_configs.py similarity index 73% rename from merge_docs_configs.py rename to scripts/merge_docs_configs.py index 13ca290a..062cb6b3 100644 --- a/merge_docs_configs.py +++ b/scripts/merge_docs_configs.py @@ -12,8 +12,8 @@ def __init__(self, output_file: str = "docs.json", subfolder: str = "", addition self.output_file = output_file self.subfolder = subfolder.strip("/") if subfolder else "" # Remove leading/trailing slashes - # Default assets that should always go to root - default_assets = {"style.css", "images", "img", "logo", "favicon.ico", "favicon.png", "snippets"} + # Default assets that should always go to root (snippets handled separately) + default_assets = {"style.css", "images", "img", "logo", "favicon.ico", "favicon.png"} # Add any additional assets specified by user if additional_assets: @@ -23,8 +23,10 @@ def __init__(self, output_file: str = "docs.json", subfolder: str = "", addition # Define fallback version priority - will be overridden by branches config self.version_priority = [] # Start empty, populate from branches-config.json - self.version_labels = {} # Maps folder -> label + self.version_labels = {} # Maps target_folder -> label self.external_versions = {} # Store external version info + self.source_folders = {} # Maps target_folder -> source_folder + self.target_folders = {} # Maps source_folder -> target_folder self.latest_version = None self.main_version = None # Track main branch with most current assets @@ -52,6 +54,8 @@ def load_branches_config(self, branches_config_path: str) -> Dict: self.version_priority = [] self.version_labels = {} self.external_versions = {} # Store external version info + self.source_folders = {} # Maps target_folder -> source_folder + self.target_folders = {} # Maps source_folder -> target_folder for version_info in versions: is_external = version_info.get('isExternal', False) @@ -68,20 +72,41 @@ def load_branches_config(self, branches_config_path: str) -> Dict: } print(f"πŸ”— External version: {label} β†’ {external_url}") else: - # Handle local versions - folder = version_info.get('folder', '') - label = version_info.get('label', folder) + # Handle local versions with support for separate source/target folders + source_folder = version_info.get('sourceFolder', version_info.get('folder', '')) + target_folder = version_info.get('targetFolder', version_info.get('folder', '')) + + # Fallback to 'folder' if neither sourceFolder nor targetFolder specified + if not source_folder and not target_folder: + folder = version_info.get('folder', '') + source_folder = target_folder = folder + elif not source_folder: + source_folder = target_folder + elif not target_folder: + target_folder = source_folder + + label = version_info.get('label', target_folder) is_latest = version_info.get('isLatest', False) is_main = version_info.get('isMain', False) - if folder: - self.version_priority.append(folder) - self.version_labels[folder] = label + if source_folder and target_folder: + # Use target_folder for priority and labels (what appears in navigation) + self.version_priority.append(target_folder) + self.version_labels[target_folder] = label + + # Store folder mappings + self.source_folders[target_folder] = source_folder + self.target_folders[source_folder] = target_folder if is_latest: - self.latest_version = folder + self.latest_version = target_folder if is_main: - self.main_version = folder + self.main_version = target_folder + + if source_folder != target_folder: + print(f"πŸ“ Version {target_folder}: source='{source_folder}' β†’ target='{target_folder}'") + else: + print(f"πŸ“ Version {target_folder}: unified folder='{source_folder}'") print(f"πŸ“‹ Local version priority: {self.version_priority}") print(f"🏷️ Local version labels: {self.version_labels}") @@ -131,22 +156,21 @@ def collect_from_branches_config(self, branches_config_path: str, base_dir: str base_path = Path(base_dir) - # Process each version from the config - for version_info in branches_config.get('versions', []): - folder = version_info.get('folder', '') - if not folder: - continue + # Process each target version (what appears in navigation) + for target_version in self.version_priority: + # Get the source folder for this target version + source_folder = self.source_folders.get(target_version, target_version) - version_dir = base_path / folder + version_dir = base_path / source_folder config_file = version_dir / "docs.json" if config_file.exists(): - config = self.load_version_config(str(config_file), folder) + config = self.load_version_config(str(config_file), target_version) if config: - configs[folder] = config + configs[target_version] = config else: - label = self.version_labels.get(folder, folder) - print(f"⚠️ Config file not found: {config_file} for {label}") + label = self.version_labels.get(target_version, target_version) + print(f"⚠️ Config file not found: {config_file} for {label} (source: {source_folder})") return configs @@ -238,7 +262,9 @@ def merge_assets_from_all_versions(self, version_configs: Dict[str, Dict], all_files = {} # filename -> [(version, filepath, size)] for version in priority_versions: - version_dir = source_path / version + # Get the source folder for this target version + source_folder = self.source_folders.get(version, version) + version_dir = source_path / source_folder # Check if asset_type is a directory or individual file asset_path = version_dir / asset_type @@ -276,7 +302,7 @@ def merge_assets_from_all_versions(self, version_configs: Dict[str, Dict], continue # Handle individual files (like favicon.png, style.css) - is_individual_file = any((source_path / v / asset_type).is_file() for v in priority_versions) + is_individual_file = any((source_path / self.source_folders.get(v, v) / asset_type).is_file() for v in priority_versions) if is_individual_file: # Handle individual files - copy directly to target root @@ -351,6 +377,116 @@ def merge_assets_from_all_versions(self, version_configs: Dict[str, Dict], if files: print(f" πŸ“‚ {asset_type}: {len(files)} files") + def copy_snippets_with_version_structure(self, version_configs: Dict[str, Dict], + source_dir: str = ".", target_dir: str = ".") -> None: + """Copy snippets to version-specific directories with link rewriting.""" + + # Build priority order: main -> latest -> others + priority_versions = [] + + if self.main_version and self.main_version in version_configs: + priority_versions.append(self.main_version) + + if self.latest_version and self.latest_version in version_configs and self.latest_version != self.main_version: + priority_versions.append(self.latest_version) + + # Add remaining versions in config order + for version in self.version_priority: + if version in version_configs and version not in priority_versions: + priority_versions.append(version) + + if not priority_versions: + print("❌ No versions found for snippet processing") + return + + print(f"πŸ“ Processing snippets from {len(priority_versions)} versions...") + print(f"🎯 Priority order: {' > '.join(priority_versions)}") + + source_path = Path(source_dir) + target_root_path = Path(target_dir) + snippets_root = target_root_path / "snippets" + + # Track processed snippets for reporting + processed_snippets = {} # version -> {filename: status} + total_processed = 0 + + # Process each version + for version in priority_versions: + # Get the source folder for this target version + source_folder = self.source_folders.get(version, version) + version_source = source_path / source_folder / "snippets" + + if not version_source.exists(): + print(f" ⚠️ No snippets directory found in {source_folder} (target: {version})") + continue + + is_latest = (version == self.latest_version) + version_label = self.version_labels.get(version, version) + + # Create version-specific snippets directory + version_snippets_dir = snippets_root / version + version_snippets_dir.mkdir(parents=True, exist_ok=True) + + print(f"\n πŸ“‚ Processing snippets for {version} ({version_label})...") + + processed_snippets[version] = {} + version_count = 0 + + # Process all files in the snippets directory + for item in version_source.rglob("*"): + if item.is_file(): + # Get relative path from snippets root + rel_path = item.relative_to(version_source) + target_file = version_snippets_dir / rel_path + + # Create parent directories if needed + target_file.parent.mkdir(parents=True, exist_ok=True) + + try: + # Check if this is a content file that needs link rewriting + if item.suffix.lower() in ['.mdx', '.md']: + # Read, rewrite, and write content + with open(item, 'r', encoding='utf-8') as f: + content = f.read() + + # Rewrite internal links for this version + modified_content = self.rewrite_internal_links(content, version, is_latest) + + # Write modified content + with open(target_file, 'w', encoding='utf-8') as f: + f.write(modified_content) + + processed_snippets[version][str(rel_path)] = "rewritten" + print(f" πŸ“ Processed with link rewriting: {rel_path}") + else: + # For non-content files, just copy + shutil.copy2(item, target_file) + processed_snippets[version][str(rel_path)] = "copied" + print(f" πŸ“„ Copied: {rel_path}") + + version_count += 1 + total_processed += 1 + + except Exception as e: + print(f" ❌ Error processing {rel_path}: {e}") + processed_snippets[version][str(rel_path)] = f"error: {e}" + + if version_count > 0: + print(f" βœ… Processed {version_count} snippet files for {version}") + else: + print(f" ⚠️ No snippet files found in {version}") + + # Generate summary + print(f"\nβœ… Snippet processing complete:") + print(f" πŸ“ Total files processed: {total_processed}") + print(f" πŸ“‚ Versions processed: {len([v for v in processed_snippets.keys() if processed_snippets[v]])}") + + for version, files in processed_snippets.items(): + if files: + rewritten_count = len([f for f in files.values() if f == "rewritten"]) + copied_count = len([f for f in files.values() if f == "copied"]) + print(f" πŸ“‚ {version}: {len(files)} files ({rewritten_count} rewritten, {copied_count} copied)") + def organize_content_files(self, version_configs: Dict[str, Dict], source_dir: str = ".", target_dir: str = ".") -> None: """Copy latest version docs to subfolder and merge assets from all versions.""" @@ -369,16 +505,23 @@ def organize_content_files(self, version_configs: Dict[str, Dict], source_path = Path(source_dir) target_root_path = Path(target_dir) - latest_source = source_path / latest_version + + # Get the source folder for the latest version + latest_source_folder = self.source_folders.get(latest_version, latest_version) + latest_source = source_path / latest_source_folder if not latest_source.exists(): - print(f"❌ Latest version directory {latest_source} not found") + print(f"❌ Latest version directory {latest_source} not found (target: {latest_version}, source: {latest_source_folder})") return # Step 1: Merge assets from all versions print("🎨 Step 1: Merging assets from all versions...") self.merge_assets_from_all_versions(version_configs, source_dir, target_dir) + # Step 1.5: Process snippets with version structure + print(f"\nπŸ“ Step 1.5: Processing snippets with version structure...") + self.copy_snippets_with_version_structure(version_configs, source_dir, target_dir) + # Step 2: Copy documentation content from all versions print(f"\nπŸ“š Step 2: Copying documentation content from all versions...") @@ -389,9 +532,12 @@ def organize_content_files(self, version_configs: Dict[str, Dict], if version not in version_configs: continue - version_source = source_path / version + # Get the source folder for this target version + source_folder = self.source_folders.get(version, version) + version_source = source_path / source_folder + if not version_source.exists(): - print(f"⚠️ Version directory {version_source} not found") + print(f"⚠️ Version directory {version_source} not found (target: {version}, source: {source_folder})") continue is_latest = (version == latest_version) @@ -402,10 +548,10 @@ def organize_content_files(self, version_configs: Dict[str, Dict], else: print(f"\n πŸ“‚ Processing older version ({version} - {version_label})...") - # Define system files to exclude - exclude_items = {"docs.json", ".git", ".gitignore", ".DS_Store", "__pycache__"} + # Define system files to exclude (including snippets since they're processed separately) + exclude_items = {"docs.json", ".git", ".gitignore", ".DS_Store", "__pycache__", "snippets"} - # Get list of items to copy (exclude assets since they're already merged) + # Get list of items to copy (exclude assets and snippets since they're already processed) items_to_process = [] for item in version_source.iterdir(): if item.name not in exclude_items and item.name not in self.assets: @@ -448,8 +594,10 @@ def organize_content_files(self, version_configs: Dict[str, Dict], if target_item.exists(): print(f" πŸ—‘οΈ Removing existing: {item.name}/") shutil.rmtree(target_item) - shutil.copytree(item, target_item) - print(f" πŸ“‚ Copied directory: {item.name}/ β†’ {location_desc}") + + # Copy directory with recursive link rewriting + self.copy_directory_with_link_rewriting(item, target_item, version, is_latest) + print(f" πŸ“‚ Copied directory with link rewriting: {item.name}/ β†’ {location_desc}") version_copied += 1 else: # Copy file with link rewriting for content files @@ -464,8 +612,11 @@ def organize_content_files(self, version_configs: Dict[str, Dict], with open(item, 'r', encoding='utf-8') as f: content = f.read() - # Rewrite internal links - modified_content = self.rewrite_internal_links(content, version, is_latest) + # First: Rewrite snippet imports (always uses version) + modified_content = self.rewrite_snippet_imports(content, version) + + # Then: Rewrite internal links (uses subfolder logic) + modified_content = self.rewrite_internal_links(modified_content, version, is_latest) # Write modified content with open(target_item, 'w', encoding='utf-8') as f: @@ -705,9 +856,9 @@ def add_redirect(source, destination, permanent=False): if latest_version: # Find a good default page from latest version config latest_config = version_configs[latest_version] - default_page = "getting-started/" + default_page = None - # Try to find a better default from navigation + # Try to find a default from navigation nav = latest_config.get("navigation", {}) if "tabs" in nav and nav["tabs"]: first_tab = nav["tabs"][0] @@ -718,23 +869,15 @@ def add_redirect(source, destination, permanent=False): if isinstance(first_page, str): default_page = first_page - # Build the destination path with subfolder prefix - destination_path = f"/{self.subfolder}/{default_page}" if self.subfolder else f"/{default_page}" + # Only create root redirect if we found a default page + if default_page: + # Build the destination path with subfolder prefix + destination_path = f"/{self.subfolder}/{default_page}" if self.subfolder else f"/{default_page}" - # Root redirect to default page (with subfolder prefix if applicable) - add_redirect("/", destination_path, False) + # Root redirect to default page (with subfolder prefix if applicable) + add_redirect("/", destination_path, False) - # Latest alias redirects - if self.subfolder: - # Redirect /latest/* to /subfolder/:splat - add_redirect("/latest/*", f"/{self.subfolder}/:splat", False) - - # Redirect versioned latest URLs to subfolder paths - add_redirect(f"/{latest_version}/*", f"/{self.subfolder}/:splat", True) - else: - # Standard redirects when no subfolder - add_redirect("/latest/*", "/:splat", False) - add_redirect(f"/{latest_version}/*", "/:splat", True) + # No automatic latest/version redirects - keep it simple # Collect version-specific redirects for version, config in version_configs.items(): @@ -797,6 +940,56 @@ def clean_config(self, config: Dict) -> Dict: cleaned[key] = value return cleaned + def rewrite_snippet_imports(self, content, version): + """Dedicated function for snippet import rewriting - always uses version""" + import re + + print(f" πŸ“ Processing snippet imports for version: {version}") + changes_made = 0 + + def replace_snippet_import(match): + nonlocal changes_made + # Extract the import parts + import_part = match.group(1) # "import Something" or "import { Something }" + quote = match.group(2) # Quote character (' or ") + snippet_path = match.group(3) # The snippet path + + print(f" πŸ” Found snippet import: {snippet_path}") + + # Only process snippet imports that start with /snippets/ + if not snippet_path.startswith('/snippets/'): + print(f" ⚠️ Skipping non-snippet import: {snippet_path}") + return match.group(0) + + # Check if already has version by looking for known version patterns + path_parts = snippet_path.split('/') + print(f" πŸ” Path parts: {path_parts} (length: {len(path_parts)})") + + # Check if the third part (index 2) looks like a version + if len(path_parts) >= 4: # ['', 'snippets', 'potential_version', 'file.mdx'] + potential_version = path_parts[2] + # Check if it looks like a version (contains digits or is 'nightly') + if potential_version == 'nightly' or any(char.isdigit() for char in potential_version): + print(f" βœ… Already has version '{potential_version}', skipping: {snippet_path}") + return match.group(0) # Already versioned + + # Extract filename after /snippets/ + snippet_file = snippet_path[10:] # Remove '/snippets/' prefix + new_snippet_path = f"/snippets/{version}/{snippet_file}" + + changes_made += 1 + print(f" πŸ“ Snippet import: {snippet_path} β†’ {new_snippet_path}") + return f'{import_part} from {quote}{new_snippet_path}{quote}' + + # Match both import patterns: + # import Something from '/snippets/...' + # import { Something } from '/snippets/...' + pattern = r'(import\s+(?:\{[^}]+\}|\w+))\s+from\s+(["\'])(/snippets/[^"\']+)\2' + content = re.sub(pattern, replace_snippet_import, content) + + print(f" πŸ“ Snippet imports: {changes_made} changes made") + return content + def rewrite_internal_links(self, content, version, is_latest): """Rewrite internal links to include subfolder and version paths""" import re @@ -865,25 +1058,27 @@ def replace_markdown(match): text = match.group(1) path = match.group(2) + # Strip leading/trailing whitespace from path + path_stripped = path.strip() + # Skip external links and mailto (including angle bracket wrapped) - if (path.startswith('http') or path.startswith('//') or - path.startswith('<mailto:') or 'mailto:' in path): + if (path_stripped.startswith('http') or path_stripped.startswith('//') or + path_stripped.startswith('<mailto:') or 'mailto:' in path_stripped): + return match.group(0) + + # Only process actual internal links that start with / + if not path_stripped.startswith('/'): return match.group(0) changes_made += 1 - return f'[{text}]({prefix}{path})' + return f'[{text}]({prefix}{path_stripped})' content = re.sub(r'\[([^\]]+)\]\((/[^)]*)\)', replace_markdown, content) - # 3. Fix data paths: path: "/path" or path: 'path' - def replace_path(match): - nonlocal changes_made - quote = match.group(1) - path = match.group(2) - changes_made += 1 - return f'path: {quote}{prefix}/{path.lstrip("/")}{quote}' - - content = re.sub(r'path:\s*(["\'])/?([^"\']+)\1', replace_path, content) + # 3. Skip data paths - keep them clean, let JSX handle prefixing + # This prevents double prefixing by not modifying path: "..." declarations + # The JSX template literals will add the prefix instead + print(f" ⚠️ Skipping data path prefixing to prevent double prefixes") # 4. Fix template string hrefs: href={`/${path}`} def replace_template(match): @@ -940,16 +1135,23 @@ def replace_relative_markdown(match): text = match.group(1) path = match.group(2) + # Strip leading/trailing whitespace from path + path_stripped = path.strip() + # Skip external links and already processed absolute paths - if (path.startswith('http') or path.startswith('//') or - path.startswith('mailto:') or path.startswith('<mailto:') or - path.startswith('/') or path.startswith(prefix)): + if (path_stripped.startswith('http') or path_stripped.startswith('//') or + path_stripped.startswith('mailto:') or path_stripped.startswith('<mailto:') or + path_stripped.startswith('/') or path_stripped.startswith(prefix)): return match.group(0) changes_made += 1 - return f'[{text}]({prefix}/{path})' + return f'[{text}]({prefix}/{path_stripped})' - content = re.sub(r'\[([^\]]+)\]\(([^)]+)\)', replace_relative_markdown, content) + # This regex is too broad and conflicts with the absolute path regex above + # It should only match relative paths (not starting with /) + content = re.sub(r'\[([^\]]+)\]\(([^/)][^)]*)\)', replace_relative_markdown, content) + + # Note: Snippet imports are now handled by the dedicated rewrite_snippet_imports function print(f" βœ… Link rewriting complete: {changes_made} changes made") @@ -964,6 +1166,39 @@ def replace_relative_markdown(match): return content + def copy_directory_with_link_rewriting(self, source_dir: Path, target_dir: Path, version: str, is_latest: bool) -> None: + """Recursively copy directory with link rewriting for content files.""" + target_dir.mkdir(parents=True, exist_ok=True) + + for item in source_dir.iterdir(): + target_item = target_dir / item.name + + if item.is_dir(): + # Recursively copy subdirectories + self.copy_directory_with_link_rewriting(item, target_item, version, is_latest) + else: + # Copy file with link rewriting if it's a content file + if item.suffix.lower() in ['.mdx', '.md']: + try: + # Read, rewrite, and write content + with open(item, 'r', encoding='utf-8') as f: + content = f.read() + + # First: Rewrite snippet imports (always uses version) + modified_content = self.rewrite_snippet_imports(content, version) + + # Then: Rewrite internal links (uses subfolder logic) + modified_content = self.rewrite_internal_links(modified_content, version, is_latest) + + # Write modified content + with open(target_item, 'w', encoding='utf-8') as f: + f.write(modified_content) + except Exception as e: + print(f" ⚠️ Error processing {item.relative_to(source_dir)}, copying as-is: {e}") + shutil.copy2(item, target_item) + else: + # For non-content files, just copy + shutil.copy2(item, target_item) def create_unified_config(self, version_configs: Dict[str, Dict]) -> Dict: """Create unified configuration with version-based navigation.""" @@ -998,7 +1233,8 @@ def create_unified_config(self, version_configs: Dict[str, Dict]) -> Dict: optional_fields = [ "theme", "logo", "favicon", "colors", "description", "api", "integrations", "styling", "search", "seo", - "background", "fonts", "appearance", "footer", "banner" + "background", "fonts", "appearance", "footer", "banner", "navbar", "contextual","thumbnails","icons", + "errors" ] for field in optional_fields: @@ -1142,4 +1378,4 @@ def main(): ) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/scripts/validate_mintlify_docs.py b/scripts/validate_mintlify_docs.py new file mode 100644 index 00000000..88683493 --- /dev/null +++ b/scripts/validate_mintlify_docs.py @@ -0,0 +1,810 @@ +#!/usr/bin/env python3 +""" +Script to check for broken links in converted Mintlify documentation. + + +WHAT THIS SCRIPT VALIDATES: + +1. BROKEN LINKS & IMAGES: + - Scans all MDX files for internal links and verifies they exist + - Checks image references and verifies image files exist + - Handles various file patterns (.mdx extensions, index files, etc.) + - Considers redirects when checking link existence + +2. DOCS.JSON VALIDATION (--validate-redirects flag): + - Self-referencing redirects (source = destination) + - Navigation-redirect conflicts (same path in both navigation and redirects) + - Invalid redirects (empty source or destination) + - Missing navigation files (navigation entries pointing to non-existent files) + - Missing redirect destinations (redirect destinations pointing to non-existent files) + +3. COMPREHENSIVE CHECKS: + - Validates all navigation links point to existing files + - Ensures all redirect destinations are valid and exist + - Skips external URLs, anchors, and special protocols appropriately + - Handles anchor fragments and query parameters correctly + +USAGE: + python validate_mintlify_docs.py [directory] [options] + make check-redirects # Validate redirects and navigation only + make validate-all # Full validation with verbose output + python scripts/validate_mintlify_docs.py . --external-links --external-timeout 5 --external-delay 0.5 +""" + +import os +import re +import glob +import json +import argparse +import urllib.parse +import requests +import time +from pathlib import Path +from typing import Set, List, Tuple, Dict, Any + +def remove_comments_and_code(content: str) -> str: + """Remove JSX comments, HTML comments, and code blocks from content.""" + # Remove JSX comments: {/* ... */} (including multi-line) + jsx_comment_pattern = r'\{/\*.*?\*/\}' + content = re.sub(jsx_comment_pattern, '', content, flags=re.DOTALL) + + # Remove HTML comments: <!-- ... --> (including multi-line) + html_comment_pattern = r'<!--.*?-->' + content = re.sub(html_comment_pattern, '', content, flags=re.DOTALL) + + # Remove fenced code blocks: ``` ... ``` (including multi-line) + fenced_code_pattern = r'```.*?```' + content = re.sub(fenced_code_pattern, '', content, flags=re.DOTALL) + + # Remove inline code: ` ... ` + inline_code_pattern = r'`[^`\n]*`' + content = re.sub(inline_code_pattern, '', content) + + return content + +def find_internal_links(mdx_content: str) -> Set[str]: + """Extract all internal links from MDX content, excluding links in comments and code blocks.""" + # Remove comments and code blocks first to avoid parsing commented-out or code example links + clean_content = remove_comments_and_code(mdx_content) + + internal_links = set() + + # Pattern for markdown links: [text](path) - exclude external URLs + markdown_pattern = r'\[([^\]]*)\]\(([^)]+)\)' + + # Pattern for HTML anchor tags: <a href="path"> + html_pattern = r'<a[^>]+href=["\']([^"\']+)["\']' + + # Pattern for Card components: <Card href="path" ...> + card_pattern = r'<Card[^>]+href=["\']([^"\']+)["\']' + + # Find all matches + for pattern in [markdown_pattern, html_pattern, card_pattern]: + matches = re.findall(pattern, clean_content, re.IGNORECASE) + for match in matches: + # For markdown pattern, match[1] is the URL, for HTML/Card patterns, match is the URL + url = match[1] if isinstance(match, tuple) and len(match) > 1 else match + + # Skip external URLs, anchors, and special protocols (including malformed mailto) + if (not url.startswith(('http://', 'https://', 'mailto:', 'tel:', 'ftp://')) and + not url.startswith('#') and + not url.startswith('<mailto:') and # Skip malformed mailto links + url.strip()): + # Clean up the path + clean_url = url.strip().strip('"\'') + + # Skip external URLs after cleaning (handles spaces before URLs) + if clean_url.startswith(('http://', 'https://', 'mailto:', 'tel:', 'ftp://')): + continue + + # Remove URL fragments (anchors) + if '#' in clean_url: + clean_url = clean_url.split('#')[0] + # Remove query parameters + if '?' in clean_url: + clean_url = clean_url.split('?')[0] + # Remove leading slash if present (normalize paths) + if clean_url.startswith('/'): + clean_url = clean_url[1:] + # Remove trailing slash if present (normalize paths) + if clean_url.endswith('/'): + clean_url = clean_url[:-1] + if clean_url: # Only add non-empty paths + internal_links.add(clean_url) + + return internal_links + +def find_image_references(mdx_content: str) -> Set[str]: + """Extract all image references from MDX content, excluding images in comments and code blocks.""" + # Remove comments and code blocks first to avoid parsing commented-out or code example images + clean_content = remove_comments_and_code(mdx_content) + + image_refs = set() + + # Pattern for markdown images: ![alt](path) + markdown_pattern = r'!\[.*?\]\(([^)]+)\)' + + # Pattern for HTML img tags: <img src="path" ...> + html_pattern = r'<img[^>]+src=["\']([^"\']+)["\']' + + # Pattern for JSX img elements: <img src={"/path"} or <img src="/path" + jsx_pattern = r'<img[^>]+src=\{?["\']([^"\']+)["\']' + + # Pattern for Card components: <Card img="path" ...> + card_pattern = r'<Card[^>]+img=["\']([^"\']+)["\']' + + # Find all matches + for pattern in [markdown_pattern, html_pattern, jsx_pattern, card_pattern]: + matches = re.findall(pattern, clean_content, re.IGNORECASE) + for match in matches: + # Skip external URLs + if not match.startswith(('http://', 'https://', 'data:')): + # Clean up the path + clean_path = match.strip().strip('"\'') + # Remove leading slash if present (normalize paths) + if clean_path.startswith('/'): + clean_path = clean_path[1:] + image_refs.add(clean_path) + + return image_refs + +def find_external_links(mdx_content: str) -> Set[str]: + """Extract all external HTTP/HTTPS links from MDX content, excluding links in comments and code blocks.""" + # Remove comments and code blocks first to avoid parsing commented-out or code example links + clean_content = remove_comments_and_code(mdx_content) + + external_links = set() + + # Pattern for markdown links: [text](https://...) + markdown_pattern = r'\[([^\]]*)\]\((https?://[^)]+)\)' + + # Pattern for HTML anchor tags: <a href="https://..."> + html_pattern = r'<a[^>]+href=["\']?(https?://[^"\'>\s]+)["\']?' + + # Pattern for Card components: <Card href="https://..." ...> + card_pattern = r'<Card[^>]+href=["\']?(https?://[^"\'>\s]+)["\']?' + + # Find all matches + for pattern in [markdown_pattern, html_pattern, card_pattern]: + matches = re.findall(pattern, clean_content, re.IGNORECASE) + for match in matches: + # For markdown pattern, match[1] is the URL, for HTML/Card patterns, match is the URL + url = match[1] if isinstance(match, tuple) and len(match) > 1 else match + + # Clean up the URL + clean_url = url.strip().strip('"\'') + if clean_url: + external_links.add(clean_url) + + return external_links + +def scan_mdx_files(directory: str, check_external: bool = False) -> Tuple[Dict[str, Set[str]], Dict[str, Set[str]], Dict[str, Set[str]]]: + """Scan all MDX files and return links, images, and external links by file.""" + file_links = {} + file_images = {} + file_external_links = {} + mdx_files = glob.glob(os.path.join(directory, '**/*.mdx'), recursive=True) + + print(f"Scanning {len(mdx_files)} MDX files for links and images...") + + for mdx_file in mdx_files: + try: + with open(mdx_file, 'r', encoding='utf-8') as f: + content = f.read() + + # Get relative path for reporting + rel_path = os.path.relpath(mdx_file, directory) + + # Find links and images + links = find_internal_links(content) + images = find_image_references(content) + + if links: + file_links[rel_path] = links + if images: + file_images[rel_path] = images + + # Find external links if requested + if check_external: + external_links = find_external_links(content) + if external_links: + file_external_links[rel_path] = external_links + + except Exception as e: + print(f"Error reading {mdx_file}: {e}") + + return file_links, file_images, file_external_links + +def load_redirects(directory: str) -> Dict[str, str]: + """Load redirects from docs.json or mint.json configuration files.""" + redirects = {} + + # Look for configuration files + config_files = ['docs.json', 'mint.json'] + + for config_file in config_files: + config_path = os.path.join(directory, config_file) + if os.path.exists(config_path): + try: + with open(config_path, 'r', encoding='utf-8') as f: + config = json.load(f) + + # Extract redirects if they exist + if 'redirects' in config: + for redirect in config['redirects']: + if 'source' in redirect and 'destination' in redirect: + source = redirect['source'].strip('/') + destination = redirect['destination'].strip('/') + redirects[source] = destination + + print(f"Loaded {len(redirects)} redirects from {config_file}") + break # Use first config file found + + except Exception as e: + print(f"Error reading {config_file}: {e}") + + return redirects + +def find_existing_files(directory: str) -> Set[str]: + """Find all existing files in the directory.""" + existing_files = set() + + for root, dirs, files in os.walk(directory): + for file in files: + # Get relative path from the base directory + rel_path = os.path.relpath(os.path.join(root, file), directory) + existing_files.add(rel_path) + + # Also add without extension for MDX files (common linking pattern) + if file.endswith('.mdx'): + rel_path_no_ext = rel_path[:-4] # Remove .mdx + existing_files.add(rel_path_no_ext) + + return existing_files + +def check_link_exists(link: str, existing_files: Set[str], redirects: Dict[str, str]) -> bool: + """Check if a link exists, considering redirects and various file patterns.""" + + # Check exact match + if link in existing_files: + return True + + # Check with .mdx extension + if f"{link}.mdx" in existing_files: + return True + + # Check if it's a directory with index file + if f"{link}/index.mdx" in existing_files: + return True + + if f"{link}/index" in existing_files: + return True + + # Check if there's a redirect for this link + if link in redirects: + destination = redirects[link] + # Recursively check if the redirect destination exists + return check_link_exists(destination, existing_files, redirects) + + # Check if link with leading slash has a redirect + if f"/{link}" in redirects: + destination = redirects[f"/{link}"] + return check_link_exists(destination.lstrip('/'), existing_files, redirects) + + return False + +def check_external_link(url: str, timeout: int = 10) -> Dict[str, Any]: + """Check if an external URL is accessible.""" + try: + # Use HEAD request for faster checking + headers = { + 'User-Agent': 'Mozilla/5.0 (compatible; Mintlify-Link-Checker/1.0)' + } + response = requests.head(url, timeout=timeout, allow_redirects=True, headers=headers) + + return { + 'url': url, + 'status_code': response.status_code, + 'accessible': response.status_code < 400, + 'final_url': response.url if response.url != url else None, + 'error': None, + 'response_time': None + } + except requests.exceptions.Timeout: + return { + 'url': url, + 'status_code': None, + 'accessible': False, + 'final_url': None, + 'error': f'Timeout after {timeout}s', + 'response_time': None + } + except requests.exceptions.ConnectionError: + return { + 'url': url, + 'status_code': None, + 'accessible': False, + 'final_url': None, + 'error': 'Connection refused', + 'response_time': None + } + except Exception as e: + return { + 'url': url, + 'status_code': None, + 'accessible': False, + 'final_url': None, + 'error': str(e), + 'response_time': None + } + +def check_external_links(file_external_links: Dict[str, Set[str]], timeout: int = 10, delay: float = 1.0) -> Dict[str, List[Dict[str, Any]]]: + """Check all external links and return results by file.""" + # Collect all unique URLs to avoid duplicate checking + all_urls = set() + for urls in file_external_links.values(): + all_urls.update(urls) + + if not all_urls: + return {} + + print(f"\nπŸ” Checking {len(all_urls)} unique external URLs...") + + # Check each unique URL + url_results = {} + for i, url in enumerate(sorted(all_urls), 1): + print(f" [{i}/{len(all_urls)}] {url[:60]}{'...' if len(url) > 60 else ''}", end=' ') + + start_time = time.time() + result = check_external_link(url, timeout) + result['response_time'] = time.time() - start_time + + # Print status + if result['accessible']: + if result['final_url']: + print(f"πŸ”„ {result['status_code']} β†’ {result['final_url'][:40]}{'...' if len(result['final_url']) > 40 else ''}") + else: + print(f"βœ… {result['status_code']}") + else: + if result['error']: + print(f"❌ {result['error']}") + else: + print(f"❌ {result['status_code']}") + + url_results[url] = result + + # Rate limiting + if i < len(all_urls): + time.sleep(delay) + + # Map results back to files + file_results = {} + for file_path, urls in file_external_links.items(): + file_results[file_path] = [url_results[url] for url in urls] + + return file_results + +def check_broken_links(base_dir: str, check_external: bool = False, external_timeout: int = 10, external_delay: float = 1.0) -> Tuple[Dict[str, List[str]], Dict[str, List[str]], Dict[str, str], Dict[str, List[Dict[str, Any]]]]: + """Check for broken internal links, missing images, and optionally external links.""" + print(f"Checking for broken links in: {base_dir}") + + # Load redirects from configuration + redirects = load_redirects(base_dir) + + # Scan all MDX files + file_links, file_images, file_external_links = scan_mdx_files(base_dir, check_external) + + # Find all existing files + existing_files = find_existing_files(base_dir) + + # Check for broken links + broken_links = {} + broken_images = {} + + # Check internal links + for file_path, links in file_links.items(): + file_broken_links = [] + for link in links: + if not check_link_exists(link, existing_files, redirects): + file_broken_links.append(link) + + if file_broken_links: + broken_links[file_path] = file_broken_links + + # Check image references + for file_path, images in file_images.items(): + file_broken_images = [] + for image in images: + if image not in existing_files: + file_broken_images.append(image) + + if file_broken_images: + broken_images[file_path] = file_broken_images + + # Check external links if requested + external_results = {} + if check_external and file_external_links: + external_results = check_external_links(file_external_links, external_timeout, external_delay) + + return broken_links, broken_images, redirects, external_results + + +def extract_navigation_links(navigation: Dict[str, Any]) -> Set[str]: + """Extract all navigation links from the navigation structure.""" + links = set() + + def process_item(item): + if isinstance(item, str): + # Direct page reference + links.add(item) + elif isinstance(item, dict): + if 'pages' in item: + # Group with pages + for page in item['pages']: + process_item(page) + elif 'groups' in item: + # Tab with groups + for group in item['groups']: + process_item(group) + + # Process tabs + if 'tabs' in navigation: + for tab in navigation['tabs']: + if 'pages' in tab: + for page in tab['pages']: + process_item(page) + if 'groups' in tab: + for group in tab['groups']: + process_item(group) + + return links + + +def validate_navigation_files(directory: str, navigation: Dict[str, Any]) -> List[str]: + """Check if all navigation entries point to existing files.""" + missing_files = [] + existing_files = find_existing_files(directory) + redirects = load_redirects(directory) + + nav_links = extract_navigation_links(navigation) + + for link in nav_links: + # Clean the link path (remove leading slash if present) + clean_link = link.lstrip('/') + + # Check if the navigation file exists using the same logic as broken link checking + if not check_link_exists(clean_link, existing_files, redirects): + missing_files.append(link) + + return missing_files + + +def validate_redirect_destinations(directory: str, redirects_list: List[Dict[str, str]]) -> List[Dict[str, str]]: + """Check if all redirect destinations point to existing files.""" + missing_destinations = [] + existing_files = find_existing_files(directory) + redirects_dict = load_redirects(directory) + + for redirect in redirects_list: + source = redirect.get('source', '') + destination = redirect.get('destination', '') + + if not destination: + continue # Skip invalid redirects (handled elsewhere) + + # Skip external URLs + if destination.startswith(('http://', 'https://', 'mailto:', 'tel:', 'ftp://')): + continue + + # Skip anchor-only links + if destination.startswith('#'): + continue + + # Clean the destination path (remove leading slash if present) + clean_destination = destination.lstrip('/') + + # Remove anchor fragments from destination for file existence check + if '#' in clean_destination: + clean_destination = clean_destination.split('#')[0] + + # Remove query parameters from destination for file existence check + if '?' in clean_destination: + clean_destination = clean_destination.split('?')[0] + + # Skip empty destinations after cleaning + if not clean_destination: + continue + + # Check if the redirect destination exists using the same logic as broken link checking + if not check_link_exists(clean_destination, existing_files, redirects_dict): + missing_destinations.append({ + 'source': source, + 'destination': destination, + 'clean_destination': clean_destination + }) + + return missing_destinations + + +def validate_docs_json(directory: str) -> Dict[str, Any]: + """Validate docs.json for problematic redirects and navigation conflicts.""" + docs_json_path = os.path.join(directory, 'docs.json') + + if not os.path.exists(docs_json_path): + return { + 'error': f'docs.json not found in {directory}', + 'self_referencing_redirects': [], + 'navigation_redirect_conflicts': [], + 'invalid_redirects': [], + 'missing_navigation_files': [] + } + + try: + with open(docs_json_path, 'r', encoding='utf-8') as f: + docs_config = json.load(f) + except Exception as e: + return { + 'error': f'Error reading docs.json: {e}', + 'self_referencing_redirects': [], + 'navigation_redirect_conflicts': [], + 'invalid_redirects': [], + 'missing_navigation_files': [] + } + + # Extract navigation and redirects + navigation = docs_config.get('navigation', {}) + redirects = docs_config.get('redirects', []) + + # Find self-referencing redirects + self_referencing = [] + invalid_redirects = [] + + for redirect in redirects: + source = redirect.get('source', '') + destination = redirect.get('destination', '') + + # Check for self-referencing redirects + if source == destination: + self_referencing.append(redirect) + + # Check for invalid redirects (empty source or destination) + if not source or not destination: + invalid_redirects.append(redirect) + + # Find navigation-redirect conflicts + nav_links = extract_navigation_links(navigation) + nav_paths = set() + + # Convert nav links to paths with leading slash + for link in nav_links: + if not link.startswith('/'): + nav_paths.add('/' + link) + else: + nav_paths.add(link) + + # Find conflicts + conflicts = [] + for redirect in redirects: + source = redirect.get('source', '') + if source in nav_paths: + conflicts.append({ + 'source': source, + 'destination': redirect.get('destination', ''), + 'redirect': redirect + }) + + # Check for missing navigation files + missing_nav_files = validate_navigation_files(directory, navigation) + + # Check for missing redirect destinations + missing_redirect_destinations = validate_redirect_destinations(directory, redirects) + + return { + 'total_redirects': len(redirects), + 'total_navigation_links': len(nav_links), + 'self_referencing_redirects': self_referencing, + 'navigation_redirect_conflicts': conflicts, + 'invalid_redirects': invalid_redirects, + 'missing_navigation_files': missing_nav_files, + 'missing_redirect_destinations': missing_redirect_destinations + } + + +def main(): + parser = argparse.ArgumentParser(description='Check for broken links in converted documentation') + parser.add_argument('directory', nargs='?', default='converted', help='Directory to scan (default: converted)') + parser.add_argument('--verbose', '-v', action='store_true', help='Show detailed output') + parser.add_argument('--images-only', action='store_true', help='Only check for broken images') + parser.add_argument('--links-only', action='store_true', help='Only check for broken links') + parser.add_argument('--validate-redirects', action='store_true', help='Validate docs.json for problematic redirects') + parser.add_argument('--external-links', action='store_true', help='Check external HTTP/HTTPS links') + parser.add_argument('--external-timeout', type=int, default=10, help='Timeout for external link requests (default: 10s)') + parser.add_argument('--external-delay', type=float, default=1.0, help='Delay between external link requests (default: 1.0s)') + parser.add_argument('--external-only', action='store_true', help='Only check external links') + parser.add_argument('--external-errors-only', action='store_true', help='Only show failed external links') + + args = parser.parse_args() + + if not os.path.exists(args.directory): + print(f"Error: Directory '{args.directory}' does not exist") + return 1 + + # Check for broken links and images + broken_links, broken_images, redirects, external_results = check_broken_links( + args.directory, + check_external=args.external_links or args.external_only, + external_timeout=args.external_timeout, + external_delay=args.external_delay + ) + + # Validate docs.json if requested + validation_results = None + if args.validate_redirects: + validation_results = validate_docs_json(args.directory) + + # Report results + print(f"\n{'='*60}") + print("BROKEN LINK ANALYSIS RESULTS") + print(f"{'='*60}") + + total_broken_links = sum(len(links) for links in broken_links.values()) + total_broken_images = sum(len(images) for images in broken_images.values()) + + if not args.images_only and not args.external_only and broken_links: + print(f"\nπŸ”— BROKEN INTERNAL LINKS ({total_broken_links} total):") + for file_path, links in sorted(broken_links.items()): + print(f"\n πŸ“„ {file_path}:") + for link in sorted(links): + print(f" ❌ {link}") + + if not args.links_only and not args.external_only and broken_images: + print(f"\nπŸ–ΌοΈ BROKEN IMAGE REFERENCES ({total_broken_images} total):") + for file_path, images in sorted(broken_images.items()): + print(f"\n πŸ“„ {file_path}:") + for image in sorted(images): + print(f" ❌ {image}") + + # Report external link results + if external_results: + # Count results + total_external = sum(len(results) for results in external_results.values()) + successful_external = sum(1 for results in external_results.values() for result in results if result['accessible']) + failed_external = total_external - successful_external + redirected_external = sum(1 for results in external_results.values() for result in results if result['final_url']) + + if args.external_errors_only: + # Show only failed external links + print(f"\n🌐 FAILED EXTERNAL LINKS ({failed_external} total):") + for file_path, results in sorted(external_results.items()): + failed_results = [r for r in results if not r['accessible']] + if failed_results: + print(f"\n πŸ“„ {file_path}:") + for result in failed_results: + if result['error']: + print(f" ❌ {result['url']} ({result['error']})") + else: + print(f" ❌ {result['url']} ({result['status_code']})") + else: + # Show all external link results + print(f"\n🌐 EXTERNAL LINK VALIDATION RESULTS ({total_external} total):") + for file_path, results in sorted(external_results.items()): + print(f"\n πŸ“„ {file_path}:") + for result in results: + if result['accessible']: + if result['final_url']: + print(f" πŸ”„ {result['url']} ({result['status_code']} β†’ {result['final_url']})") + else: + print(f" βœ… {result['url']} ({result['status_code']})") + else: + if result['error']: + print(f" ❌ {result['url']} ({result['error']})") + else: + print(f" ❌ {result['url']} ({result['status_code']})") + + # Show external link summary + print(f"\nπŸ“Š EXTERNAL LINK SUMMARY:") + print(f" βœ… Successful (2xx): {successful_external} links") + print(f" πŸ”„ Redirects (3xx): {redirected_external} links") + print(f" ❌ Failed: {failed_external} links") + + # Report validation results + if validation_results: + print(f"\n{'='*60}") + print("DOCS.JSON VALIDATION RESULTS") + print(f"{'='*60}") + + if 'error' in validation_results: + print(f"❌ {validation_results['error']}") + else: + print(f"πŸ“Š Total redirects: {validation_results['total_redirects']}") + print(f"πŸ“Š Total navigation links: {validation_results['total_navigation_links']}") + + # Self-referencing redirects + self_ref = validation_results['self_referencing_redirects'] + if self_ref: + print(f"\n⚠️ SELF-REFERENCING REDIRECTS ({len(self_ref)} found):") + for redirect in self_ref: + print(f" ❌ {redirect['source']} β†’ {redirect['destination']}") + else: + print("\nβœ… No self-referencing redirects found!") + + # Navigation-redirect conflicts + conflicts = validation_results['navigation_redirect_conflicts'] + if conflicts: + print(f"\n⚠️ NAVIGATION-REDIRECT CONFLICTS ({len(conflicts)} found):") + for conflict in conflicts: + print(f" ❌ {conflict['source']} β†’ {conflict['destination']} (exists in navigation)") + else: + print("\nβœ… No navigation-redirect conflicts found!") + + # Invalid redirects + invalid = validation_results['invalid_redirects'] + if invalid: + print(f"\n⚠️ INVALID REDIRECTS ({len(invalid)} found):") + for redirect in invalid: + print(f" ❌ Source: '{redirect.get('source', '')}' β†’ Destination: '{redirect.get('destination', '')}'") + else: + print("\nβœ… No invalid redirects found!") + + # Missing navigation files + missing_nav = validation_results['missing_navigation_files'] + if missing_nav: + print(f"\n⚠️ MISSING NAVIGATION FILES ({len(missing_nav)} found):") + for missing_file in missing_nav: + print(f" ❌ {missing_file} (referenced in navigation but file doesn't exist)") + else: + print("\nβœ… All navigation files exist!") + + # Missing redirect destinations + missing_destinations = validation_results['missing_redirect_destinations'] + if missing_destinations: + print(f"\n⚠️ MISSING REDIRECT DESTINATIONS ({len(missing_destinations)} found):") + for missing_dest in missing_destinations: + print(f" ❌ {missing_dest['source']} β†’ {missing_dest['destination']} (destination file doesn't exist)") + else: + print("\nβœ… All redirect destinations exist!") + + # Summary + print(f"\n{'='*60}") + print("SUMMARY") + print(f"{'='*60}") + + if not args.images_only: + if broken_links: + print(f"❌ Found {total_broken_links} broken internal links in {len(broken_links)} files") + else: + print("βœ… No broken internal links found!") + + if not args.links_only: + if broken_images: + print(f"❌ Found {total_broken_images} broken image references in {len(broken_images)} files") + else: + print("βœ… No broken image references found!") + + if validation_results and 'error' not in validation_results: + total_issues = (len(validation_results['self_referencing_redirects']) + + len(validation_results['navigation_redirect_conflicts']) + + len(validation_results['invalid_redirects'])) + if total_issues > 0: + print(f"❌ Found {total_issues} redirect validation issues") + else: + print("βœ… No redirect validation issues found!") + + if args.verbose: + print(f"\nScanned files in: {args.directory}") + print(f"Total MDX files processed: {len(glob.glob(os.path.join(args.directory, '**/*.mdx'), recursive=True))}") + + # Return non-zero exit code if any issues found + has_broken_links = broken_links and not args.images_only + has_broken_images = broken_images and not args.links_only + has_validation_issues = (validation_results and 'error' not in validation_results and + (validation_results['self_referencing_redirects'] or + validation_results['navigation_redirect_conflicts'] or + validation_results['invalid_redirects'] or + validation_results['missing_navigation_files'] or + validation_results['missing_redirect_destinations'])) + + return 1 if (has_broken_links or has_broken_images or has_validation_issues) else 0 + +if __name__ == '__main__': + exit(main()) diff --git a/snippets/AIStudioCards.mdx b/snippets/AIStudioCards.mdx new file mode 100644 index 00000000..fca2e1b8 --- /dev/null +++ b/snippets/AIStudioCards.mdx @@ -0,0 +1,26 @@ +import { FeatureCards } from '/snippets/FeatureCards.mdx'; + +export const components = [ + { + title: "Centralized AI Management", + icon: "πŸ—„οΈ", + description: "Unify and govern AI usage with role-based access, rate limiting, monitoring, and compliance controls." + }, + { + title: "AI Gateway", + icon: "πŸšͺ", + description: "Securely proxy LLMs, integrate custom models, and track usage/costs through a central gateway." + }, + { + title: "AI Portal", + icon: "πŸ”", + description: "Empower developers with a curated catalog of AI tools and services for faster innovation." + }, + { + title: "AI Chat", + icon: "πŸ’¬", + description: "Enable direct, secure interaction with AI tools and data sources via intuitive chat interfaces." + } +]; + +<FeatureCards components={components} /> diff --git a/snippets/Badge.mdx b/snippets/Badge.mdx new file mode 100644 index 00000000..25f723db --- /dev/null +++ b/snippets/Badge.mdx @@ -0,0 +1,36 @@ +export const BadgeCard = ({ + type = '', + href = '', + read = '', + title = '', + titleStyle = '', + image = '', + imageStyle = '', + alt = '', + children +}) => { + const cleanedHref = href.replace(/^\/?(docs\/)?/, ''); + + return ( + <a className={`badge ${type} ${image ? 'with_image' : ''}`} href={`/${cleanedHref}`}> + {read && <div className="nav read">{read}</div>} + {title && <div className="nav title" style={{ ...(titleStyle ? { style: titleStyle } : {}) }}>{title}</div>} + + <p> + {image && <img src={image} style={imageStyle ? { style: imageStyle } : {}} alt={alt} />} + {title ? ( + <div style={{ textAlign: 'center' }}> + {children} + </div> + ) : ( + children + )} + </p> + + <div className="bottom"> + <div className="read">{read}</div> + <span className="button center button-black">Keep reading</span> + </div> + </a> + ); +}; diff --git a/snippets/ButtonLeft.mdx b/snippets/ButtonLeft.mdx new file mode 100644 index 00000000..a4b13db1 --- /dev/null +++ b/snippets/ButtonLeft.mdx @@ -0,0 +1,54 @@ +export const ButtonLeft = ({ href, color, content }) => { + const buttonStyle = { + display: 'inline-block', + padding: '5px 16px', + fontSize: '14px', + fontWeight: '500', + textDecoration: 'none', + borderRadius: '25px', + transition: 'all 0.2s ease', + cursor: 'pointer', + border: '1.2px solid black' + }; + + const colorStyles = { + green: { + backgroundColor: '#20EDBA', + color: 'black' + }, + red: { + backgroundColor: '#dc2626', + color: 'white' + }, + black: { + backgroundColor: '#1f2937', + color: 'white' + } + }; + + const hoverStyle = { + transform: 'translateY(-1px)', + boxShadow: '0 4px 8px rgba(0,0,0,0.15)' + }; + + const finalStyle = { + ...buttonStyle, + ...(colorStyles[color] || colorStyles.black) + }; + + return ( + <a + href={href} + style={finalStyle} + onMouseEnter={(e) => { + Object.assign(e.target.style, hoverStyle); + }} + onMouseLeave={(e) => { + e.target.style.transform = 'translateY(0)'; + e.target.style.boxShadow = 'none'; + }} + > + {content} + </a> + ); +}; diff --git a/snippets/DatabaseCalculator.mdx b/snippets/DatabaseCalculator.mdx new file mode 100644 index 00000000..8952fad2 --- /dev/null +++ b/snippets/DatabaseCalculator.mdx @@ -0,0 +1,5 @@ +#### Database Storage Calculator + +Use this calculator to estimate the storage requirements for your database when using Tyk. This tool helps you plan your database infrastructure by calculating storage needs based on your API traffic, analytics retention, and configuration data. The calculator considers factors like requests per second, analytics time-to-live, and the number of APIs and policies you manage. Results show the total storage requirement accounting for your specified utilization threshold. + +*[Interactive Database Calculator will be implemented here]* diff --git a/snippets/FeatureCards.mdx b/snippets/FeatureCards.mdx new file mode 100644 index 00000000..4016a320 --- /dev/null +++ b/snippets/FeatureCards.mdx @@ -0,0 +1,18 @@ +export const FeatureCards = ({ components }) => { + return ( + <div className="feature-grid"> + {components.map((item, index) => ( + <div key={index} className="feature-item"> + <div className="feature-icon">{item.icon}</div> + <div className="feature-title">{item.title}</div> + <p className="feature-description">{item.description}</p> + {item.path && ( + <div className="feature-button"> + <a href={item.path}>{item.button_text || 'Learn More'}</a> + </div> + )} + </div> + ))} + </div> + ); +}; diff --git a/snippets/GitHubStarButton.mdx b/snippets/GitHubStarButton.mdx new file mode 100644 index 00000000..aee3669b --- /dev/null +++ b/snippets/GitHubStarButton.mdx @@ -0,0 +1,16 @@ +export const GitHubStarButton = ({ owner, repo, showCount = "true" }) => { + return ( + <a + className="github-button" + href={`https://github.com/${owner}/${repo}`} + data-icon="octicon-star" + aria-label={`Star ${owner}/${repo} on GitHub`} + data-show-count={showCount} + > + ⭐ Star {owner}/{repo} + </a> + ); +}; + +<GitHubStarButton {...props} /> + diff --git a/snippets/Grid.mdx b/snippets/Grid.mdx new file mode 100644 index 00000000..b81a18b2 --- /dev/null +++ b/snippets/Grid.mdx @@ -0,0 +1,19 @@ +export const GridWrapper = ({ children, type = '', next }) => { + return ( + <div className={`grid-container ${type}`}> + {children} + + {next && ( + <a className="badge next" href={next}> + View more resources  + <object + data="/images/arrows.png" + style={{ display: 'inline', verticalAlign: 'middle', marginTop: '-3px' }} + /> + </a> + )} + + <div style={{ clear: 'both' }} /> + </div> + ); +}; diff --git a/snippets/RedisCalculator.mdx b/snippets/RedisCalculator.mdx new file mode 100644 index 00000000..3d6fe8af --- /dev/null +++ b/snippets/RedisCalculator.mdx @@ -0,0 +1,6 @@ + +#### Redis RAM Calculator + +Use this calculator to estimate the RAM requirements for your Redis instance when using Tyk. This tool helps you plan your infrastructure by calculating memory needs based on your expected API traffic, caching requirements, and analytics storage. The calculator considers factors like requests per second, cache hit rates, number of API keys, and analytics settings. Results show the total RAM per host accounting for your specified utilization threshold. + +*[Interactive Redis Calculator will be implemented here]* diff --git a/snippets/RedisVersions.mdx b/snippets/RedisVersions.mdx new file mode 100644 index 00000000..5672e3aa --- /dev/null +++ b/snippets/RedisVersions.mdx @@ -0,0 +1,4 @@ +### Supported Versions + +- Tyk 5.3 supports Redis 6.2.x, 7.0.x, and 7.2.x +- Tyk 5.2.x and earlier supports Redis 6.0.x and Redis 6.2.x only. diff --git a/snippets/ReleaseSummaryComponent.mdx b/snippets/ReleaseSummaryComponent.mdx new file mode 100644 index 00000000..51ec7370 --- /dev/null +++ b/snippets/ReleaseSummaryComponent.mdx @@ -0,0 +1,67 @@ +export const ReleaseSummaryComponent = ({ releaseData }) => { + const linkStyle = { + fontWeight: 'normal', + textDecoration: 'none', + borderBottom: 'none', + color: 'rgb(var(--primary))', + transition: 'border-bottom 0.2s ease' + }; + + const handleMouseEnter = (e) => { + e.target.style.borderBottom = '1px solid rgb(var(--primary))'; + }; + + const handleMouseLeave = (e) => { + e.target.style.borderBottom = 'none'; + }; + + const renderComponent = (component) => ( + <Card + key={component.name} + title={`${component.name} v${component.latest}${component.lts ? ` | v${component.lts} LTS` : ''}`} + > + Click to <a href={`/${component.home}`} style={linkStyle} onMouseEnter={handleMouseEnter} onMouseLeave={handleMouseLeave}>discover more</a> + + {component.latestData && ( + <> + <div style={{marginBottom: '4px', marginTop: '8px', fontWeight: 'bold', fontSize: '14px', color: '#333'}}>Docker</div> + {component.latestData.docker && ( + <div style={{fontSize: '13px'}}>- <a href={component.latestData.docker} style={linkStyle} onMouseEnter={handleMouseEnter} onMouseLeave={handleMouseLeave}>Latest {component.latest}</a></div> + )} + {component.ltsData?.docker && ( + <div style={{fontSize: '13px'}}>- <a href={component.ltsData.docker} style={linkStyle} onMouseEnter={handleMouseEnter} onMouseLeave={handleMouseLeave}>LTS {component.lts}</a></div> + )} + {component.latestData.tag && !component.licensed && ( + <> + <div style={{marginBottom: '4px', marginTop: '8px', fontWeight: 'bold', fontSize: '14px', color: '#333'}}>GitHub</div> + <div style={{fontSize: '13px'}}>- <a href={component.latestData.tag} style={linkStyle} onMouseEnter={handleMouseEnter} onMouseLeave={handleMouseLeave}>Latest {component.latest}</a></div> + {component.ltsData?.tag && ( + <div style={{fontSize: '13px'}}>- <a href={component.ltsData.tag} style={linkStyle} onMouseEnter={handleMouseEnter} onMouseLeave={handleMouseLeave}>LTS {component.lts}</a></div> + )} + </> + )} + </> + )} + + <div style={{marginTop: '12px', paddingTop: '8px', borderTop: '1px solid #eee'}}> + <a href={`/${component.releaseNotesPath}`} style={linkStyle} onMouseEnter={handleMouseEnter} onMouseLeave={handleMouseLeave}> + Release Notes β†’ + </a> + </div> + </Card> + ); + + return ( + <> + <h2>Licensed</h2> + <ResponsiveGrid> + {releaseData.licensed.map(renderComponent)} + </ResponsiveGrid> + + <h2>Open Source</h2> + <ResponsiveGrid> + {releaseData.opensource.map(renderComponent)} + </ResponsiveGrid> + </> + ); +}; diff --git a/snippets/ResponsiveGrid.mdx b/snippets/ResponsiveGrid.mdx new file mode 100644 index 00000000..0749b6aa --- /dev/null +++ b/snippets/ResponsiveGrid.mdx @@ -0,0 +1,7 @@ +export const ResponsiveGrid = ({ children }) => { + return ( + <div className="responsive-grid"> + {children} + </div> + ); +}; diff --git a/snippets/UIExampleCards.mdx b/snippets/UIExampleCards.mdx new file mode 100644 index 00000000..acefc840 --- /dev/null +++ b/snippets/UIExampleCards.mdx @@ -0,0 +1,34 @@ +import { FeatureCards } from '/snippets/FeatureCards.mdx'; + +export const components = [ + { + title: "Documentation", + icon: "πŸ“š", + description: "Comprehensive documentation with examples and guides.", + path: "/developer-support/contributing", + button_text: "Learn More" + }, + { + title: "Shortcodes", + icon: "🧩", + description: "Reusable components to enhance your content.", + path: "/ui-examples/feature-cards", + button_text: "View Shortcodes" + }, + { + title: "Styling", + icon: "🎨", + description: "Consistent visual elements for a polished look.", + path: "/portal/customization", + button_text: "Explore Styles" + }, + { + title: "Templates", + icon: "πŸ“‹", + description: "Pre-built layouts for common documentation needs.", + path: "/tyk-developer-portal/customise", + button_text: "See Templates" + } +]; + +<FeatureCards components={components} /> diff --git a/snippets/api-def-graphql.mdx b/snippets/api-def-graphql.mdx new file mode 100644 index 00000000..668c36e1 --- /dev/null +++ b/snippets/api-def-graphql.mdx @@ -0,0 +1,136 @@ +--- +--- + +Tyk Classic is the home of Tyk GraphQL. All of the specific settings required to configure Tyk Gateway to proxy GraphQL services is gathered in a single `graphql` object in the API definition. + +An example is shown below that composes two different data sources: + +1. *countries* which is a `GraphQLDataSource` +2. *people* which is a `HTTPJSONDataSource` + +```json expandable +{ + "graphql": { + "enabled": true, + "execution_mode": "executionEngine", + "schema": "type Country {\n code: String\n}\n\ntype People {\n count: Int\n}\n\ntype Query {\n countries: [Country]\n people: People\n}\n", + "type_field_configurations": [ + { + "type_name": "Query", + "field_name": "countries", + "mapping": { + "disabled": false, + "path": "countries" + }, + "data_source": { + "kind": "GraphQLDataSource", + "data_source_config": { + "url": "https://countries.trevorblades.com/", + "method": "POST" + } + } + }, + { + "type_name": "Query", + "field_name": "people", + "mapping": { + "disabled": true, + "path": "" + }, + "data_source": { + "kind": "HTTPJSONDataSource", + "data_source_config": { + "url": "https://swapi.dev/api/people/", + "method": "GET", + "body": "", + "headers": [], + "default_type_name": "People", + "status_code_type_name_mappings": [ + { + "status_code": 200, + "type_name": "" + } + ] + } + } + } + ], + "playground": { + "enabled": true, + "path": "/playground" + } + } +} +``` + +**Field: `graphql`** +All the GraphQL configuration is gathered in the `graphql` object. + +**Field: `graphql.enabled`** +If set to `true`, this means the API definition describes a GraphQL API. Tyk GraphQL middleware will be enabled. + +**Field: `graphql.execution_mode`** +The mode of a GraphQL API. There are two options: + +- `proxyOnly`: there is a single upstream which is a GraphQL API. +- `executionEngine`: configure your own GraphQL API with multiple data sources; you will compose your own schema. + +**Field: `graphql.schema`** +The GraphQL schema of your API, stored in SDL format + +**Field: `graphql.type_field_configurations`** +A list of configurations used when `execution_mode` is `executionEngine`. For your schema, you can set data sources for fields in your types. + +**Field: `graphql.type_field_configurations.type_name`** +A type of the schema that a field of it will be data source configured. + +**Field: `graphql.type_field_configurations.field_name`** +A field of the type that will be data source configured. + +**Field: `graphql.type_field_configurations.mapping`** +- `mapping`: Mapping configurations of a field. It is used to map the field in the received data and a field in the schema. It is used to represent a field with a different name in the schema. + +**Field: `graphql.type_field_configurations.mapping.disabled`** +If it is `false`, it means enabled. + +**Field: `graphql.type_field_configurations.mapping.path`** +Original name of the field in the received data. + +**Field: `graphql.type_field_configurations.data_source`** +Configuration of a specific data source. + +**Field: `graphql.type_field_configurations.data_source.kind`** +Kind of the upstream. It can be one of `HTTPJSONDataSource` or `GraphQLDataSource`. + +**Field: `graphql.type_field_configurations.data_source.data_source_config`** +The details of the `data_source` + +**Field: `graphql.type_field_configurations.data_source.data_source_config.url`** +URL of the upstream data source like `https://swapi.dev/api` or it can be another Tyk API which you can set like `tyk://<tyk-api-name>` or `tyk://<tyk-api-id>`. Also, you can pass parameters e.g. `"/my-path/{{ .arguments.id }}`, where `id` is passed as query variable in a GraphQL request. + +**Field: `graphql.type_field_configurations.data_source.data_source_config.method`** +HTTP request method which the upstream server waits for the url e.g. `GET`, `POST`, `UPDATE`, `DELETE`. + +**Field: `graphql.type_field_configurations.data_source.data_source_config.body`** +HTTP request body to send to upstream. + +**Field: `graphql.type_field_configurations.data_source.data_source_config.headers`** +HTTP headers to send to upstream composed of `key` and `value` pairs. + +**Field: `graphql.type_field_configurations.data_source.data_source_config.default_type_name`** +The optional variable to define a default type name for the response object. It is useful in case the response might be a `Union` or `Interface` type which uses `status_code_type_name_mappings`. - only valid for `HTTPJSONDataSource` + +**Field: `graphql.type_field_configurations.data_source.data_source_config.status_code_type_name_mappings`** +A list of mappings from `http.StatusCode` to GraphQL `type_name`. It can be used when the `type_name` depends on the response code. - only valid for `HTTPJSONDataSource` +- `status_code`: The HTTP response code to map to `type_name`. +- `type_name`: Type name to be mapped to `status_code`. + +**Field: `graphql.playground`** +Configuration of the playground which is exposed from the Gateway route. + +**Field: `graphql.playground.enabled`** +If it is `true`, it means the playground will be exposed. + +**Field: `graphql.playground.path`** +The path where playground will sit e.g. if it is `/playground` in your API with name `composed`, you can access to the playground by `https://tyk-gateway/composed/playground`. + diff --git a/snippets/create-api-include.mdx b/snippets/create-api-include.mdx new file mode 100644 index 00000000..57f64584 --- /dev/null +++ b/snippets/create-api-include.mdx @@ -0,0 +1,204 @@ +--- +--- + +{/* START OMIT */} + + +<Note> +**Note: Integration with your OpenAPI documentation** + +In Tyk v4.1 we introduced support for APIs defined according to the [OpenAPI Specification v3.0.3](https://spec.openapis.org/oas/v3.0.3) (OAS). +This introduces a standard way to describe the vendor-agnostic elements of an API (the OpenAPI Definition, stored as an OpenAPI Document); we take this and add Tyk-specific configuration options to create the *Tyk OAS API Definition*. You can import your own OpenAPI document and Tyk will use this to generate the Tyk OAS API Definition. +For a detailed tutorial on using OAS with Tyk Gateway, check out our guide to [creating a Tyk OAS API Definition](/api-management/gateway-config-managing-oas#creating-an-api). +</Note> + + +**Prerequisites** + +In order to complete this tutorial, you need to have [Tyk Self Managed installed](/tyk-self-managed/install). + +<ButtonLeft href="https://tyk.io/sign-up/#self" color="green" content="Try it free" /> + +#### Create an API with the Dashboard + +We have a video walkthrough for creating an API and testing an endpoint via Postman. + +<iframe width="560" height="315" src="https://www.youtube.com/embed/qJOHn8BuMpw" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + +We will use the Tyk Dashboard to create a very simple API that has no special elements set up. + +1. **Select "APIs" from the "System Management" section** + + <img src="/img/2.10/apis_menu.png" alt="API Menu" /> + +2. **Click "ADD NEW API"** + + <img src="/img/2.10/add_api.png" alt="Add API button location" /> + +3. **Set up the basic configuration of your API** + + <img src="/img/dashboard/4.1-updates/create-api.png" alt="Create API" /> + + - In the **Overview** section, add a **Name** for your API and select the **Type** of API you wish to create. We will use HTTP for this tutorial. + - In the **Details** section, add the **Upstream URL**. This is the Target URL that hosts the service to which you want to proxy incoming requests. You can configure Tyk to perform round-robin load balancing between multiple upstream servers (Target URLs) by selecting **Enable round-robin load balancing**; see [Load Balancing](/planning-for-production/ensure-high-availability/load-balancing) for more details. For this tutorial, we will use a single upstream target: [http://httpbin.org](http://httpbin.org). + - Click **Configure API** when you have finished. + +4. **Set up authentication for your API** + + Take a look at the **Authentication** section: + + <img src="/img/2.10/authentication.png" alt="Authentication" /> + + You have the following options: + + - **Authentication mode**: This is the method that Tyk should use to authenticate requests to call your API. Tyk supports several different authentication modes - see [Client Authentication](/api-management/client-authentication) for more details on securing your API. For this tutorial, you should select `Open (Keyless)`. + - **Strip Authorization Data**: Select this option to ensure that any security (authentication) tokens provided to authorize requests to your API on Tyk are not leaked to the upstream. You can leave this unchecked for this tutorial. + - **Auth Key Header Name**: The header parameter that will hold the authentication token (or key) for requests to this API; the default for this is `Authorization`. + - **Allow query parameter as well as header**: This option allows the authentication token to be set in the query parameter, not just in the Request Header. For this tutorial, leave this unchecked. + - **Use Cookie Value**: Tyk also supports the use of a cookie value as an alternative authentication token location. For this tutorial, leave this unchecked. + - **Enable client certificate**: Tyk supports the use of Mutual TLS to authenticate requests to your API; you would use this checkbox to enable this mode. See [Mutual TLS](/basic-config-and-security/security/mutual-tls/client-mtls#why-use-mutual-tls) for details on implementing this feature. For this tutorial, leave this unchecked. + +5. **Save the API** + + Click **SAVE** + + <img src="/img/2.10/save.png" alt="Save button" /> + + Once saved, you will be taken back to the API list, where your new API will be displayed. + + If you select the API from the list to open it again, the API URL will be displayed in the top of the editor. This is the URL that your consumers will need to call to invoke your API. + + <img src="/img/2.10/api_url.png" alt="API URL location" /> + +#### Create an API with the Dashboard API + +It is easy to create APIs using Tyk Dashboard's own REST API. +You will need an API key for your organization (to authenticate with the Dashboard API) and issue a request using these credentials to create your new API and make it live. + +1. **Obtain your Tyk Dashboard API access credentials key & Dashboard URL** + + - From the Tyk Dashboard, select "Users" in the "System Management" section. + - Click **Edit** for your username, then scroll to the bottom of the page. + - Your personal API key, granting you access to the Dashboard API, is labeled **Tyk Dashboard API Access Credentials** key + + <img src="/img/2.10/user_api_id.png" alt="API key location" /> + + - Store your Dashboard Key, Dashboard URL & Gateway URL as environment variables so you don't need to keep typing them in + + ```bash + export DASH_KEY=db8adec7615d40db6419a2e4688678e0 + + # Locally installed dashboard + export DASH_URL=http://localhost:3000/api + + # Locally installed gateway + export GATEWAY_URL=http://localhost:8080 + + + ### Step 2: Query the `/api/apis` endpoint to see what APIs are loaded on the Gateway + + ```curl + curl -H "Authorization: ${DASH_KEY}" ${DASH_URL}/apis + {"apis":[],"pages":1} + ``` + + As you've got a fresh install, you will see that no APIs currently exist + +2. **Create your first API** + + We've created a simple Tyk Classic API definition that configures the Tyk Gateway to reverse proxy to the [http://httpbin.org](http://httpbin.org) + request/response service. The API definition object is stored here: https://bit.ly/2PdEHuv. + + To load the API definition to the Gateway via the Dashboard API you issue this command: + + ```curl + curl -H "Authorization: ${DASH_KEY}" -H "Content-Type: application/json" ${DASH_URL}/apis \ + -d "$(wget -qO- https://bit.ly/2PdEHuv)" + {"Status":"OK","Message":"API created","Meta":"5de83a40767e0271d024661a"} + ``` + + **Important** Take note of the API ID returned in the `Meta` field - you will need it later as this is the Tyk Gateway's internal identifier for the new API. + + ``` + export API_ID=5de83a40767e0271d024661a + ``` + +3. **Test your new API** + + You can now make a call to your new API as follows: + + ```curl + curl ${GATEWAY_URL}/httpbin/get + { + "args": {}, + "headers": { + "Accept": "*/*", + "Accept-Encoding": "gzip", + "Host": "httpbin.org", + "User-Agent": "curl/7.54.0" + }, + "origin": "127.0.0.1, 188.220.131.154, 127.0.0.1", + "url": "https://httpbin.org/get" + } + ``` + + We sent a request to the gateway on the listen path `/httpbin`. Using this path-based-routing, the gateway was able to identify the API the client intended to target. + + The gateway stripped the listen path and reverse proxied the request to http://httpbin.org/get + +4. **Protect your API** + + Let's grab the API definition we created before and store the output to a file locally. + + ```curl + curl -s -H "Authorization: ${DASH_KEY}" -H "Content-Type: application/json" ${DASH_URL}/apis/${API_ID} | python -mjson.tool > api.httpbin.json + ``` + + We can now edit the `api.httpbin.json` file we just created, and modify a couple of fields to enable authentication. + + Change `use_keyless` from `true` to `false`. + + Change `auth_configs.authToken.auth_header_name` to `apikey`. + + **Note** Prior to ** Tyk v2.9.2** `auth_configs.authToken.auth_header_name` was called `auth.auth_header_name` + + Then send a `PUT` request back to Tyk Dashboard to update its configuration. + + ```curl + curl -H "Authorization: ${DASH_KEY}" -H "Content-Type: application/json" ${DASH_URL}/apis/${API_ID} -X PUT -d "@api.httpbin.json" + {"Status":"OK","Message":"Api updated","Meta":null} + ``` + +5. **Test your protected API** + + First try sending a request without any credentials, as before: + + ```curl + curl -I ${GATEWAY_URL}/httpbin/get + HTTP/1.1 401 Unauthorized + Content-Type: application/json + X-Generator: tyk.io + Date: Wed, 04 Dec 2019 23:35:34 GMT + Content-Length: 46 + ``` + + As you can see, you received an `HTTP 401 Unauthorized` response. + + Now send a request with incorrect credentials: + + ```curl + curl -I ${GATEWAY_URL}/httpbin/get -H 'apikey: somejunk' + HTTP/1.1 403 Forbidden + Content-Type: application/json + X-Generator: tyk.io + Date: Wed, 04 Dec 2019 23:36:16 GMT + Content-Length: 57 + ``` + + As you can see, you received an `HTTP 403 Forbidden` response. + + Try sending another request, this time with a valid API key. + + Congratulations - You have just created your first keyless API, then protected it using Tyk! + +{/* END OMIT */} diff --git a/snippets/create-api-key-include.mdx b/snippets/create-api-key-include.mdx new file mode 100644 index 00000000..dfcdbb7d --- /dev/null +++ b/snippets/create-api-key-include.mdx @@ -0,0 +1,104 @@ +--- +--- + +#### Create an API Key with the Dashboard + +The Tyk Dashboard is the simplest way to generate a new Key. + +We have a video walkthrough for creating an API Key. + +<iframe width="560" height="315" src="https://www.youtube.com/embed/sydrO2qv88Y" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + + +1. **Select "Keys" from the "System Management" section** + + <img src="/img/2.10/keys_menu.png" alt="Keys menu" /> + +2. **Click CREATE** + + <img src="/img/2.10/add_key.png" alt="Add key" /> + +3. **Add a Policy or API to your Key** + + You have the option to add your new key to either an existing Policy or an existing individual API. For this Tutorial we are going to use an API. + + **Add an API to your Key** + + To select an API, you can either: + + * Scroll through your API Name list + * Use the Search field + * You can also Group by Authentication Type to filter your APIs + * You can also Group by Category + + You can leave all other options at their default settings. + +4. **Add Configuration Details** + + You use the Configuration section to set the following: + + 1. Enable Detailed Logging. This is disabled by default and isn't required for this tutorial + 2. Give your Key an Alias. This makes your key easier + 3. Set an Expiry time after which the key will expire. Select a value from the drop-down list. This is a required setting. See [Key Expiry](/api-management/policies#key-expiry) for more details. + 4. Add Tags to your policy. Any tags you add can be used when filtering Analytics Data. Tags are case sensitive. + 5. Add Metadata to your policy. Adding metadata such as User IDs can be used by middleware components. See [Session Metadata](/api-management/policies#what-is-a-session-metadata) for more details. + +4. **Click CREATE** + + <img src="/img/2.10/create_key.png" alt="Create button" /> + + A **Key successfully generated** pop-up will be displayed with the key shown. You **must** save this somewhere for future reference as it will not be displayed again. Click **Copy to clipboard** and paste into a text document. + + <img src="/img/2.10/key_success.png" alt="Key success message location" /> + + That's it, you've created a key - now you can try and use it. + +#### Create an API Key with the API + +To create an API key, you will need the API ID that we wish to grant the key access to. Creating the token is then an API call to the endpoint. + +You will also need your own API Key, to get these values: + +1. Select **Users** from the **System Management** section. +2. In the users list, click **Edit** for your user. +3. The API key is the **Tyk Dashboard API Access Credentials**, copy this somewhere you can reference it. <img src="/img/2.10/user_api_id.png" alt="API key location" /> +4. Select **APIs** from the **System Management** section. +5. From the **Actions** menu for your API, select Copy API ID + + <img src="/img/2.10/api_id.png" alt="API ID location" /> + + Once you have these values, you can use them to access the Dashboard API, the below `curl` command will generate a key for one of your APIs: + + + + <Note> + 1. Replace the `authorization` header value with your Tyk Dashboard API Access Credentials + 2. Replace the API ID (`ad5004d961a147d4649fd3216694ebe2`) with your API ID + 3. It's recommended to validate the JSON using JSON validator to avoid any `malformed input` error + </Note> + + +```{.copyWrapper} expandable +curl -X POST -H "authorization: 1238b7e0e2ff4c2957321724409ee2eb" \ + -s \ + -H "Content-Type: application/json" \ + -X POST \ + -d '{ + "allowance": 1000, + "rate": 1000, + "per": 1, + "expires": -1, + "quota_max": -1, + "quota_renews": 1449051461, + "quota_remaining": -1, + "quota_renewal_rate": 60, + "access_rights": { + "ad5004d961a147d4649fd3216694ebe2": { + "api_id": "ad5004d961a147d4649fd3216694ebe2", + "api_name": "test-api", + "versions": ["Default"] + } + }, + "meta_data": {} + }' https://admin.cloud.tyk.io/api/keys | python -mjson.tool +``` diff --git a/snippets/create-portal-entry-include.mdx b/snippets/create-portal-entry-include.mdx new file mode 100644 index 00000000..3e807965 --- /dev/null +++ b/snippets/create-portal-entry-include.mdx @@ -0,0 +1,102 @@ +--- +--- + +## Tutorial: Add an API and Swagger based Docs to a Portal Catalog + +You can use the Tyk Dashboard to create a portal that allows developers to access the APIs you create. + +<iframe width="560" height="315" src="https://www.youtube.com/embed/cywF9Dvg6lI" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + + +### Prerequisites for a portal catalog entry: + +- An API configured and live on your Tyk Gateway +- The API must be **Closed** (i.e. it must use either Auth Token or Basic Auth security mechanisms) +- A security policy configured to grant access to the API + + + + + <Note> + If you intend to use the developer portal, you need to configure it with a different hostname to your dashboard. The developer portal cannot be accessed via an IP address. + </Note> + + + +<Warning> +Without these prerequisites, you may get a 404 error when trying to access the portal. +</Warning> + + + +### Step 1: Select "Catalog" from the "Portal Management" section + +<img src="/img/2.10/catalogue_menu.png" alt="Catalogue menu" /> + +### Step 2: Click ADD NEW API + +This page displays all of the catalog entries you have defined, whether they have documentation attached and whether they are active on the portal or not. Click **ADD NEW API**. + +<img src="/img/2.10/add_catalogue_entry.png" alt="Add new API button" /> + +## Add API Details + +### Step 3: Show the API + +By default your entry will be published automatically to your Portal Catalog. Deselect this option to not publish it. + +### Step 4: Set a Public API Name and associate a security policy + +When publishing an API with Tyk, you are not publishing a single API, but instead you are publishing access to a group of APIs. The reason for this is to ensure that it is possible to compose and bundle APIs that are managed into APIs that are published. Tyk treats these as separate, so the thing that is published on the portal is not the same as the actual API being managed by Tyk, one is a logical managed API and the other (the published catalog version) is a facade. + +Since API policies allow the bundling of access control lists of multiple APIs, it is actually this that you are granting access to. Any developer that signs up for this API, will be granted a bearer token that has this policy as a baseline template, or as a "plan". + +<img src="/img/2.10/public_name_catalogue.png" alt="Portal name and security policy" /> + +Please note: + +1. You will only see security policies for valid APIs in the **Public API Name** drop-down list for the policies +2. The **Available policies** selected must be "closed" (see Prerequisites) + +### Step 5: Add a description + +All catalog entries can have a description. You can use Markdown formatting in these fields: + +<img src="/img/2.10/catalogue_description.png" alt="Description" /> + +You can also add an email address if you require notification that an API subscription has been submitted or granted. We'll leave that blank for this tutorial. The same goes for Custom Fields. See [Custom Portal](/tyk-developer-portal/tyk-portal-classic/customise/custom-developer-portal#updating-a-developer-example-adding-custom-fields) for an example of a Custom Field implementation. + + +### Step 6: Attach Documentation + +You can add import documentation in the following formats: + +- From a Swagger JSON file (OpenAPI 2.0 and 3.0 are supported) +- From a Swagger URL +- From API Blueprint + + + + <Note> + Support for API Blueprint is being deprecated. See [Importing APIs](/api-management/gateway-config-managing-classic#api-blueprint-is-being-deprecated) for more details. + </Note> + + +You can add your documentation before or after saving your API. + +### Settings Options + +We are not going to do anything with these options for this tutorial. For more information: + +* See [OAuth Clients](/tyk-developer-portal/tyk-portal-classic/portal-oauth-clients) for details of using OAuth with your catalog entry. +* See [Portal Customization](/tyk-developer-portal/customise)for details about redirection of key requests and developer signup customization. + +### Step 6: Save the API + +To save the API, click **SAVE**. + +### Step 7: Take a look + +You can now visit your portal to see the API catalog entry. Select **Open Your Portal** from the **Your Developer Portal** menu: + +<img src="/img/getting-started/portal_menu.png" alt="Portal nav menu location" /> diff --git a/snippets/create-security-policy-include.mdx b/snippets/create-security-policy-include.mdx new file mode 100644 index 00000000..bc962311 --- /dev/null +++ b/snippets/create-security-policy-include.mdx @@ -0,0 +1,162 @@ +--- +--- + +A security policy encapsulates several options that can be applied to a key. It acts as a template that can override individual sections of an API key (or identity) in Tyk. + +See [What is a Security Policy?](/api-management/policies#what-is-a-security-policy) + + +#### Create a security policy with the Dashboard + +We have a video walkthrough for creating an security policy with the Dashboard. + +<iframe width="560" height="315" src="https://www.youtube.com/embed/y4xVUvUvFRE" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + + +To create a security policy with the Dashboard, follow these steps: + +1. **Select "Policies" from the "System Management" section** + + <img src="/img/2.10/policies_menu.png" alt="Policies menu link location" /> + + Your current policies will be displayed + + <img src="/img/2.10/policy_list.png" alt="Current Policies" /> + +2. **Click ADD POLICY** + + <img src="/img/2.10/add_policy.png" alt="Add policy button" /> + +3. **Select an API to apply the policy Access Rights to** + + <img src="/img/2.10/select_api_policy.png" alt="Policy name form" /> + + To select an API, you can either: + + * Scroll through your API Name list + * Use the Search field + * You can also Group by Authentication Type to filter your APIs + * You can also Group by Category + + All policies require a descriptive name, this helps you to reference it later, and it will appear in drop-down options where you can attach policies to objects such as Keys or OAuth client IDs. + +4. **Setting Global Rate Limits and Quota** + + <img src="/img/2.10/global_limits_policies.png" alt="Global Rates" /> + + These settings will be applied to all APIs that the policy is applied to. You can override these settings by turning **Set per API Rate Limits and Quota** on for the API you selected in Step 3. We will leave these settings at their default for this tutorial. + + **Rate Limiting** + + A rate limit is enforced on all keys, set the number of requests per second that a user of a key with this policy is allowed to use. See [Rate Limiting](/api-management/rate-limit#rate-limiting-layers) for more details. **Note: The Rate Limit set by a policy will override the limits applied to an individual key.** + + **Throttling** + + When hitting rate limits, you can set Tyk Gateway to automatically queue and auto-retry client requests. Throttling can be configured at a key or policy level. See [Request Throttling](/api-management/request-throttling) for more details. + + **Usage Quotas** + + Usage quotas limit the number of total requests a user is allowed to have over a longer period of time. So while a rate limit is a rolling window, a quota is an absolute maximum that a user is allowed to have over a week, a day or a month. See [Request Quotas](/api-management/request-quotas) for more details. + + Usage quotas can only be a positive number, or -1 (unlimited). **Note: The Usage Quota set by a policy will override a quota applied to an individual key.** + + **Policy Partitioning** + + In some cases, the all-or-nothing approach of policies, where all the components of access control, quota and rate limit are set together isn’t ideal, and instead you may wish to have only one or two segments of a token managed at a policy level and other segments in another policy or on the key itself. We call this [Policy Partitioning](/api-management/policies#partitioned-policies). + + ###### Path Based Permissions + + You can also use a security policy to apply restrictions on a particular path and method. Granular path control allows you to define which methods and paths a key is allowed to access on a per API-version basis. See [Secure your APIs by Method and Path](/api-management/policies#secure-your-apis-by-method-and-path) for more details + + <img src="/img/2.10/path_and_method.png" alt="Path and Method" /> + +5. **Add Configuration Details** + + You use the Configuration section to set the following: + + 1. Give your policy a name. This is a required setting + 2. Set the policy state. You can set your policy to one of the following states: + * Active (the default) + * Draft + * Access Denied + 3. Set a time after which any Keys subscribed to your policy expire. Select a value from the drop-down list. This is a required setting. See [Key Expiry](/api-management/policies#key-expiry) for more details. + 4. Add Tags to your policy. Any tags you add can be used when filtering Analytics Data. Tags are case sensitive. + 5. Add Metadata to your policy. Adding metadata such as User IDs can be used by middleware components. See [Session Metadata](/api-management/policies#what-is-a-session-metadata) for more details. + +6. **Save the policy** + + Click **CREATE** . Once the policy is saved, you will be able to use it when creating keys, OAuth clients and custom JWT tokens. + +#### Create a security policy with the API + +Security Policies can be created with a single call to the API. It is very similar to the token creation process. To generate a simple security policy using the Tyk Dashboard API you can use the following curl command: +```{.copyWrapper} expandable +curl -X POST -H "authorization: {API-TOKEN}" \ + -s \ + -H "Content-Type: application/json" \ + -X POST \ + -d '{ + "access_rights": { + "{API-ID}": { + "allowed_urls": [], + "api_id": "{API-ID}", + "api_name": "{API-NAME}", + "versions": [ + "Default" + ] + } + }, + "active": true, + "name": "POLICY NAME", + "rate": 100, + "per": 1, + "quota_max": 10000, + "quota_renewal_rate": 3600, + "state": "active", + "tags": ["Startup Users"] + }' https://admin.cloud.tyk.io/api/portal/policies | python -mjson.tool +``` + +You must replace: + +* `{API-TOKEN}`: Your API Token for the Dashboard API. +* `{API-ID}`: The API ID you wish this policy to grant access to, there can be more than one of these entries. +* `{API-NAME}`: The name of the API that is being granted access to (this is not required, but helps when debugging or auditing). +* `POLICY NAME`: The name of this security policy. + +The important elements: + +* `access_rights`: A list of objects representing which APIs that you have configured to grant access to. +* `rate` and `per`: The number of requests to allow per period. +* `quota_max`: The maximum number of allowed requests over a quota period. +* `quota_renewal_rate`: how often the quota resets, in seconds. In this case we have set it to renew every hour. +* `state`: New from **v3.0**, this can be used instead of `active` and `is_inactive`. You can use the following values: + * `active` - all keys connected to the policy are active and new keys can be created + * `draft` - all keys connected to the policy are active but new keys cannot be created + * `deny` - all keys are deactivated and no keys can be created. + + + + <Note> + Setting a `state` value will automatically override the `active` or `is_inactive` setting. + </Note> + + +When you send this request, you should see the following reply with your Policy ID: +``` +{ + "Message": "577a8589428a6b0001000017", + "Meta": null, + "Status": "OK" +} +``` + +You can then use this policy ID in the `apply_policy_id` field of an API token. Please see the relevant documentation on session objects for more information about how tokens are attached to policies. + + +<Note> +`apply_policy_id` is supported, but has now been deprecated. `apply_policies` is now used to list your policy IDs as an array. This supports the **Multiple Policy** feature introduced in the **v2.4/1.4** release. +</Note> + + +For more information on how policies are constructed and a detailed explanation of their properties, please see the [Security Policies](/api-management/policies) section. diff --git a/snippets/dashboard-config.mdx b/snippets/dashboard-config.mdx new file mode 100644 index 00000000..6b06cf40 --- /dev/null +++ b/snippets/dashboard-config.mdx @@ -0,0 +1,1379 @@ +### listen_port +ENV: <b>TYK_DB_LISTENPORT</b><br /> +Type: `int`<br /> + +Setting this value will change the port that Tyk Dashboard listens on. Default: 3000. + +### tyk_api_config +This section contains details for a Tyk Gateway node that the Tyk Dashboard can speak to. The Dashboard controls Tyk using the Gateway API and only requires visibility to one node, so long as all nodes are using the same API Definitions. + + +<Note> +If the Dashboard cannot see a Tyk node, key management functions will not work properly. +</Note> + +In a sharded environment, the Gateway node specified in tyk_api_config must not be sharded. + + +### tyk_api_config.Host +ENV: <b>TYK_DB_TYKAPI_HOST</b><br /> +Type: `string`<br /> + +This is the full URL of your Tyk node. + +### tyk_api_config.Port +ENV: <b>TYK_DB_TYKAPI_PORT</b><br /> +Type: `string`<br /> + +The port that Tyk is running on + +### tyk_api_config.Secret +ENV: <b>TYK_DB_TYKAPI_SECRET</b><br /> +Type: `string`<br /> + +The secret set in your tyk.conf file. This is the key that Tyk Dashboard will use to speak to the Tyk node’s Gateway API. Note that this value **has to match** the secret value in tyk.conf. + +### mongo_url +ENV: <b>TYK_DB_MONGOURL</b><br /> +Type: `string`<br /> + +The full URL to your MongoDB instance, this can be a clustered instance if necessary and should include the database and username / password data. + +### mongo_use_ssl +ENV: <b>TYK_DB_MONGOUSESSL</b><br /> +Type: `bool`<br /> + +Set to true to enable Mongo SSL connection + +### mongo_ssl_insecure_skip_verify +ENV: <b>TYK_DB_MONGOSSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Allows the use of self-signed certificates when connecting to an encrypted MongoDB database. + +### mongo_ssl_allow_invalid_hostnames +ENV: <b>TYK_DB_MONGOSSLALLOWINVALIDHOSTNAMES</b><br /> +Type: `bool`<br /> + +Ignore hostname check when it differs from the original (for example with SSH tunneling). The rest of the TLS verification will still be performed. + +### mongo_ssl_ca_file +ENV: <b>TYK_DB_MONGOSSLCAFILE</b><br /> +Type: `string`<br /> + +Path to the PEM file with trusted root certificates + +### mongo_ssl_pem_keyfile +ENV: <b>TYK_DB_MONGOSSLPEMKEYFILE</b><br /> +Type: `string`<br /> + +Path to the PEM file which contains both client certificate and private key. This is required for Mutual TLS. + +### mongo_session_consistency +ENV: <b>TYK_DB_MONGOSESSIONCONSISTENCY</b><br /> +Type: `string`<br /> + +Mongo session constency: β€œstrong”, β€œeventual”, or β€œmonotonic”. default is β€œstrong” + +### mongo_batch_size +ENV: <b>TYK_DB_MONGOBATCHSIZE</b><br /> +Type: `int`<br /> + +Sets the batch size for mongo results. Defaults to 2000. +Increasing this number can decrease dashboard performance. This value cannot be lower than 100 and will fallback to 100 if a lower value has been set. + +### mongo_driver +ENV: <b>TYK_DB_MONGODRIVER</b><br /> +Type: `string`<br /> + +Determines the MongoDB driver used. It could be `mongo-go` to use the official [mongo driver for go v1.12](https://www.mongodb.com/docs/drivers/go/v1.12/) or `mgo` to use [mgo driver](https://github.com/go-mgo/mgo). Since v5.3, the default value is `mongo-go`. It can be set at storage level as well if the database type is mongo. This config is available since dashboard v5.0.2. + +### mongo_direct_connection +ENV: <b>TYK_DB_MONGODIRECTCONNECTION</b><br /> +Type: `bool`<br /> + +MongoDirectConnection informs whether to establish connections only with the specified seed servers, +or to obtain information for the whole cluster and establish connections with further servers too. +If true, the client will only connect to the host provided in the ConnectionString +and won't attempt to discover other hosts in the cluster. Useful when network restrictions +prevent discovery, such as with SSH tunneling. Default is false. + +### page_size +ENV: <b>TYK_DB_PAGESIZE</b><br /> +Type: `int`<br /> + +The page size that the dashboard should use. Defaults to 10. + +### storage +This option allows you to store different types of data in different databases. For example, logs can be stored in one database, analytics in another, and main resources in another. + +### storage.main +Main database where the dashboard resources are stored (users, orgs, policies, etc) + +### storage.main.type +ENV: <b>TYK_DB_STORAGE_MAIN_TYPE</b><br /> +Type: `DBType`<br /> + +Type is the type of the database. +Possible values are: + - "mongo": Use MongoDB. + - "postgres": Use PostgreSQL. + - "mysql": Use MySQL. + +### storage.main.connection_string +ENV: <b>TYK_DB_STORAGE_MAIN_CONNECTIONSTRING</b><br /> +Type: `string`<br /> + +ConnectionString is the connection string for the database. +Overrides ReadConnectionString and WriteConnectionString. + +### storage.main.read_connection_string +ENV: <b>TYK_DB_STORAGE_MAIN_READCONNECTIONSTRING</b><br /> +Type: `string`<br /> + +ReadConnectionString is the connection string for read operations. +Only used if ConnectionString is not set. + +### storage.main.write_connection_string +ENV: <b>TYK_DB_STORAGE_MAIN_WRITECONNECTIONSTRING</b><br /> +Type: `string`<br /> + +WriteConnectionString is the connection string for write operations. +Only used if ConnectionString is not set. + +### storage.main.mongo +Connection setting for a mongo database + +### storage.main.mongo.driver +ENV: <b>TYK_DB_STORAGE_MAIN_MONGO_DRIVER</b><br /> +Type: `string`<br /> + +Driver to use when connected to a mongo database. It could be `mongo-go` to use the official [mongo driver for go v1.12](https://www.mongodb.com/docs/drivers/go/v1.12/) or `mgo` to use [mgo driver](https://github.com/go-mgo/mgo). Since v5.3, the default value is `mongo-go`. This config is available since dashboard v5.0.2 + +### storage.main.postgres +Connection settings for a Postgres database + +### storage.main.postgres.prefer_simple_protocol +ENV: <b>TYK_DB_STORAGE_MAIN_POSTGRES_PREFERSIMPLEPROTOCOL</b><br /> +Type: `bool`<br /> + +disables implicit prepared statement usage + +### storage.main.mysql +Connection settings for a MySQL database + +### storage.main.mysql.default_string_size +ENV: <b>TYK_DB_STORAGE_MAIN_MYSQL_DEFAULTSTRINGSIZE</b><br /> +Type: `uint`<br /> + +default size for string fields. By default set to: 256 + +### storage.main.mysql.disable_datetime_precision +ENV: <b>TYK_DB_STORAGE_MAIN_MYSQL_DISABLEDATETIMEPRECISION</b><br /> +Type: `bool`<br /> + +disable datetime precision, which not supported before MySQL 5.6 + +### storage.main.mysql.dont_support_rename_index +ENV: <b>TYK_DB_STORAGE_MAIN_MYSQL_DONTSUPPORTRENAMEINDEX</b><br /> +Type: `bool`<br /> + +drop & create when rename index, rename index not supported before MySQL 5.7, MariaDB + +### storage.main.mysql.dont_support_rename_column +ENV: <b>TYK_DB_STORAGE_MAIN_MYSQL_DONTSUPPORTRENAMECOLUMN</b><br /> +Type: `bool`<br /> + +`change` when rename column, rename column not supported before MySQL 8, MariaDB + +### storage.main.mysql.skip_initialize_with_version +ENV: <b>TYK_DB_STORAGE_MAIN_MYSQL_SKIPINITIALIZEWITHVERSION</b><br /> +Type: `bool`<br /> + +auto configure based on currently MySQL version + +### storage.main.table_sharding +ENV: <b>TYK_DB_STORAGE_MAIN_TABLESHARDING</b><br /> +Type: `bool`<br /> + +Enable table sharding for the database + +### storage.analytics +Where all the analytics related data is stored + +### storage.analytics.type +ENV: <b>TYK_DB_STORAGE_ANALYTICS_TYPE</b><br /> +Type: `DBType`<br /> + +Type is the type of the database. +Possible values are: + - "mongo": Use MongoDB. + - "postgres": Use PostgreSQL. + - "mysql": Use MySQL. + +### storage.analytics.connection_string +ENV: <b>TYK_DB_STORAGE_ANALYTICS_CONNECTIONSTRING</b><br /> +Type: `string`<br /> + +ConnectionString is the connection string for the database. +Overrides ReadConnectionString and WriteConnectionString. + +### storage.analytics.read_connection_string +ENV: <b>TYK_DB_STORAGE_ANALYTICS_READCONNECTIONSTRING</b><br /> +Type: `string`<br /> + +ReadConnectionString is the connection string for read operations. +Only used if ConnectionString is not set. + +### storage.analytics.write_connection_string +ENV: <b>TYK_DB_STORAGE_ANALYTICS_WRITECONNECTIONSTRING</b><br /> +Type: `string`<br /> + +WriteConnectionString is the connection string for write operations. +Only used if ConnectionString is not set. + +### storage.analytics.mongo +Connection setting for a mongo database + +### storage.analytics.mongo.driver +ENV: <b>TYK_DB_STORAGE_ANALYTICS_MONGO_DRIVER</b><br /> +Type: `string`<br /> + +Driver to use when connected to a mongo database. It could be `mongo-go` to use the official [mongo driver for go v1.12](https://www.mongodb.com/docs/drivers/go/v1.12/) or `mgo` to use [mgo driver](https://github.com/go-mgo/mgo). Since v5.3, the default value is `mongo-go`. This config is available since dashboard v5.0.2 + +### storage.analytics.postgres +Connection settings for a Postgres database + +### storage.analytics.postgres.prefer_simple_protocol +ENV: <b>TYK_DB_STORAGE_ANALYTICS_POSTGRES_PREFERSIMPLEPROTOCOL</b><br /> +Type: `bool`<br /> + +disables implicit prepared statement usage + +### storage.analytics.mysql +Connection settings for a MySQL database + +### storage.analytics.mysql.default_string_size +ENV: <b>TYK_DB_STORAGE_ANALYTICS_MYSQL_DEFAULTSTRINGSIZE</b><br /> +Type: `uint`<br /> + +default size for string fields. By default set to: 256 + +### storage.analytics.mysql.disable_datetime_precision +ENV: <b>TYK_DB_STORAGE_ANALYTICS_MYSQL_DISABLEDATETIMEPRECISION</b><br /> +Type: `bool`<br /> + +disable datetime precision, which not supported before MySQL 5.6 + +### storage.analytics.mysql.dont_support_rename_index +ENV: <b>TYK_DB_STORAGE_ANALYTICS_MYSQL_DONTSUPPORTRENAMEINDEX</b><br /> +Type: `bool`<br /> + +drop & create when rename index, rename index not supported before MySQL 5.7, MariaDB + +### storage.analytics.mysql.dont_support_rename_column +ENV: <b>TYK_DB_STORAGE_ANALYTICS_MYSQL_DONTSUPPORTRENAMECOLUMN</b><br /> +Type: `bool`<br /> + +`change` when rename column, rename column not supported before MySQL 8, MariaDB + +### storage.analytics.mysql.skip_initialize_with_version +ENV: <b>TYK_DB_STORAGE_ANALYTICS_MYSQL_SKIPINITIALIZEWITHVERSION</b><br /> +Type: `bool`<br /> + +auto configure based on currently MySQL version + +### storage.analytics.table_sharding +ENV: <b>TYK_DB_STORAGE_ANALYTICS_TABLESHARDING</b><br /> +Type: `bool`<br /> + +Enable table sharding for the database + +### storage.logs.type +ENV: <b>TYK_DB_STORAGE_LOGS_TYPE</b><br /> +Type: `DBType`<br /> + +Type is the type of the database. +Possible values are: + - "mongo": Use MongoDB. + - "postgres": Use PostgreSQL. + - "mysql": Use MySQL. + +### storage.logs.connection_string +ENV: <b>TYK_DB_STORAGE_LOGS_CONNECTIONSTRING</b><br /> +Type: `string`<br /> + +ConnectionString is the connection string for the database. +Overrides ReadConnectionString and WriteConnectionString. + +### storage.logs.read_connection_string +ENV: <b>TYK_DB_STORAGE_LOGS_READCONNECTIONSTRING</b><br /> +Type: `string`<br /> + +ReadConnectionString is the connection string for read operations. +Only used if ConnectionString is not set. + +### storage.logs.write_connection_string +ENV: <b>TYK_DB_STORAGE_LOGS_WRITECONNECTIONSTRING</b><br /> +Type: `string`<br /> + +WriteConnectionString is the connection string for write operations. +Only used if ConnectionString is not set. + +### storage.logs.mongo +Connection setting for a mongo database + +### storage.logs.mongo.driver +ENV: <b>TYK_DB_STORAGE_LOGS_MONGO_DRIVER</b><br /> +Type: `string`<br /> + +Driver to use when connected to a mongo database. It could be `mongo-go` to use the official [mongo driver for go v1.12](https://www.mongodb.com/docs/drivers/go/v1.12/) or `mgo` to use [mgo driver](https://github.com/go-mgo/mgo). Since v5.3, the default value is `mongo-go`. This config is available since dashboard v5.0.2 + +### storage.logs.postgres +Connection settings for a Postgres database + +### storage.logs.postgres.prefer_simple_protocol +ENV: <b>TYK_DB_STORAGE_LOGS_POSTGRES_PREFERSIMPLEPROTOCOL</b><br /> +Type: `bool`<br /> + +disables implicit prepared statement usage + +### storage.logs.mysql +Connection settings for a MySQL database + +### storage.logs.mysql.default_string_size +ENV: <b>TYK_DB_STORAGE_LOGS_MYSQL_DEFAULTSTRINGSIZE</b><br /> +Type: `uint`<br /> + +default size for string fields. By default set to: 256 + +### storage.logs.mysql.disable_datetime_precision +ENV: <b>TYK_DB_STORAGE_LOGS_MYSQL_DISABLEDATETIMEPRECISION</b><br /> +Type: `bool`<br /> + +disable datetime precision, which not supported before MySQL 5.6 + +### storage.logs.mysql.dont_support_rename_index +ENV: <b>TYK_DB_STORAGE_LOGS_MYSQL_DONTSUPPORTRENAMEINDEX</b><br /> +Type: `bool`<br /> + +drop & create when rename index, rename index not supported before MySQL 5.7, MariaDB + +### storage.logs.mysql.dont_support_rename_column +ENV: <b>TYK_DB_STORAGE_LOGS_MYSQL_DONTSUPPORTRENAMECOLUMN</b><br /> +Type: `bool`<br /> + +`change` when rename column, rename column not supported before MySQL 8, MariaDB + +### storage.logs.mysql.skip_initialize_with_version +ENV: <b>TYK_DB_STORAGE_LOGS_MYSQL_SKIPINITIALIZEWITHVERSION</b><br /> +Type: `bool`<br /> + +auto configure based on currently MySQL version + +### storage.logs.table_sharding +ENV: <b>TYK_DB_STORAGE_LOGS_TABLESHARDING</b><br /> +Type: `bool`<br /> + +Enable table sharding for the database + +### storage.uptime +Where all the uptime related data is stored + +### storage.uptime.type +ENV: <b>TYK_DB_STORAGE_UPTIME_TYPE</b><br /> +Type: `DBType`<br /> + +Type is the type of the database. +Possible values are: + - "mongo": Use MongoDB. + - "postgres": Use PostgreSQL. + - "mysql": Use MySQL. + +### storage.uptime.connection_string +ENV: <b>TYK_DB_STORAGE_UPTIME_CONNECTIONSTRING</b><br /> +Type: `string`<br /> + +ConnectionString is the connection string for the database. +Overrides ReadConnectionString and WriteConnectionString. + +### storage.uptime.read_connection_string +ENV: <b>TYK_DB_STORAGE_UPTIME_READCONNECTIONSTRING</b><br /> +Type: `string`<br /> + +ReadConnectionString is the connection string for read operations. +Only used if ConnectionString is not set. + +### storage.uptime.write_connection_string +ENV: <b>TYK_DB_STORAGE_UPTIME_WRITECONNECTIONSTRING</b><br /> +Type: `string`<br /> + +WriteConnectionString is the connection string for write operations. +Only used if ConnectionString is not set. + +### storage.uptime.mongo +Connection setting for a mongo database + +### storage.uptime.mongo.driver +ENV: <b>TYK_DB_STORAGE_UPTIME_MONGO_DRIVER</b><br /> +Type: `string`<br /> + +Driver to use when connected to a mongo database. It could be `mongo-go` to use the official [mongo driver for go v1.12](https://www.mongodb.com/docs/drivers/go/v1.12/) or `mgo` to use [mgo driver](https://github.com/go-mgo/mgo). Since v5.3, the default value is `mongo-go`. This config is available since dashboard v5.0.2 + +### storage.uptime.postgres +Connection settings for a Postgres database + +### storage.uptime.postgres.prefer_simple_protocol +ENV: <b>TYK_DB_STORAGE_UPTIME_POSTGRES_PREFERSIMPLEPROTOCOL</b><br /> +Type: `bool`<br /> + +disables implicit prepared statement usage + +### storage.uptime.mysql +Connection settings for a MySQL database + +### storage.uptime.mysql.default_string_size +ENV: <b>TYK_DB_STORAGE_UPTIME_MYSQL_DEFAULTSTRINGSIZE</b><br /> +Type: `uint`<br /> + +default size for string fields. By default set to: 256 + +### storage.uptime.mysql.disable_datetime_precision +ENV: <b>TYK_DB_STORAGE_UPTIME_MYSQL_DISABLEDATETIMEPRECISION</b><br /> +Type: `bool`<br /> + +disable datetime precision, which not supported before MySQL 5.6 + +### storage.uptime.mysql.dont_support_rename_index +ENV: <b>TYK_DB_STORAGE_UPTIME_MYSQL_DONTSUPPORTRENAMEINDEX</b><br /> +Type: `bool`<br /> + +drop & create when rename index, rename index not supported before MySQL 5.7, MariaDB + +### storage.uptime.mysql.dont_support_rename_column +ENV: <b>TYK_DB_STORAGE_UPTIME_MYSQL_DONTSUPPORTRENAMECOLUMN</b><br /> +Type: `bool`<br /> + +`change` when rename column, rename column not supported before MySQL 8, MariaDB + +### storage.uptime.mysql.skip_initialize_with_version +ENV: <b>TYK_DB_STORAGE_UPTIME_MYSQL_SKIPINITIALIZEWITHVERSION</b><br /> +Type: `bool`<br /> + +auto configure based on currently MySQL version + +### storage.uptime.table_sharding +ENV: <b>TYK_DB_STORAGE_UPTIME_TABLESHARDING</b><br /> +Type: `bool`<br /> + +Enable table sharding for the database + +### admin_secret +ENV: <b>TYK_DB_ADMINSECRET</b><br /> +Type: `string`<br /> + +This secret is to be used by a special set of endpoints that we call β€œAdmin APIs”. This API is part of the super-admin context and therefore has a separate endpoint prefix `/admin`. It also requires a special auth header called admin-auth. This purpose of these endpoints is to allow functionality that regular Dashboard users should not have, such as create new organizations, create super users etc. See the [Admin API](https://tyk.io/docs/dashboard-admin-api/) for more information on these endpoints. + +### shared_node_secret +ENV: <b>TYK_DB_NODESECRET</b><br /> +Type: `string`<br /> + +This value should match with the node_secret Gateway configuration option value. +Each node communicates with the Dashboard via a shared secret (this setting) and a nonce to ensure that out-of-band requests cannot be made. Nodes will send a heartbeat every few seconds to notify the Dashboard that they are running. + +### redis_port +ENV: <b>TYK_DB_REDISPORT</b><br /> +Type: `int`<br /> + +The port that your Redis installation listens on. + + +<Note> +The Tyk Dashboard uses Redis to store its session data and to communicate with your Tyk Gateway nodes occasionally. The Redis details used by the dashboard must be the same as those set for your Tyk installation. +</Note> + + +### redis_host +ENV: <b>TYK_DB_REDISHOST</b><br /> +Type: `string`<br /> + +The hostname for the Redis collection and can be an IP address. + +### redis_addrs +ENV: <b>TYK_DB_REDISADDRS</b><br /> +Type: `[]string`<br /> + +Used for configuring Redis clusters. See [Redis Cluster and Tyk Dashboard](https://tyk.io/docs/tyk-open-source/#redis-cluster-and-tyk-dashboard) for more info. Example: +``` + "addrs": [ + "server1:6379", + "server2:6380", + "server3:6381" + ], +``` + +### redis_hosts +ENV: <b>TYK_DB_HOSTS</b><br /> +Type: `map[string]string`<br /> + +**DEPRECATED**. Use `redis_addrs` instead. You can also specify multiple Redis hosts here. Tyk will use this array if it is not empty, or it will use the individual legacy parameters above. You can specify multiple host:port combinations here. + +### redis_username +ENV: <b>TYK_DB_REDISUSERNAME</b><br /> +Type: `string`<br /> + +If you are using Redis AUTH using its `requirepass` setting, enter your username here (recommended). If this is not used, the Dashboard will not attempt to login to Redis. + +### redis_password +ENV: <b>TYK_DB_REDISPASSWORD</b><br /> +Type: `string`<br /> + +The password for your Redis Auth username. + +### redis_master_name +ENV: <b>TYK_DB_REDISMASTERNAME</b><br /> +Type: `string`<br /> + +Redis Sentinel Master name + +### redis_sentinel_password +ENV: <b>TYK_DB_REDISSENTINELPASSWORD</b><br /> +Type: `string`<br /> + +Redis Sentinel password + +### redis_timeout +ENV: <b>TYK_DB_REDISTIMEOUT</b><br /> +Type: `int`<br /> + +Set a custom Redis network timeout. Default value is 5 seconds. + +### redis_database +ENV: <b>TYK_DB_REDISDATABASE</b><br /> +Type: `int`<br /> + +Set this to the index of your Redis database if you are using more than one. + +### enable_cluster +ENV: <b>TYK_DB_ENABLECLUSTER</b><br /> +Type: `bool`<br /> + +Set this to true if you are using a Redis cluster. + +### redis_use_ssl +ENV: <b>TYK_DB_REDISUSESSL</b><br /> +Type: `bool`<br /> + +Use Redis SSL connection + +### redis_ssl_insecure_skip_verify +ENV: <b>TYK_DB_REDISSSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Ignore TLS verification for Redis connections. + +### redis_ca_file +ENV: <b>TYK_DB_REDISCAFILE</b><br /> +Type: `string`<br /> + +Redis SSL CA File + +The SSL CA file is imported into an X509 certificate pool. It +contains the set of root certificate authorities. When establishing +a connection to redis, Tyk will use this to verify server certificates. + +If empty, Tyk will use the host's root CA set. + +### redis_cert_file +ENV: <b>TYK_DB_REDISCERTFILE</b><br /> +Type: `string`<br /> + +Redis SSL Cert file. + +The cert file and the key file combine to form an X509 certificate. +The certificate is presented when establishing a connection to redis. + +For more information, see [crypto/tls#X509KeyPair](https://pkg.go.dev/crypto/tls#X509KeyPair). + +### redis_key_file +ENV: <b>TYK_DB_REDISKEYFILE</b><br /> +Type: `string`<br /> + +Redis SSL Key file. + +The cert file and the key file combine to form an X509 certificate. +The certificate is presented when establishing a connection to redis. + +For more information, see [crypto/tls#X509KeyPair](https://pkg.go.dev/crypto/tls#X509KeyPair). + +### redis_tls_max_version +ENV: <b>TYK_DB_REDISTLSMAXVERSION</b><br /> +Type: `string`<br /> + +Maximum TLS version that is supported. + +Options: ["1.0", "1.1", "1.2", "1.3"]. +Defaults to "1.3". + +### redis_tls_min_version +ENV: <b>TYK_DB_REDISTLSMINVERSION</b><br /> +Type: `string`<br /> + +Minimum TLS version that is supported. + +Options: ["1.0", "1.1", "1.2", "1.3"]. +Defaults to "1.2". + +### redis_max_active +ENV: <b>TYK_DB_REDISMAXACTIVE</b><br /> +Type: `int`<br /> + +Set the number of maximum connections in the Redis connection pool, which defaults to 500. Set to a higher value if you are expecting more traffic. + +### notify_on_change +ENV: <b>TYK_DB_NOTIFYONCHANGE</b><br /> +Type: `bool`<br /> + +Licensed users can use this setting to enable/disable whether the Tyk Dashboard will notify all Tyk Gateway nodes to hot-reload when an API definition is changed. + +### license_key +ENV: <b>TYK_DB_LICENSEKEY</b><br /> +Type: `string`<br /> + +Your Tyk Dashboard license key + +### hash_keys +ENV: <b>TYK_DB_HASHKEYS</b><br /> +Type: `bool`<br /> + +If your Tyk Gateway is using hashed keys, set this value to true so it matches. The Dashboard will now operate in a mode that is compatible with key hashing. + +### disable_key_actions_by_username +ENV: <b>TYK_DB_DISABLEKEYACTIONSBYUSERNAME</b><br /> +Type: `bool`<br /> + +DisableKeyActionsByUsername disables basic auth key operation by username. +When this is set to `true` you are able to search for keys only by keyID or key hash (if `hash_keys` is also set to `true`) +Note that if `hash_keys` is also set to `true` then the keyID will not be provided for APIs secured using basic auth. In this scenario the only search option would be to use key hash +You must configure this setting with the same value in both Gateway and Dashboard + +### enable_delete_key_by_hash +ENV: <b>TYK_DB_ENABLEDELETEKEYBYHASH</b><br /> +Type: `bool`<br /> + +To delete a key by its hash, set this option to true + +### enable_update_key_by_hash +ENV: <b>TYK_DB_ENABLEUPDATEKEYBYHASH</b><br /> +Type: `bool`<br /> + +To update a key by its hash, set this option to true. + +### enable_hashed_keys_listing +ENV: <b>TYK_DB_ENABLEHASHEDKEYSLISTING</b><br /> +Type: `bool`<br /> + +To retrieve a list of all key hash listings, set this option to true. + +### email_backend +Tyk supports an interface-based email back-end system. We support `mandrill`, `sendgrid`, `amazonses` and `mailgun`. See [Outbound Email Configuration](https://tyk.io/docs/configure/outbound-email-configuration/) for more details on configuring these different providers. + +### email_backend.enable_email_notifications +ENV: <b>TYK_DB_EMAILBACKEND_ENABLEEMAILNOTIFICATIONS</b><br /> +Type: `bool`<br /> + +Set to `true` to have Tyk send emails for things such as key approvals and portal sign ups. + +### email_backend.code +ENV: <b>TYK_DB_EMAILBACKEND_CODE</b><br /> +Type: `string`<br /> + +The code of the back-end to use, `mandrill`, `sendgrid`, `amazonses` and `mailgun` are supported. + +### email_backend.settings +ENV: <b>TYK_DB_EMAILBACKEND_SETTINGS</b><br /> +Type: `map[string]string`<br /> + +The custom settings sections for the back end system. + +### email_backend.default_from_email +ENV: <b>TYK_DB_EMAILBACKEND_DEFAULTFROMEMAIL</b><br /> +Type: `string`<br /> + +The address to send email from. + +### email_backend.default_from_name +ENV: <b>TYK_DB_EMAILBACKEND_DEFAULTFROMNAME</b><br /> +Type: `string`<br /> + +The name to use when sending emails. + +### email_backend.dashboard_hostname +ENV: <b>TYK_DB_EMAILBACKEND_DASHBOARDHOSTNAME</b><br /> +Type: `string`<br /> + +Your public dashboard hostname. + +### hide_listen_path +ENV: <b>TYK_DB_HIDELISTENPATH</b><br /> +Type: `bool`<br /> + +If you set this option to `true`, then the listen path will not be editable or visible in the Dashboard. + +### use_sentry +ENV: <b>TYK_DB_USESENTRY</b><br /> +Type: `bool`<br /> + +The Tyk Dashboard has Sentry integration to externalise logging. Set this to true to enable the logger. + +### sentry_code +ENV: <b>TYK_DB_SENTRYCODE</b><br /> +Type: `string`<br /> + +If you have a Sentry setup, or are using Getsentry, you can add the Sentry DSN here and Tyk will begin sending events. + +### sentry_js_code +ENV: <b>TYK_DB_SENTRYJSCODE</b><br /> +Type: `string`<br /> + +To have the Dashboard report Javascript errors to you, add a separate DSN here. + +### enable_master_keys +ENV: <b>TYK_DB_ENABLEMASTERKEYS</b><br /> +Type: `bool`<br /> + +If this is set to true, session objects (key definitions) that do not have explicit access rights set will be allowed by Tyk. This means that keys that are created have access to ALL APIs, which in many cases is unwanted behavior unless you are sure about what you are doing. To use this setting also requires the corresponding Gateway configuration setting `allow_master_keys` to be set to `true`. + +### enable_duplicate_slugs +ENV: <b>TYK_DB_ENABLEDUPLICATESLUGS</b><br /> +Type: `bool`<br /> + +Setting this option to `true` will cause the dashboard to not validate against other listen paths. + +### show_org_id +ENV: <b>TYK_DB_SHOWORGID</b><br /> +Type: `bool`<br /> + +Determines whether the Org ID will be shown in the Users -> Username detail page. This can be useful for quickly identifying your Org ID. + +### host_config +Section to manage dashboard host names and domains + +### host_config.enable_host_names +ENV: <b>TYK_DB_HOSTCONFIG_ENABLEHOSTNAMES</b><br /> +Type: `bool`<br /> + +The Tyk Dashboard can bind the Dashboard application to a specific domain name. Enable this option to have the Dashboard only allow access on a specific domain and 404 on any other host access (not recommended). + +### host_config.disable_org_slug_prefix +ENV: <b>TYK_DB_HOSTCONFIG_DISABLEORGSLUGPREFIX</b><br /> +Type: `bool`<br /> + +By default, for developer portal, Tyk will add orgID prefix. Set to `true` if you have single tenant application or each portal on separate domain. + +### host_config.hostname +ENV: <b>TYK_DB_HOSTCONFIG_HOSTNAME</b><br /> +Type: `string`<br /> + +The hostname to bind the Dashboard to. This must be a proper hostname and not localhost. + +### host_config.override_hostname +ENV: <b>TYK_DB_HOSTCONFIG_GATEWAYHOSTNAME</b><br /> +Type: `string`<br /> + +Set this value to whatever hostname your **Tyk Gateway** is running on. + +### host_config.portal_domains +ENV: <b>TYK_DB_HOSTCONFIG_PORTALDOMAINS</b><br /> +Type: `map[string]string`<br /> + +It is possible to hard-code portal domains (these override settings set by the Dashboard for routing purposes). + +Example: +``` +"portal_domains": { +. "portal.com": "<orgID>" +} +``` + +### host_config.portal_root_path +ENV: <b>TYK_DB_HOSTCONFIG_PORTALROOTPATH</b><br /> +Type: `string`<br /> + +The root path for the portal. + +### host_config.generate_secure_paths +ENV: <b>TYK_DB_HOSTCONFIG_GENERATEHTTPS</b><br /> +Type: `bool`<br /> + +If you prefer to have your URLs start with https, set this option to true. + +### host_config.secure_cookies +ENV: <b>TYK_DB_HOSTCONFIG_SECURECOOKIES</b><br /> +Type: `bool`<br /> + +This enables HTTPS β€œsecure” cookies. + +### http_server_options +This section is reserved for settings relating to the HTTP server that powers the Dashboard. + +### http_server_options.use_ssl +ENV: <b>TYK_DB_HTTPSERVEROPTIONS_USESSL</b><br /> +Type: `bool`<br /> + +Enable to use SSL. + +### http_server_options.certificates +ENV: <b>TYK_DB_HTTPSERVEROPTIONS_CERTIFICATES</b><br /> +Type: `CertsData`<br /> + +Add a certificate block for each domain being covered by the application. + +For example: + +``` +{ + "domain_name": "*.banana.com", + "cert_file": "new.cert.cert", + "key_file": "new.cert.key" +} +``` + +### http_server_options.ssl_certificates +ENV: <b>TYK_DB_HTTPSERVEROPTIONS_SSLCERTIFICATES</b><br /> +Type: `[]string`<br /> + +SSL certificates used by your Gateway server. A list of certificate path to files. + +### http_server_options.min_version +ENV: <b>TYK_DB_HTTPSERVEROPTIONS_MINVERSION</b><br /> +Type: `uint16`<br /> + +Minimum TLS version. Possible values: https://tyk.io/docs/api-management/certificates#supported-tls-versions + +### http_server_options.ssl_ciphers +ENV: <b>TYK_DB_HTTPSERVEROPTIONS_CIPHERSUITES</b><br /> +Type: `[]string`<br /> + +Array of allowed cipher suites as defined at https://golang.org/pkg/crypto/tls/#pkg-constants + +### http_server_options.ssl_insecure_skip_verify +ENV: <b>TYK_DB_HTTPSERVEROPTIONS_SSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Disable TLS verifiation + +### http_server_options.prefer_server_ciphers +ENV: <b>TYK_DB_HTTPSERVEROPTIONS_PREFERSERVERCIPHERSUITES</b><br /> +Type: `bool`<br /> + +PreferServerCipherSuites is a legacy field and has no effect. + +More info: https://github.com/golang/go/issues/45430. + +Deprecated: PreferServerCipherSuites is ignored. + +### security +This section controls login limits for both the Dashboard and the Developer Portal. The path for you audit log is also set here. + +### security.allow_admin_reset_password +ENV: <b>TYK_DB_SECURITY_ALLOWADMINRESETPASSWORD</b><br /> +Type: `bool`<br /> + +This allows an admin user to reset the password of other users. The default is false. + +### security.login_failure_username_limit +ENV: <b>TYK_DB_SECURITY_LOGINFAILUREUSERNAMELIMIT</b><br /> +Type: `int`<br /> + +Controls how many time a user can attempt to log in before being denied access. The default is 0. + +### security.login_failure_ip_limit +ENV: <b>TYK_DB_SECURITY_LOGINFAILUREIPLIMIT</b><br /> +Type: `int`<br /> + +Controls how many times an IP Address can be used to attempt to log in before being denied access. The default is 0. + +### security.login_failure_expiration +ENV: <b>TYK_DB_SECURITY_LOGINFAILUREEXPIRATION</b><br /> +Type: `int`<br /> + +Controls how long before the failure limits are reset in seconds. The default is 900 seconds. + +### security.hide_login_failure_limit_error +ENV: <b>TYK_DB_SECURITY_HIDELOGINFAILURELIMITERROR</b><br /> +Type: `bool`<br /> + +By default it will show message like "Retry in N seconds.". In some secure environments it can be treated as leaking of secure context. This option makes failed login attempt to be shown as standard login failure. + +### security.login_disallow_forward_proxy +ENV: <b>TYK_DB_SECURITY_LOGINDISALLOWFORWARDPROXY</b><br /> +Type: `bool`<br /> + +Set to `true` to allow the Tyk Dashboard login to ignore the host from the `X-Forwarded-For` header when accessing the Dashboard via a proxy. This can be useful for limiting retry attempts. + +### security.audit_log_path +ENV: <b>TYK_DB_SECURITY_AUDITLOGPATH</b><br /> +Type: `string`<br /> + +This sets the path to your audit log and enables audit with default settings. It will log all user actions and response statuses to it. Security information such as passwords are not logged. + +### security.user_password_max_days +ENV: <b>TYK_DB_SECURITY_USERPASSWORDMAXDAYS</b><br /> +Type: `int`<br /> + +Set the maximum lifetime of a password for a user. They will be prompted to reset if password lifetime exceeds the configured expiry value. e.g. if value set to 30 any user password set over 30 days in past will be considered invalid and must be reset. + +### security.enforce_password_history +ENV: <b>TYK_DB_SECURITY_ENFORCEPASSWORDHISTORY</b><br /> +Type: `int`<br /> + +Set a maximum number of previous passwords used by a user that cannot be reused. For example, If set to 5 the user upon setting their password cannot reuse any of their 5 most recently used password for that Tyk user account. + +### security.force_first_login_pw_reset +ENV: <b>TYK_DB_SECURITY_FORCEFIRSTLOGINPWRESET</b><br /> +Type: `bool`<br /> + +A newly created user will be forced to reset their password upon first login. Defaults to false. + +### security.enable_content_security_policy +ENV: <b>TYK_DB_SECURITY_ENABLECONTENTSECURITYPOLICY</b><br /> +Type: `bool`<br /> + +Enable browser Content-Security-Policy, e.g. CSP. The default is false. + +### security.allowed_content_sources +ENV: <b>TYK_DB_SECURITY_ALLOWEDCONTENTSOURCES</b><br /> +Type: `string`<br /> + +If CSP enabled, specify space separated string, with list of allowed resources. + +### security.open_policy +OpenPolicy configuration + +### security.open_policy.enabled +ENV: <b>TYK_DB_SECURITY_OPENPOLICY_ENABLED</b><br /> +Type: `bool`<br /> + +Enable OpenPolicy + +### security.open_policy.debug +ENV: <b>TYK_DB_SECURITY_OPENPOLICY_DEBUG</b><br /> +Type: `bool`<br /> + +Enable OpenPolicy debug mode + +### security.open_policy.enable_api +ENV: <b>TYK_DB_SECURITY_OPENPOLICY_ENABLEAPI</b><br /> +Type: `bool`<br /> + +Enable modify OpenPolicy rules via UI and API + +### security.additional_permissions +ENV: <b>TYK_DB_SECURITY_ADDITIONALPERMISSIONS</b><br /> +Type: `map[ObjectGroup]string`<br /> + +Through this options, you can provide a list of additional permissions, that can be applied for existing or newly created users or user groups. Example: + +``` +{ + "api_developer": "API Developer", + "custom_permission": "Custom Permission" +} +``` + +### security.private_certificate_encoding_secret +ENV: <b>TYK_DB_SECURITY_PRIVATECERTIFICATEENCODINGSECRET</b><br /> +Type: `string`<br /> + +When using SAML with embedded identity broker, is required to upload a certificate that is encoded by the gateway to store it safely, TIB needs the private key as well, hence it needs the same encoding secret so the information is decoded successfully. This value should match with the encoding secret set in the gateway config file, if not set then it will use by default tyk_api_config.secret to attempt to decode the certificate. + +### security.forbid_admin_view_access_token +ENV: <b>TYK_DB_SECURITY_FORBIDADMINVIEWACCESSTOKEN</b><br /> +Type: `bool`<br /> + +ForbidAdminViewAccessToken is a security feature that allows you to prevent user admins from viewing the Dashboard API access tokens of other users. The default is `false`, however we recommend setting this to `true` for enhanced security. + +### security.forbid_admin_reset_access_token +ENV: <b>TYK_DB_SECURITY_FORBIDADMINRESETACCESSTOKEN</b><br /> +Type: `bool`<br /> + +ForbidAdminResetAccessToken is a security feature that allows you to prevent user admins from resetting the Dashboard API access tokens of other users. The default is `false`, however we recommend setting this to `true` for enhanced security. + +### ui +This section controls various settings for the look and feel of the Dashboard UI. + +### ui.languages +ENV: <b>TYK_DB_UI_LANGUAGES</b><br /> +Type: `map[string]string`<br /> + +This section lists the current languages the Dashboard UI supports + +### ui.trial +Trial section defines the information about the cloud trial period. + +### ui.trial.end_date +ENV: <b>TYK_DB_UI_TRIAL_ENDDATE</b><br /> +Type: `int64`<br /> + +EndDate contains the timestamp of end date of the trial in unix UTC timestamp. + +### ui.trial.hubspot_form +HubspotForm contains the hubspot form details. + +### ui.trial.hubspot_form.region +ENV: <b>TYK_DB_UI_TRIAL_HUBSPOTFORM_REGION</b><br /> +Type: `string`<br /> + +The region of the account where the form was created. + +### ui.trial.hubspot_form.portal_id +ENV: <b>TYK_DB_UI_TRIAL_HUBSPOTFORM_PORTALID</b><br /> +Type: `string`<br /> + +The ID of the HubSpot account that the form was created in. + +### ui.trial.hubspot_form.form_id +ENV: <b>TYK_DB_UI_TRIAL_HUBSPOTFORM_FORMID</b><br /> +Type: `string`<br /> + +The form's ID, which is used to retrieve the form definition. + +### ui.hide_help +ENV: <b>TYK_DB_UI_HIDEHELP</b><br /> +Type: `bool`<br /> + +Set to true to hide the help tips. + +### ui.default_lang +ENV: <b>TYK_DB_UI_DEFAULTLANG</b><br /> +Type: `string`<br /> + +This settings sets the default language for the UI. Default setting is `en`. Can be set to any of the other languages listed under `ui.languages`. + +### ui.dont_allow_license_management +ENV: <b>TYK_DB_UI_DONTALLOWLICENSEMANAGEMENT</b><br /> +Type: `bool`<br /> + +Do not allow license management screen + +### ui.labs +Feature flags for the UI + +### ui.dev +ENV: <b>TYK_DB_UI_DEV</b><br /> +Type: `bool`<br /> + +Temporary : Enable dev mode feature on UI + +### ui.onboarding +Onboarding section controls the onboarding quick start wizard. + +### ui.onboarding.enabled +ENV: <b>TYK_DB_UI_ONBOARDING_ENABLED</b><br /> +Type: `bool`<br /> + +Enabled is a boolean flag that enables the onboarding quick start wizard. + +### home_dir +ENV: <b>TYK_DB_HOMEDIR</b><br /> +Type: `string`<br /> + +The path to the home directory of Tyk Dashboard, this must be set in order for Portal templates and other files to be loadable. By default this is `/opt/tyk-dashboard/`. + +### identity_broker +Tyk Dashboard has some preset Tyk Identity Broker configurations set up, for this integration to work, the Dashboard must be able to see an Identity Broker instance. The settings in this section are to enable this integration. + +### identity_broker.enabled +ENV: <b>TYK_DB_TIB_ENABLED</b><br /> +Type: `bool`<br /> + +A boolean setting to enable the TIB integration (otherwise it will not appear in the UI). + +### identity_broker.host +When using external TIB, this is the URL where it's reachable + +### identity_broker.host.connection_string +ENV: <b>TYK_DB_TIB_HOST_CONNECTIONSTRING</b><br /> +Type: `string`<br /> + +The URL to the host. It must be in the form: http://domain:port. +Set this value only if you need to use external Tyk Identity Broker + +### identity_broker.host.secret +ENV: <b>TYK_DB_TIB_HOST_SECRET</b><br /> +Type: `string`<br /> + +The shared secret between TIB and the Dashboard. This ensures all API requests between Dashboard and TIB are valid. + +### identity_broker.ssl_insecure_skip_verify +ENV: <b>TYK_DB_TIB_SSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Skip the TLS verification in the transport layer of the HTTP client. Is intended to have it enable for POC and testing purposes, do not use in production. Defaults to false. + +### use_sharded_analytics +ENV: <b>TYK_DB_USESHARDEDANLAYTICS</b><br /> +Type: `bool`<br /> + +If using the `mongo-pump-selective` pump, where data is written to org-id-specific collections in MongoDB, then enabling this option will switch querying for analytics over to the independent collection entries. + +### enable_aggregate_lookups +ENV: <b>TYK_DB_ENABLEAGGREGATELOOKUPS</b><br /> +Type: `bool`<br /> + +If using the new Aggregate Pump, Tyk Analytics can make use of the newer, faster Analytics lookup, to ensure that this can be made backwards compatible. This option must be set to `true`, in conjunction with the `aggregate_lookup_cutoff` value. + +### aggregate_lookup_cutoff +ENV: <b>TYK_DB_AGGREGATELOOKUPCUTOFF</b><br /> +Type: `string`<br /> + +Set this to a date value of the form `DD/MM/YYYY`. Any analytics queries before this date will fall back to the raw base log data collection (slower). This is to ensure continuity of service and a smooth upgrade process with no loss of data. + +### maintenance_mode +ENV: <b>TYK_DB_MAINTENANCEMODE</b><br /> +Type: `bool`<br /> + +Set to true to enable special maintenance screen for portal and dashboard + +### allow_explicit_policy_id +ENV: <b>TYK_DB_ALLOWEXPLICITPOLICYID</b><br /> +Type: `bool`<br /> + +Set this value to `true` if you planning to use Tyk Sync or Tyk Operator + +### disable_parallel_sessions +ENV: <b>TYK_DB_DISABLEPARALLELSESSIONS</b><br /> +Type: `bool`<br /> + +If set to true, it restricts an account to a single session. When an account logs in, any other open sessions for that account are logged out. + +### dashboard_session_lifetime +ENV: <b>TYK_DB_DASHBOARDSESSIONLIFETIME</b><br /> +Type: `int64`<br /> + +Dashboard session lifetime + +### portal_session_lifetime +ENV: <b>TYK_DB_PORTALSESSIONLIFETIME</b><br /> +Type: `int`<br /> + +Portal session lifetime + +### alternative_dashboard_url +ENV: <b>TYK_DB_ALTERNATIVEDASHBOARDURL</b><br /> +Type: `string`<br /> + +Redirect all dashboard users to another URL + +### sso_permission_defaults +ENV: <b>TYK_DB_SSOPERMISSIONDEFAULTS</b><br /> +Type: `map[ObjectGroup]string`<br /> + +Specify permissions of the user who logged in using Admin SSO API (for example Tyk Identity Broker). See [Dashboard Admin SSO API](https://tyk.io/docs/api-management/dashboard-configuration#single-sign-on-api-1) for more details. + +### sso_default_group_id +ENV: <b>TYK_DB_SSODEFAULTUSERGROUP</b><br /> +Type: `string`<br /> + +Default User Group which will be assigned to SSO users. + +### sso_custom_login_url +ENV: <b>TYK_DB_SSOCUSTOMLOGINURL</b><br /> +Type: `string`<br /> + +Specify a custom dashboard login URL if you are using 3rd party authentication like TIB. + +### sso_custom_portal_login_url +ENV: <b>TYK_DB_SSOCUSTOMPORTALLOGINURL</b><br /> +Type: `string`<br /> + +Specify custom portal login URL if you are using 3rd party authentication like TIB. + +### sso_enable_user_lookup +ENV: <b>TYK_DB_SSOENABLEUSERLOOKUP</b><br /> +Type: `bool`<br /> + +When enabled, if dashboard already have user with given email found, it will be used for the login process + +### sso_custom_login_error_url +ENV: <b>TYK_DB_SSOCUSTOMLOGINERRORURL</b><br /> +Type: `string`<br /> + +SSOCustomLoginErrorURL is an URL to redirect the user in case that SSO fails. If empty the user will be redirected to the error page of dashboard + +### audit +Enable dashboard audit. Example: + +``` expandable +"audit": { + "enabled": true, + "format": "json", + "path": "/tmp/audit.log", + "detailed_recording": false + }, +``` + +Audit records the following fields for json format: + * req_id - unique request ID + * org_id - organization ID + * date - date in RFC1123 format + * timestamp - unix timestamp + * ip - IP address the request originated from + * user - Dashboard user who performed the request + * action - description of the action performed (i.e. Update User`) + * method - HTTP-method of the request + * url - URL of the request + * status - HTTP response status of the request + * diff - provides a diff of changed fields (available only for PUT requests) + * request_dump - HTTP request copy (available if detailed_recording is set to true) + * response_dump - HTTP response copy (available if detailed_recording is set to true) + +### audit.enabled +ENV: <b>TYK_DB_AUDIT_ENABLED</b><br /> +Type: `bool`<br /> + +Enables audit logging, set to false by default. + +### audit.format +ENV: <b>TYK_DB_AUDIT_FORMAT</b><br /> +Type: `string`<br /> + +Format of audit log file. Possible values are `json` and `text` (text is default value) + +### audit.path +ENV: <b>TYK_DB_AUDIT_PATH</b><br /> +Type: `string`<br /> + +Path to the audit log + +### audit.detailed_recording +ENV: <b>TYK_DB_AUDIT_DETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Enables detailed records in the audit log. Set to false by default. If set to `true` then audit log records will contain the http-request (without body) and full http-response including the body` + +### audit.store_type +ENV: <b>TYK_DB_AUDIT_STORETYPE</b><br /> +Type: `string`<br /> + +StoreType defines the method used to store audit logs. +Possible values are: + - "db": Store logs in a database. + - "file": Store logs in a file. + - "no_op": Disable logging (no operation). + +This field allows you to configure how audit logs are persisted. +The default value is "file". + +### enable_multi_org_users +ENV: <b>TYK_DB_ENABLEMULTIORGUSERS</b><br /> +Type: `bool`<br /> + +Enable support for users with the same email for multiple organizations + +### health_check_endpoint_name +ENV: <b>TYK_DB_HEALTHCHECKENDPOINTNAME</b><br /> +Type: `string`<br /> + +Health check endpoint name. Default: /health + +### edge_endpoints +ENV: <b>TYK_DB_EDGEENDPOINTS</b><br /> +Type: `EdgeEndpoints`<br /> + +List of Edge Gateways, that will be displayed in the Dashboard UI, so that you can select to which specific Gateway(s) you want to load an API into. Example: +``` expandable + "edge_endpoints": [ + { + "name": "Private Gateway", + "endpoint": "https://payable-matter-gw.aws-euw2.cloud-ara.tyk.io", + "tags": ["edge", "private-gw"] + }, + { + "name": "Public Gateway", + "endpoint": "video-taped-gokart-gw.aws-usw2.cloud-ara.tyk.io", + "tags": ["edge", "public-gw"] + } + ] +``` + +For every `Edge Gateway` there needs to be defined, its name, the ingress URL and a list of tags that APIs will use for triggering Gateways to load its configuration. +Note: For the Hybrid setup, users must fill in the Gateway URLs manually in the Tyk OAS API Definition servers section. + + +### portal_session_secret +ENV: <b>TYK_DB_PORTALSESSIONSECRET</b><br /> +Type: `string`<br /> + +Portal session secret + +### dcr_ssl_insecure_skip_verify +ENV: <b>TYK_DB_DCRSSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Ignore TLS verification for DCR calls + +### private_key_path +ENV: <b>TYK_DB_PRIVATEKEYPATH</b><br /> +Type: `string`<br /> + +Private key path used to sign notifications coming to the gateways + +### oauth_redirect_uri_separator +ENV: <b>TYK_DB_OAUTHREDIRECTURISEPARATOR</b><br /> +Type: `string`<br /> + +oAuth redirect URI separator + +### statsd_connection_string +ENV: <b>TYK_DB_STATSDCONNECTIONSTRING</b><br /> +Type: `string`<br /> + +Enable StatsD monitoring when set to non empty. StatsD connection string. + +### statsd_prefix +ENV: <b>TYK_DB_STATSDPREFIX</b><br /> +Type: `string`<br /> + +StatsD prefix + +### allow_unsafe_oas +ENV: <b>TYK_DB_ALLOWUNSAFEOAS</b><br /> +Type: `bool`<br /> + +Allow the modification of Tyk OAS APIs via the Tyk Classic API endpoints. Note that this is not recommended but is provided for early adopters and will be deprecated later + +### oas_config +OAS holds the configuration for various OpenAPI-specific functionalities + +### oas_config.validate_examples +ENV: <b>TYK_DB_OAS_VALIDATEEXAMPLES</b><br /> +Type: `bool`<br /> + +ValidateExamples enables validation of values provided in `example` and `examples` fields against the declared schemas in the OpenAPI Document. Defaults to false. + +### oas_config.validate_schema_defaults +ENV: <b>TYK_DB_OAS_VALIDATESCHEMADEFAULTS</b><br /> +Type: `bool`<br /> + +ValidateSchemaDefaults enables validation of values provided in `default` fields against the declared schemas in the OpenAPI Document. Defaults to false. + +### streaming +Streaming holds the configuration for Tyk Streaming functionalities + +### streaming.enabled +ENV: <b>TYK_DB_STREAMING_ENABLED</b><br /> +Type: `bool`<br /> + +This flag enables the Tyk Streaming feature. + +### labs +Experimental and beta features configuration settings + +### disable_telemetry +ENV: <b>TYK_DB_DISABLETELEMETRY</b><br /> +Type: `bool`<br /> + +Enable or disable sending telemetry data such as analytics, API configurations, etc. + diff --git a/snippets/gateway-config.mdx b/snippets/gateway-config.mdx new file mode 100644 index 00000000..f6f16adf --- /dev/null +++ b/snippets/gateway-config.mdx @@ -0,0 +1,2201 @@ +### hostname +ENV: <b>TYK_GW_HOSTNAME</b><br /> +Type: `string`<br /> + +Force your Gateway to work only on a specific domain name. Can be overridden by API custom domain. + +### listen_address +ENV: <b>TYK_GW_LISTENADDRESS</b><br /> +Type: `string`<br /> + +If your machine has multiple network devices or IPs you can force the Gateway to use the IP address you want. + +### listen_port +ENV: <b>TYK_GW_LISTENPORT</b><br /> +Type: `int`<br /> + +Setting this value will change the port that Tyk listens on. Default: 8080. + +### control_api_hostname +ENV: <b>TYK_GW_CONTROLAPIHOSTNAME</b><br /> +Type: `string`<br /> + +Custom hostname for the Control API + +### control_api_port +ENV: <b>TYK_GW_CONTROLAPIPORT</b><br /> +Type: `int`<br /> + +Set this to expose the Tyk Gateway API on a separate port. You can protect it behind a firewall if needed. Please make sure you follow this guide when setting the control port https://tyk.io/docs/tyk-self-managed/#change-your-control-port. + +### secret +ENV: <b>TYK_GW_SECRET</b><br /> +Type: `string`<br /> + +This should be changed as soon as Tyk is installed on your system. +This value is used in every interaction with the Tyk Gateway API. It should be passed along as the X-Tyk-Authorization header in any requests made. +Tyk assumes that you are sensible enough not to expose the management endpoints publicly and to keep this configuration value to yourself. + +### node_secret +ENV: <b>TYK_GW_NODESECRET</b><br /> +Type: `string`<br /> + +The shared secret between the Gateway and the Dashboard to ensure that API Definition downloads, heartbeat and Policy loads are from a valid source. + +### pid_file_location +ENV: <b>TYK_GW_PIDFILELOCATION</b><br /> +Type: `string`<br /> + +Linux PID file location. Do not change unless you know what you are doing. Default: /var/run/tyk/tyk-gateway.pid + +### allow_insecure_configs +ENV: <b>TYK_GW_ALLOWINSECURECONFIGS</b><br /> +Type: `bool`<br /> + +Can be set to disable Dashboard message signature verification. When set to `true`, `public_key_path` can be ignored. + +### public_key_path +ENV: <b>TYK_GW_PUBLICKEYPATH</b><br /> +Type: `string`<br /> + +While communicating with the Dashboard. By default, all messages are signed by a private/public key pair. Set path to public key. + +### allow_remote_config +ENV: <b>TYK_GW_ALLOWREMOTECONFIG</b><br /> +Type: `bool`<br /> + +Allow your Dashboard to remotely set Gateway configuration via the Nodes screen. + +### security +Global Certificate configuration + +### security.private_certificate_encoding_secret +ENV: <b>TYK_GW_SECURITY_PRIVATECERTIFICATEENCODINGSECRET</b><br /> +Type: `string`<br /> + +Set the AES256 secret which is used to encode certificate private keys when they uploaded via certificate storage + +### security.control_api_use_mutual_tls +ENV: <b>TYK_GW_SECURITY_CONTROLAPIUSEMUTUALTLS</b><br /> +Type: `bool`<br /> + +Enable Gateway Control API to use Mutual TLS. Certificates can be set via `security.certificates.control_api` section + +### security.pinned_public_keys +ENV: <b>TYK_GW_SECURITY_PINNEDPUBLICKEYS</b><br /> +Type: `map[string]string`<br /> + +Specify public keys used for Certificate Pinning on global level. + +### security.certificates.upstream +ENV: <b>TYK_GW_SECURITY_CERTIFICATES_UPSTREAM</b><br /> +Type: `map[string]string`<br /> + +Upstream is used to specify the certificates to be used in mutual TLS connections to upstream services. These are set at gateway level as a map of domain -> certificate id or path. +For example if you want Tyk to use the certificate `ab23ef123` for requests to the `example.com` upstream and `/certs/default.pem` for all other upstreams then: +In `tyk.conf` you would configure `"security": {"certificates": {"upstream": {"*": "/certs/default.pem", "example.com": "ab23ef123"}}}` +And if using environment variables you would set this to `*:/certs/default.pem,example.com:ab23ef123`. + +### security.certificates.control_api +ENV: <b>TYK_GW_SECURITY_CERTIFICATES_CONTROLAPI</b><br /> +Type: `[]string`<br /> + +Certificates used for Control API Mutual TLS + +### security.certificates.dashboard_api +ENV: <b>TYK_GW_SECURITY_CERTIFICATES_DASHBOARD</b><br /> +Type: `[]string`<br /> + +Used for communicating with the Dashboard if it is configured to use Mutual TLS + +### security.certificates.mdcb_api +ENV: <b>TYK_GW_SECURITY_CERTIFICATES_MDCB</b><br /> +Type: `[]string`<br /> + +Certificates used for MDCB Mutual TLS + +### http_server_options +Gateway HTTP server configuration + +### http_server_options.read_timeout +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_READTIMEOUT</b><br /> +Type: `int`<br /> + +API Consumer -> Gateway network read timeout. Not setting this config, or setting this to 0, defaults to 120 seconds + +### http_server_options.write_timeout +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_WRITETIMEOUT</b><br /> +Type: `int`<br /> + +API Consumer -> Gateway network write timeout. Not setting this config, or setting this to 0, defaults to 120 seconds + +### http_server_options.use_ssl +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_USESSL</b><br /> +Type: `bool`<br /> + +Set to true to enable SSL connections + +### http_server_options.enable_http2 +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_ENABLEHTTP2</b><br /> +Type: `bool`<br /> + +Enable HTTP2 protocol handling + +### http_server_options.enable_strict_routes +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_ENABLESTRICTROUTES</b><br /> +Type: `bool`<br /> + +EnableStrictRoutes changes the routing to avoid nearest-neighbour requests on overlapping routes + +- if disabled, `/apple` will route to `/app`, the current default behavior, +- if enabled, `/app` only responds to `/app`, `/app/` and `/app/*` but not `/apple` + +Regular expressions and parameterized routes will be left alone regardless of this setting. + +### http_server_options.enable_path_prefix_matching +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_ENABLEPATHPREFIXMATCHING</b><br /> +Type: `bool`<br /> + +EnablePathPrefixMatching changes how the gateway matches incoming URL paths against routes (patterns) defined in the API definition. +By default, the gateway uses wildcard matching. When EnablePathPrefixMatching is enabled, it switches to prefix matching. For example, a defined path such as `/json` will only match request URLs that begin with `/json`, rather than matching any URL containing `/json`. + +The gateway checks the request URL against several variations depending on whether path versioning is enabled: +- Full path (listen path + version + endpoint): `/listen-path/v4/json` +- Non-versioned full path (listen path + endpoint): `/listen-path/json` +- Path without version (endpoint only): `/json` + +For patterns that start with `/`, the gateway prepends `^` before performing the check, ensuring a true prefix match. +For patterns that start with `^`, the gateway will already perform prefix matching so EnablePathPrefixMatching will have no impact. +This option allows for more specific and controlled routing of API requests, potentially reducing unintended matches. Note that you may need to adjust existing route definitions when enabling this option. + +Example: + +With wildcard matching, `/json` might match `/api/v1/data/json`. +With prefix matching, `/json` would not match `/api/v1/data/json`, but would match `/json/data`. + +Combining EnablePathPrefixMatching with EnablePathSuffixMatching will result in exact URL matching, with `/json` being evaluated as `^/json$`. + +### http_server_options.enable_path_suffix_matching +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_ENABLEPATHSUFFIXMATCHING</b><br /> +Type: `bool`<br /> + +EnablePathSuffixMatching changes how the gateway matches incoming URL paths against routes (patterns) defined in the API definition. +By default, the gateway uses wildcard matching. When EnablePathSuffixMatching is enabled, it switches to suffix matching. For example, a defined path such as `/json` will only match request URLs that end with `/json`, rather than matching any URL containing `/json`. + +The gateway checks the request URL against several variations depending on whether path versioning is enabled: +- Full path (listen path + version + endpoint): `/listen-path/v4/json` +- Non-versioned full path (listen path + endpoint): `/listen-path/json` +- Path without version (endpoint only): `/json` + +For patterns that already end with `$`, the gateway will already perform suffix matching so EnablePathSuffixMatching will have no impact. For all other patterns, the gateway appends `$` before performing the check, ensuring a true suffix match. +This option allows for more specific and controlled routing of API requests, potentially reducing unintended matches. Note that you may need to adjust existing route definitions when enabling this option. + +Example: + +With wildcard matching, `/json` might match `/api/v1/json/data`. +With suffix matching, `/json` would not match `/api/v1/json/data`, but would match `/api/v1/json`. + +Combining EnablePathSuffixMatching with EnablePathPrefixMatching will result in exact URL matching, with `/json` being evaluated as `^/json$`. + +### http_server_options.ssl_insecure_skip_verify +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_SSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Disable TLS verification. Required if you are using self-signed certificates. + +### http_server_options.enable_websockets +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_ENABLEWEBSOCKETS</b><br /> +Type: `bool`<br /> + +Enabled WebSockets and server side events support + +### http_server_options.certificates +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_CERTIFICATES</b><br /> +Type: `CertsData`<br /> + +Deprecated: Use `ssl_certificates`instead. + +### http_server_options.ssl_certificates +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_SSLCERTIFICATES</b><br /> +Type: `[]string`<br /> + +Index of certificates available to the Gateway for use in client and upstream communication. +The string value in the array can be two of the following options: +1. The ID assigned to and used to identify a certificate in the Tyk Certificate Store +2. The path to a file accessible to the Gateway. This PEM file must contain the private key and public certificate pair concatenated together. + +### http_server_options.server_name +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_SERVERNAME</b><br /> +Type: `string`<br /> + +Start your Gateway HTTP server on specific server name + +### http_server_options.min_version +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_MINVERSION</b><br /> +Type: `uint16`<br /> + +Minimum TLS version. Possible values: https://tyk.io/docs/api-management/certificates#supported-tls-versions + +### http_server_options.max_version +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_MAXVERSION</b><br /> +Type: `uint16`<br /> + +Maximum TLS version. + +### http_server_options.skip_client_ca_announcement +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_SKIPCLIENTCAANNOUNCEMENT</b><br /> +Type: `bool`<br /> + +When mTLS enabled, this option allows to skip client CA announcement in the TLS handshake. +This option is useful when you have a lot of ClientCAs and you want to reduce the handshake overhead, as some clients can hit TLS handshake limits. +This option does not give any hints to the client, on which certificate to pick (but this is very rare situation when it is required) + +### http_server_options.flush_interval +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_FLUSHINTERVAL</b><br /> +Type: `int`<br /> + +Set this to the number of seconds that Tyk uses to flush content from the proxied upstream connection to the open downstream connection. +This option needed be set for streaming protocols like Server Side Events, or gRPC streaming. + +### http_server_options.skip_url_cleaning +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_SKIPURLCLEANING</b><br /> +Type: `bool`<br /> + +Allow the use of a double slash in a URL path. This can be useful if you need to pass raw URLs to your API endpoints. +For example: `http://myapi.com/get/http://example.com`. + +### http_server_options.skip_target_path_escaping +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_SKIPTARGETPATHESCAPING</b><br /> +Type: `bool`<br /> + +Disable automatic character escaping, allowing to path original URL data to the upstream. + +### http_server_options.ssl_ciphers +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_CIPHERS</b><br /> +Type: `[]string`<br /> + +Custom SSL ciphers applicable when using TLS version 1.2. See the list of ciphers here https://tyk.io/docs/api-management/certificates#supported-tls-cipher-suites + +### http_server_options.max_request_body_size +ENV: <b>TYK_GW_HTTPSERVEROPTIONS_MAXREQUESTBODYSIZE</b><br /> +Type: `int64`<br /> + +MaxRequestBodySize configures a maximum size limit for request body size (in bytes) for all APIs on the Gateway. + +Tyk Gateway will evaluate all API requests against this size limit and will respond with HTTP 413 status code if the body of the request is larger. + +Two methods are used to perform the comparison: + - If the API Request contains the `Content-Length` header, this is directly compared against `MaxRequestBodySize`. + - If the `Content-Length` header is not provided, the Request body is read in chunks to compare total size against `MaxRequestBodySize`. + +A value of zero (default) means that no maximum is set and API requests will not be tested. + +See more information about setting request size limits here: +https://tyk.io/docs/api-management/traffic-transformation/#request-size-limits + +### version_header +ENV: <b>TYK_GW_VERSIONHEADER</b><br /> +Type: `string`<br /> + +Expose version header with a given name. Works only for versioned APIs. + +### suppress_redis_signal_reload +ENV: <b>TYK_GW_SUPPRESSREDISSIGNALRELOAD</b><br /> +Type: `bool`<br /> + +Disable dynamic API and Policy reloads, e.g. it will load new changes only on procecss start. + +### reload_interval +ENV: <b>TYK_GW_RELOADINTERVAL</b><br /> +Type: `int64`<br /> + +ReloadInterval defines a duration in seconds within which the gateway responds to a reload event. +The value defaults to 1, values lower than 1 are ignored. + +### hash_keys +ENV: <b>TYK_GW_HASHKEYS</b><br /> +Type: `bool`<br /> + +Enable Key hashing + +### disable_key_actions_by_username +ENV: <b>TYK_GW_DISABLEKEYACTIONSBYUSERNAME</b><br /> +Type: `bool`<br /> + +DisableKeyActionsByUsername disables key search by username. +When this is set to `true` you are able to search for keys only by keyID or key hash (if `hash_keys` is also set to `true`) +Note that if `hash_keys` is also set to `true` then the keyID will not be provided for APIs secured using basic auth. In this scenario the only search option would be to use key hash +If you are using the Tyk Dashboard, you must configure this setting with the same value in both Gateway and Dashboard + +### hash_key_function +ENV: <b>TYK_GW_HASHKEYFUNCTION</b><br /> +Type: `string`<br /> + +Specify the Key hashing algorithm. Possible values: murmur64, murmur128, sha256. + +### basic_auth_hash_key_function +ENV: <b>TYK_GW_BASICAUTHHASHKEYFUNCTION</b><br /> +Type: `string`<br /> + +Specify the Key hashing algorithm for "basic auth". Possible values: murmur64, murmur128, sha256, bcrypt. +Will default to "bcrypt" if not set. + +### hash_key_function_fallback +ENV: <b>TYK_GW_HASHKEYFUNCTIONFALLBACK</b><br /> +Type: `[]string`<br /> + +Specify your previous key hashing algorithm if you migrated from one algorithm to another. + +### enable_hashed_keys_listing +ENV: <b>TYK_GW_ENABLEHASHEDKEYSLISTING</b><br /> +Type: `bool`<br /> + +Allows the listing of hashed API keys + +### min_token_length +ENV: <b>TYK_GW_MINTOKENLENGTH</b><br /> +Type: `int`<br /> + +Minimum API token length + +### template_path +ENV: <b>TYK_GW_TEMPLATEPATH</b><br /> +Type: `string`<br /> + +Path to error and webhook templates. Defaults to the current binary path. + +### policies +The policies section allows you to define where Tyk can find its policy templates. Policy templates are similar to key definitions in that they allow you to set quotas, access rights and rate limits for keys. +Policies are loaded when Tyk starts and if changed require a hot-reload so they are loaded into memory. +A policy can be defined in a file (Open Source installations) or from the same database as the Dashboard. + +### policies.policy_source +ENV: <b>TYK_GW_POLICIES_POLICYSOURCE</b><br /> +Type: `string`<br /> + +Set this value to `file` to look in the file system for a definition file. Set to `service` to use the Dashboard service. + +### policies.policy_connection_string +ENV: <b>TYK_GW_POLICIES_POLICYCONNECTIONSTRING</b><br /> +Type: `string`<br /> + +This option is required if `policies.policy_source` is set to `service`. +Set this to the URL of your Tyk Dashboard installation. The URL needs to be formatted as: http://dashboard_host:port. + +### policies.policy_record_name +ENV: <b>TYK_GW_POLICIES_POLICYRECORDNAME</b><br /> +Type: `string`<br /> + +This option only applies in OSS deployment when the `policies.policy_source` is either set +to `file` or an empty string. If `policies.policy_path` is not set, then Tyk will load policies +from the JSON file specified by `policies.policy_record_name`. + +### policies.allow_explicit_policy_id +ENV: <b>TYK_GW_POLICIES_ALLOWEXPLICITPOLICYID</b><br /> +Type: `bool`<br /> + +In a Pro installation, Tyk will load Policy IDs and use the internal object-ID as the ID of the policy. +This is not portable in cases where the data needs to be moved from installation to installation. + +If you set this value to `true`, then the id parameter in a stored policy (or imported policy using the Dashboard API), will be used instead of the internal ID. + +This option should only be used when moving an installation to a new database. + +### policies.policy_path +ENV: <b>TYK_GW_POLICIES_POLICYPATH</b><br /> +Type: `string`<br /> + +This option only applies in OSS deployment when the `policies.policy_source` is either set +to `file` or an empty string. If `policies.policy_path` is set, then Tyk will load policies +from all the JSON files under the directory specified by the `policies.policy_path` option. +In this configuration, Tyk Gateway will allow policy management through the Gateway API. + +### ports_whitelist +ENV: <b>TYK_GW_PORTWHITELIST</b><br /> +Type: `PortsWhiteList`<br /> + +Defines the ports that will be available for the API services to bind to in the format +documented here https://tyk.io/docs/api-management/non-http-protocols/#allowing-specific-ports. +Ports can be configured per protocol, e.g. https, tls etc. +If configuring via environment variable `TYK_GW_PORTWHITELIST` then remember to escape +JSON strings. + +### disable_ports_whitelist +ENV: <b>TYK_GW_DISABLEPORTWHITELIST</b><br /> +Type: `bool`<br /> + +Disable port whilisting, essentially allowing you to use any port for your API. + +### app_path +ENV: <b>TYK_GW_APPPATH</b><br /> +Type: `string`<br /> + +If Tyk is being used in its standard configuration (Open Source installations), then API definitions are stored in the apps folder (by default in /opt/tyk-gateway/apps). +This location is scanned for .json files and re-scanned at startup or reload. +See the API section of the Tyk Gateway API for more details. + +### use_db_app_configs +ENV: <b>TYK_GW_USEDBAPPCONFIGS</b><br /> +Type: `bool`<br /> + +If you are a Tyk Pro user, this option will enable polling the Dashboard service for API definitions. +On startup Tyk will attempt to connect and download any relevant application configurations from from your Dashboard instance. +The files are exactly the same as the JSON files on disk with the exception of a BSON ID supplied by the Dashboard service. + +### db_app_conf_options +This section defines API loading and shard options. Enable these settings to selectively load API definitions on a node from your Dashboard service. + +### db_app_conf_options.connection_string +ENV: <b>TYK_GW_DBAPPCONFOPTIONS_CONNECTIONSTRING</b><br /> +Type: `string`<br /> + +Set the URL to your Dashboard instance (or a load balanced instance). The URL needs to be formatted as: `http://dashboard_host:port` + +### db_app_conf_options.connection_timeout +ENV: <b>TYK_GW_DBAPPCONFOPTIONS_CONNECTIONTIMEOUT</b><br /> +Type: `int`<br /> + +Set a timeout value, in seconds, for your Dashboard connection. Default value is 30. + +### db_app_conf_options.node_is_segmented +ENV: <b>TYK_GW_DBAPPCONFOPTIONS_NODEISSEGMENTED</b><br /> +Type: `bool`<br /> + +Set to `true` to enable filtering (sharding) of APIs. + +### db_app_conf_options.tags +ENV: <b>TYK_GW_DBAPPCONFOPTIONS_TAGS</b><br /> +Type: `[]string`<br /> + +The tags to use when filtering (sharding) Tyk Gateway nodes. Tags are processed as `OR` operations. +If you include a non-filter tag (e.g. an identifier such as `node-id-1`, this will become available to your Dashboard analytics). + +### storage +This section defines your Redis configuration. + +### storage.type +ENV: <b>TYK_GW_STORAGE_TYPE</b><br /> +Type: `string`<br /> + +This should be set to `redis` (lowercase) + +### storage.host +ENV: <b>TYK_GW_STORAGE_HOST</b><br /> +Type: `string`<br /> + +The Redis host, by default this is set to `localhost`, but for production this should be set to a cluster. + +### storage.port +ENV: <b>TYK_GW_STORAGE_PORT</b><br /> +Type: `int`<br /> + +The Redis instance port. + +### storage.addrs +ENV: <b>TYK_GW_STORAGE_ADDRS</b><br /> +Type: `[]string`<br /> + +If you have multi-node setup, you should use this field instead. For example: ["host1:port1", "host2:port2"]. + +### storage.master_name +ENV: <b>TYK_GW_STORAGE_MASTERNAME</b><br /> +Type: `string`<br /> + +Redis sentinel master name + +### storage.sentinel_password +ENV: <b>TYK_GW_STORAGE_SENTINELPASSWORD</b><br /> +Type: `string`<br /> + +Redis sentinel password + +### storage.username +ENV: <b>TYK_GW_STORAGE_USERNAME</b><br /> +Type: `string`<br /> + +Redis user name + +### storage.password +ENV: <b>TYK_GW_STORAGE_PASSWORD</b><br /> +Type: `string`<br /> + +If your Redis instance has a password set for access, you can set it here. + +### storage.database +ENV: <b>TYK_GW_STORAGE_DATABASE</b><br /> +Type: `int`<br /> + +Redis database + +### storage.optimisation_max_idle +ENV: <b>TYK_GW_STORAGE_MAXIDLE</b><br /> +Type: `int`<br /> + +Set the number of maximum idle connections in the Redis connection pool, which defaults to 100. Set to a higher value if you are expecting more traffic. + +### storage.optimisation_max_active +ENV: <b>TYK_GW_STORAGE_MAXACTIVE</b><br /> +Type: `int`<br /> + +Set the number of maximum connections in the Redis connection pool, which defaults to 500. Set to a higher value if you are expecting more traffic. + +### storage.timeout +ENV: <b>TYK_GW_STORAGE_TIMEOUT</b><br /> +Type: `int`<br /> + +Set a custom timeout for Redis network operations. Default value 5 seconds. + +### storage.enable_cluster +ENV: <b>TYK_GW_STORAGE_ENABLECLUSTER</b><br /> +Type: `bool`<br /> + +Enable Redis Cluster support + +### storage.use_ssl +ENV: <b>TYK_GW_STORAGE_USESSL</b><br /> +Type: `bool`<br /> + +Enable SSL/TLS connection between your Tyk Gateway & Redis. + +### storage.ssl_insecure_skip_verify +ENV: <b>TYK_GW_STORAGE_SSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Disable TLS verification + +### storage.ca_file +ENV: <b>TYK_GW_STORAGE_CAFILE</b><br /> +Type: `string`<br /> + +Path to the CA file. + +### storage.cert_file +ENV: <b>TYK_GW_STORAGE_CERTFILE</b><br /> +Type: `string`<br /> + +Path to the cert file. + +### storage.key_file +ENV: <b>TYK_GW_STORAGE_KEYFILE</b><br /> +Type: `string`<br /> + +Path to the key file. + +### storage.tls_max_version +ENV: <b>TYK_GW_STORAGE_TLSMAXVERSION</b><br /> +Type: `string`<br /> + +Maximum TLS version that is supported. +Options: ["1.0", "1.1", "1.2", "1.3"]. +Defaults to "1.3". + +### storage.tls_min_version +ENV: <b>TYK_GW_STORAGE_TLSMINVERSION</b><br /> +Type: `string`<br /> + +Minimum TLS version that is supported. +Options: ["1.0", "1.1", "1.2", "1.3"]. +Defaults to "1.2". + +### disable_dashboard_zeroconf +ENV: <b>TYK_GW_DISABLEDASHBOARDZEROCONF</b><br /> +Type: `bool`<br /> + +Disable the capability of the Gateway to `autodiscover` the Dashboard through heartbeat messages via Redis. +The goal of zeroconf is auto-discovery, so you do not have to specify the Tyk Dashboard address in your Gateway`tyk.conf` file. +In some specific cases, for example, when the Dashboard is bound to a public domain, not accessible inside an internal network, or similar, `disable_dashboard_zeroconf` can be set to `true`, in favor of directly specifying a Tyk Dashboard address. + +### slave_options +The `slave_options` allow you to configure the RPC slave connection required for MDCB installations. +These settings must be configured for every RPC slave/worker node. + +### slave_options.use_rpc +ENV: <b>TYK_GW_SLAVEOPTIONS_USERPC</b><br /> +Type: `bool`<br /> + +Set to `true` to connect a worker Gateway using RPC. + +### slave_options.use_ssl +ENV: <b>TYK_GW_SLAVEOPTIONS_USESSL</b><br /> +Type: `bool`<br /> + +Set this option to `true` to use an SSL RPC connection. + +### slave_options.ssl_insecure_skip_verify +ENV: <b>TYK_GW_SLAVEOPTIONS_SSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Set this option to `true` to allow the certificate validation (certificate chain and hostname) to be skipped. +This can be useful if you use a self-signed certificate. + +### slave_options.connection_string +ENV: <b>TYK_GW_SLAVEOPTIONS_CONNECTIONSTRING</b><br /> +Type: `string`<br /> + +Use this setting to add the URL for your MDCB or load balancer host. + +### slave_options.rpc_key +ENV: <b>TYK_GW_SLAVEOPTIONS_RPCKEY</b><br /> +Type: `string`<br /> + +Your organization ID to connect to the MDCB installation. + +### slave_options.api_key +ENV: <b>TYK_GW_SLAVEOPTIONS_APIKEY</b><br /> +Type: `string`<br /> + +This the API key of a user used to authenticate and authorize the Gateway’s access through MDCB. +The user should be a standard Dashboard user with minimal privileges so as to reduce any risk if the user is compromised. +The suggested security settings are read for Real-time notifications and the remaining options set to deny. + +### slave_options.enable_rpc_cache +ENV: <b>TYK_GW_SLAVEOPTIONS_ENABLERPCCACHE</b><br /> +Type: `bool`<br /> + +Set this option to `true` to enable RPC caching for keys. + +### slave_options.bind_to_slugs +ENV: <b>TYK_GW_SLAVEOPTIONS_BINDTOSLUGSINSTEADOFLISTENPATHS</b><br /> +Type: `bool`<br /> + +For an Self-Managed installation this can be left at `false` (the default setting). For Legacy Cloud Gateways it must be set to β€˜true’. + +### slave_options.disable_keyspace_sync +ENV: <b>TYK_GW_SLAVEOPTIONS_DISABLEKEYSPACESYNC</b><br /> +Type: `bool`<br /> + +Set this option to `true` if you don’t want to monitor changes in the keys from a primary Gateway. + +### slave_options.group_id +ENV: <b>TYK_GW_SLAVEOPTIONS_GROUPID</b><br /> +Type: `string`<br /> + +This is the `zone` that this instance inhabits, e.g. the cluster/data-center the Gateway lives in. +The group ID must be the same across all the Gateways of a data-center/cluster which are also sharing the same Redis instance. +This ID should also be unique per cluster (otherwise another Gateway cluster can pick up your keyspace events and your cluster will get zero updates). + +### slave_options.call_timeout +ENV: <b>TYK_GW_SLAVEOPTIONS_CALLTIMEOUT</b><br /> +Type: `int`<br /> + +Call Timeout allows to specify a time in seconds for the maximum allowed duration of a RPC call. + +### slave_options.ping_timeout +ENV: <b>TYK_GW_SLAVEOPTIONS_PINGTIMEOUT</b><br /> +Type: `int`<br /> + +The maximum time in seconds that a RPC ping can last. + +### slave_options.rpc_pool_size +ENV: <b>TYK_GW_SLAVEOPTIONS_RPCPOOLSIZE</b><br /> +Type: `int`<br /> + +The number of RPC connections in the pool. Basically it creates a set of connections that you can re-use as needed. Defaults to 5. + +### slave_options.key_space_sync_interval +ENV: <b>TYK_GW_SLAVEOPTIONS_KEYSPACESYNCINTERVAL</b><br /> +Type: `float32`<br /> + +You can use this to set a period for which the Gateway will check if there are changes in keys that must be synchronized. If this value is not set then it will default to 10 seconds. + +### slave_options.rpc_cert_cache_expiration +ENV: <b>TYK_GW_SLAVEOPTIONS_RPCCERTCACHEEXPIRATION</b><br /> +Type: `float32`<br /> + +RPCCertCacheExpiration defines the expiration time of the rpc cache that stores the certificates, defined in seconds + +### slave_options.rpc_global_cache_expiration +ENV: <b>TYK_GW_SLAVEOPTIONS_RPCGLOBALCACHEEXPIRATION</b><br /> +Type: `float32`<br /> + +RPCKeysCacheExpiration defines the expiration time of the rpc cache that stores the keys, defined in seconds + +### slave_options.synchroniser_enabled +ENV: <b>TYK_GW_SLAVEOPTIONS_SYNCHRONISERENABLED</b><br /> +Type: `bool`<br /> + +SynchroniserEnabled enable this config if MDCB has enabled the synchoniser. If disabled then it will ignore signals to synchonise recources + +### management_node +ENV: <b>TYK_GW_MANAGEMENTNODE</b><br /> +Type: `bool`<br /> + +If set to `true`, distributed rate limiter will be disabled for this node, and it will be excluded from any rate limit calculation. + + +<Note> +If you set `db_app_conf_options.node_is_segmented` to `true` for multiple Gateway nodes, you should ensure that `management_node` is set to `false`. +This is to ensure visibility for the management node across all APIs. +</Note> + +For pro installations, `management_node` is not a valid configuration option. +Always set `management_node` to `false` in pro environments. + + +### auth_override +This is used as part of the RPC / Hybrid back-end configuration in a Tyk Enterprise installation and isn’t used anywhere else. + +### enable_fixed_window_rate_limiter +ENV: <b>TYK_GW_ENABLEFIXEDWINDOWRATELIMITER</b><br /> +Type: `bool`<br /> + +EnableFixedWindow enables fixed window rate limiting. + +### enable_redis_rolling_limiter +ENV: <b>TYK_GW_ENABLEREDISROLLINGLIMITER</b><br /> +Type: `bool`<br /> + +Redis based rate limiter with sliding log. Provides 100% rate limiting accuracy, but require two additional Redis roundtrips for each request. + +### enable_sentinel_rate_limiter +ENV: <b>TYK_GW_ENABLESENTINELRATELIMITER</b><br /> +Type: `bool`<br /> + +To enable, set to `true`. The sentinel-based rate limiter delivers a smoother performance curve as rate-limit calculations happen off-thread, but a stricter time-out based cool-down for clients. For example, when a throttling action is triggered, they are required to cool-down for the period of the rate limit. +Disabling the sentinel based rate limiter will make rate-limit calculations happen on-thread and therefore offers a staggered cool-down and a smoother rate-limit experience for the client. +For example, you can slow your connection throughput to regain entry into your rate limit. This is more of a β€œthrottle” than a β€œblock”. +The standard rate limiter offers similar performance as the sentinel-based limiter. This is disabled by default. + +### enable_rate_limit_smoothing +ENV: <b>TYK_GW_ENABLERATELIMITSMOOTHING</b><br /> +Type: `bool`<br /> + +EnableRateLimitSmoothing enables or disables rate limit smoothing. The rate smoothing is only supported on the +Redis Rate Limiter, or the Sentinel Rate Limiter, as both algorithms implement a sliding log. + +### enable_non_transactional_rate_limiter +ENV: <b>TYK_GW_ENABLENONTRANSACTIONALRATELIMITER</b><br /> +Type: `bool`<br /> + +An enhancement for the Redis and Sentinel rate limiters, that offers a significant improvement in performance by not using transactions on Redis rate-limit buckets. + +### drl_notification_frequency +ENV: <b>TYK_GW_DRLNOTIFICATIONFREQUENCY</b><br /> +Type: `int`<br /> + +How frequently a distributed rate limiter synchronises information between the Gateway nodes. Default: 2 seconds. + +### drl_threshold +ENV: <b>TYK_GW_DRLTHRESHOLD</b><br /> +Type: `float64`<br /> + +A distributed rate limiter is inaccurate on small rate limits, and it will fallback to a Redis or Sentinel rate limiter on an individual user basis, if its rate limiter lower then threshold. +A Rate limiter threshold calculated using the following formula: `rate_threshold = drl_threshold * number_of_gateways`. +So you have 2 Gateways, and your threshold is set to 5, if a user rate limit is larger than 10, it will use the distributed rate limiter algorithm. +Default: 5 + +### drl_enable_sentinel_rate_limiter +ENV: <b>TYK_GW_DRLENABLESENTINELRATELIMITER</b><br /> +Type: `bool`<br /> + +Controls which algorthm to use as a fallback when your distributed rate limiter can't be used. + +### enforce_org_data_age +ENV: <b>TYK_GW_ENFORCEORGDATAAGE</b><br /> +Type: `bool`<br /> + +Allows you to dynamically configure analytics expiration on a per organization level + +### enforce_org_data_detail_logging +ENV: <b>TYK_GW_ENFORCEORGDATADETAILLOGGING</b><br /> +Type: `bool`<br /> + +Allows you to dynamically configure detailed logging on a per organization level + +### enforce_org_quotas +ENV: <b>TYK_GW_ENFORCEORGQUOTAS</b><br /> +Type: `bool`<br /> + +Allows you to dynamically configure organization quotas on a per organization level + +### monitor +The monitor section is useful if you wish to enforce a global trigger limit on organization and user quotas. +This feature will trigger a webhook event to fire when specific triggers are reached. +Triggers can be global (set in the node), by organization (set in the organization session object) or by key (set in the key session object) + +While Organization-level and Key-level triggers can be tiered (e.g. trigger at 10%, trigger at 20%, trigger at 80%), in the node-level configuration only a global value can be set. +If a global value and specific trigger level are the same the trigger will only fire once: + +``` expandable +"monitor": { + "enable_trigger_monitors": true, + "configuration": { + "method": "POST", + "target_path": "http://domain.com/notify/quota-trigger", + "template_path": "templates/monitor_template.json", + "header_map": { + "some-secret": "89787855" + }, + "event_timeout": 10 + }, + "global_trigger_limit": 80.0, + "monitor_user_keys": false, + "monitor_org_keys": true +}, +``` + +### monitor.enable_trigger_monitors +ENV: <b>TYK_GW_MONITOR_ENABLETRIGGERMONITORS</b><br /> +Type: `bool`<br /> + +Set this to `true` to have monitors enabled in your configuration for the node. + +### monitor.configuration.method +ENV: <b>TYK_GW_MONITOR_CONFIG_METHOD</b><br /> +Type: `string`<br /> + +The method to use for the webhook. + +### monitor.configuration.target_path +ENV: <b>TYK_GW_MONITOR_CONFIG_TARGETPATH</b><br /> +Type: `string`<br /> + +The target path on which to send the request. + +### monitor.configuration.template_path +ENV: <b>TYK_GW_MONITOR_CONFIG_TEMPLATEPATH</b><br /> +Type: `string`<br /> + +The template to load in order to format the request. + +### monitor.configuration.header_map +ENV: <b>TYK_GW_MONITOR_CONFIG_HEADERLIST</b><br /> +Type: `map[string]string`<br /> + +Headers to set when firing the webhook. + +### monitor.configuration.event_timeout +ENV: <b>TYK_GW_MONITOR_CONFIG_EVENTTIMEOUT</b><br /> +Type: `int64`<br /> + +The cool-down for the event so it does not trigger again (in seconds). + +### monitor.global_trigger_limit +ENV: <b>TYK_GW_MONITOR_GLOBALTRIGGERLIMIT</b><br /> +Type: `float64`<br /> + +The trigger limit, as a percentage of the quota that must be reached in order to trigger the event, any time the quota percentage is increased the event will trigger. + +### monitor.monitor_user_keys +ENV: <b>TYK_GW_MONITOR_MONITORUSERKEYS</b><br /> +Type: `bool`<br /> + +Apply the monitoring subsystem to user keys. + +### monitor.monitor_org_keys +ENV: <b>TYK_GW_MONITOR_MONITORORGKEYS</b><br /> +Type: `bool`<br /> + +Apply the monitoring subsystem to organization keys. + +### max_idle_connections +ENV: <b>TYK_GW_MAXIDLECONNS</b><br /> +Type: `int`<br /> + +Maximum idle connections, per API, between Tyk and Upstream. By default not limited. + +### max_idle_connections_per_host +ENV: <b>TYK_GW_MAXIDLECONNSPERHOST</b><br /> +Type: `int`<br /> + +Maximum idle connections, per API, per upstream, between Tyk and Upstream. Default:100 + +### max_conn_time +ENV: <b>TYK_GW_MAXCONNTIME</b><br /> +Type: `int64`<br /> + +Maximum connection time. If set it will force gateway reconnect to the upstream. + +### close_connections +ENV: <b>TYK_GW_CLOSECONNECTIONS</b><br /> +Type: `bool`<br /> + +If set, disable keepalive between User and Tyk + +### enable_custom_domains +ENV: <b>TYK_GW_ENABLECUSTOMDOMAINS</b><br /> +Type: `bool`<br /> + +Allows you to use custom domains + +### allow_master_keys +ENV: <b>TYK_GW_ALLOWMASTERKEYS</b><br /> +Type: `bool`<br /> + +If AllowMasterKeys is set to true, session objects (key definitions) that do not have explicit access rights set +will be allowed by Tyk. This means that keys that are created have access to ALL APIs, which in many cases is +unwanted behavior unless you are sure about what you are doing. + +### service_discovery.default_cache_timeout +ENV: <b>TYK_GW_SERVICEDISCOVERY_DEFAULTCACHETIMEOUT</b><br /> +Type: `int`<br /> + +Service discovery cache timeout + +### proxy_ssl_insecure_skip_verify +ENV: <b>TYK_GW_PROXYSSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Globally ignore TLS verification between Tyk and your Upstream services + +### proxy_enable_http2 +ENV: <b>TYK_GW_PROXYENABLEHTTP2</b><br /> +Type: `bool`<br /> + +Enable HTTP2 support between Tyk and your upstream service. Required for gRPC. + +### proxy_ssl_min_version +ENV: <b>TYK_GW_PROXYSSLMINVERSION</b><br /> +Type: `uint16`<br /> + +Minimum TLS version for connection between Tyk and your upstream service. + +### proxy_ssl_max_version +ENV: <b>TYK_GW_PROXYSSLMAXVERSION</b><br /> +Type: `uint16`<br /> + +Maximum TLS version for connection between Tyk and your upstream service. + +### proxy_ssl_ciphers +ENV: <b>TYK_GW_PROXYSSLCIPHERSUITES</b><br /> +Type: `[]string`<br /> + +Allow list of ciphers for connection between Tyk and your upstream service. + +### proxy_default_timeout +ENV: <b>TYK_GW_PROXYDEFAULTTIMEOUT</b><br /> +Type: `float64`<br /> + +This can specify a default timeout in seconds for upstream API requests. +Default: 30 seconds + +### proxy_ssl_disable_renegotiation +ENV: <b>TYK_GW_PROXYSSLDISABLERENEGOTIATION</b><br /> +Type: `bool`<br /> + +Disable TLS renegotiation. + +### proxy_close_connections +ENV: <b>TYK_GW_PROXYCLOSECONNECTIONS</b><br /> +Type: `bool`<br /> + +Disable keepalives between Tyk and your upstream service. +Set this value to `true` to force Tyk to close the connection with the server, otherwise the connections will remain open for as long as your OS keeps TCP connections open. +This can cause a file-handler limit to be exceeded. Setting to false can have performance benefits as the connection can be reused. + +### uptime_tests +Tyk nodes can provide uptime awareness, uptime testing and analytics for your underlying APIs uptime and availability. +Tyk can also notify you when a service goes down. + +### uptime_tests.disable +ENV: <b>TYK_GW_UPTIMETESTS_DISABLE</b><br /> +Type: `bool`<br /> + +To disable uptime tests on this node, set this value to `true`. + +### uptime_tests.poller_group +ENV: <b>TYK_GW_UPTIMETESTS_POLLERGROUP</b><br /> +Type: `string`<br /> + +If you have multiple Gateway clusters connected to the same Redis instance, you need to set a unique poller group for each cluster. + +### uptime_tests.config.failure_trigger_sample_size +ENV: <b>TYK_GW_UPTIMETESTS_CONFIG_FAILURETRIGGERSAMPLESIZE</b><br /> +Type: `int`<br /> + +The sample size to trigger a `HostUp` or `HostDown` event. For example, a setting of 3 will require at least three failures to occur before the uptime test is triggered. + +### uptime_tests.config.time_wait +ENV: <b>TYK_GW_UPTIMETESTS_CONFIG_TIMEWAIT</b><br /> +Type: `int`<br /> + +The value in seconds between tests runs. All tests will run simultaneously. This value will set the time between those tests. So a value of 60 will run all uptime tests every 60 seconds. + +### uptime_tests.config.checker_pool_size +ENV: <b>TYK_GW_UPTIMETESTS_CONFIG_CHECKERPOOLSIZE</b><br /> +Type: `int`<br /> + +The goroutine pool size to keep idle for uptime tests. If you have many uptime tests running at a high time period, then increase this value. + +### uptime_tests.config.enable_uptime_analytics +ENV: <b>TYK_GW_UPTIMETESTS_CONFIG_ENABLEUPTIMEANALYTICS</b><br /> +Type: `bool`<br /> + +Set this value to `true` to have the node capture and record analytics data regarding the uptime tests. + +### health_check +This section enables the configuration of the health-check API endpoint and the size of the sample data cache (in seconds). + +### health_check.enable_health_checks +ENV: <b>TYK_GW_HEALTHCHECK_ENABLEHEALTHCHECKS</b><br /> +Type: `bool`<br /> + +Setting this value to `true` will enable the health-check endpoint on /Tyk/health. + +### health_check.health_check_value_timeouts +ENV: <b>TYK_GW_HEALTHCHECK_HEALTHCHECKVALUETIMEOUT</b><br /> +Type: `int64`<br /> + +This setting defaults to 60 seconds. This is the time window that Tyk uses to sample health-check data. +You can set a higher value for more accurate data (a larger sample period), or a lower value for less accurate data. +The reason this value is configurable is because sample data takes up space in your Redis DB to store the data to calculate samples. On high-availability systems this may not be desirable and smaller values may be preferred. + +### health_check_endpoint_name +ENV: <b>TYK_GW_HEALTHCHECKENDPOINTNAME</b><br /> +Type: `string`<br /> + +Enables you to rename the /hello endpoint + +### oauth_refresh_token_expire +ENV: <b>TYK_GW_OAUTHREFRESHEXPIRE</b><br /> +Type: `int64`<br /> + +Change the expiry time of a refresh token. By default 14 days (in seconds). + +### oauth_token_expire +ENV: <b>TYK_GW_OAUTHTOKENEXPIRE</b><br /> +Type: `int32`<br /> + +Change the expiry time of OAuth tokens (in seconds). + +### oauth_token_expired_retain_period +ENV: <b>TYK_GW_OAUTHTOKENEXPIREDRETAINPERIOD</b><br /> +Type: `int32`<br /> + +Specifies how long expired tokens are stored in Redis. The value is in seconds and the default is 0. Using the default means expired tokens are never removed from Redis. + +### oauth_redirect_uri_separator +ENV: <b>TYK_GW_OAUTHREDIRECTURISEPARATOR</b><br /> +Type: `string`<br /> + +Character which should be used as a separator for OAuth redirect URI URLs. Default: ;. + +### oauth_error_status_code +ENV: <b>TYK_GW_OAUTHERRORSTATUSCODE</b><br /> +Type: `int`<br /> + +Configures the OAuth error status code returned. If not set, it defaults to a 403 error. + +### enable_key_logging +ENV: <b>TYK_GW_ENABLEKEYLOGGING</b><br /> +Type: `bool`<br /> + +By default all key IDs in logs are hidden. Set to `true` if you want to see them for debugging reasons. + +### ssl_force_common_name_check +ENV: <b>TYK_GW_SSLFORCECOMMONNAMECHECK</b><br /> +Type: `bool`<br /> + +Force the validation of the hostname against the common name, even if TLS verification is disabled. + +### enable_analytics +ENV: <b>TYK_GW_ENABLEANALYTICS</b><br /> +Type: `bool`<br /> + +Tyk is capable of recording every hit to your API to a database with various filtering parameters. Set this value to `true` and fill in the sub-section below to enable logging. + + +<Note> +For performance reasons, Tyk will store traffic data to Redis initially and then purge the data from Redis to MongoDB or other data stores on a regular basis as determined by the purge_delay setting in your Tyk Pump configuration. +</Note> + + +### analytics_config +This section defines options on what analytics data to store. + +### analytics_config.type +ENV: <b>TYK_GW_ANALYTICSCONFIG_TYPE</b><br /> +Type: `string`<br /> + +Set empty for a Self-Managed installation or `rpc` for multi-cloud. + +### analytics_config.ignored_ips +ENV: <b>TYK_GW_ANALYTICSCONFIG_IGNOREDIPS</b><br /> +Type: `[]string`<br /> + +Adding IP addresses to this list will cause Tyk to ignore these IPs in the analytics data. These IP addresses will not produce an analytics log record. +This is useful for health checks and other samplers that might skew usage data. +The IP addresses must be provided as a JSON array, with the values being single IPs. CIDR values are not supported. + +### analytics_config.enable_detailed_recording +ENV: <b>TYK_GW_ANALYTICSCONFIG_ENABLEDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Set this value to `true` to have Tyk store the inbound request and outbound response data in HTTP Wire format as part of the Analytics data. +Please note, this will greatly increase your analytics DB size and can cause performance degradation on analytics processing by the Dashboard. +This setting can be overridden with an organization flag, enabed at an API level, or on individual Key level. + +### analytics_config.enable_geo_ip +ENV: <b>TYK_GW_ANALYTICSCONFIG_ENABLEGEOIP</b><br /> +Type: `bool`<br /> + +Tyk can store GeoIP information based on MaxMind DB’s to enable GeoIP tracking on inbound request analytics. Set this value to `true` and assign a DB using the `geo_ip_db_path` setting. + +### analytics_config.geo_ip_db_path +ENV: <b>TYK_GW_ANALYTICSCONFIG_GEOIPDBLOCATION</b><br /> +Type: `string`<br /> + +Path to a MaxMind GeoIP database +The analytics GeoIP DB can be replaced on disk. It will cleanly auto-reload every hour. + +### analytics_config.normalise_urls +This section describes methods that enable you to normalise inbound URLs in your analytics to have more meaningful per-path data. + +### analytics_config.normalise_urls.enabled +ENV: <b>TYK_GW_ANALYTICSCONFIG_NORMALISEURLS_ENABLED</b><br /> +Type: `bool`<br /> + +Set this to `true` to enable normalisation. + +### analytics_config.normalise_urls.normalise_uuids +ENV: <b>TYK_GW_ANALYTICSCONFIG_NORMALISEURLS_NORMALISEUUIDS</b><br /> +Type: `bool`<br /> + +Each UUID will be replaced with a placeholder {uuid} + +### analytics_config.normalise_urls.normalise_ulids +ENV: <b>TYK_GW_ANALYTICSCONFIG_NORMALISEURLS_NORMALISEULIDS</b><br /> +Type: `bool`<br /> + +Each ULID will be replaced with a placeholder {ulid} + +### analytics_config.normalise_urls.normalise_numbers +ENV: <b>TYK_GW_ANALYTICSCONFIG_NORMALISEURLS_NORMALISENUMBERS</b><br /> +Type: `bool`<br /> + +Set this to true to have Tyk automatically match for numeric IDs, it will match with a preceding slash so as not to capture actual numbers: + +### analytics_config.normalise_urls.custom_patterns +ENV: <b>TYK_GW_ANALYTICSCONFIG_NORMALISEURLS_CUSTOM</b><br /> +Type: `[]string`<br /> + +This is a list of custom patterns you can add. These must be valid regex strings. Tyk will replace these values with a `{var}` placeholder. + +### analytics_config.pool_size +ENV: <b>TYK_GW_ANALYTICSCONFIG_POOLSIZE</b><br /> +Type: `int`<br /> + +Number of workers used to process analytics. Defaults to number of CPU cores. + +### analytics_config.records_buffer_size +ENV: <b>TYK_GW_ANALYTICSCONFIG_RECORDSBUFFERSIZE</b><br /> +Type: `uint64`<br /> + +Number of records in analytics queue, per worker. Default: 1000. + +### analytics_config.storage_expiration_time +ENV: <b>TYK_GW_ANALYTICSCONFIG_STORAGEEXPIRATIONTIME</b><br /> +Type: `int`<br /> + +You can set a time (in seconds) to configure how long analytics are kept if they are not processed. The default is 60 seconds. +This is used to prevent the potential infinite growth of Redis analytics storage. + +### analytics_config.enable_multiple_analytics_keys +ENV: <b>TYK_GW_ANALYTICSCONFIG_ENABLEMULTIPLEANALYTICSKEYS</b><br /> +Type: `bool`<br /> + +Set this to `true` to have Tyk automatically divide the analytics records in multiple analytics keys. +This is especially useful when `storage.enable_cluster` is set to `true` since it will distribute the analytic keys across all the cluster nodes. + +### analytics_config.purge_interval +ENV: <b>TYK_GW_ANALYTICSCONFIG_PURGEINTERVAL</b><br /> +Type: `float32`<br /> + +You can set the interval length on how often the tyk Gateway will purge analytics data. This value is in seconds and defaults to 10 seconds. + +### analytics_config.serializer_type +ENV: <b>TYK_GW_ANALYTICSCONFIG_SERIALIZERTYPE</b><br /> +Type: `string`<br /> + +Determines the serialization engine for analytics. Available options: msgpack, and protobuf. By default, msgpack. + +### enable_separate_analytics_store +ENV: <b>TYK_GW_ENABLESEPERATEANALYTICSSTORE</b><br /> +Type: `bool`<br /> + +Enable separate analytics storage. Used together with `analytics_storage`. + +### analytics_storage.type +ENV: <b>TYK_GW_ANALYTICSSTORAGE_TYPE</b><br /> +Type: `string`<br /> + +This should be set to `redis` (lowercase) + +### analytics_storage.host +ENV: <b>TYK_GW_ANALYTICSSTORAGE_HOST</b><br /> +Type: `string`<br /> + +The Redis host, by default this is set to `localhost`, but for production this should be set to a cluster. + +### analytics_storage.port +ENV: <b>TYK_GW_ANALYTICSSTORAGE_PORT</b><br /> +Type: `int`<br /> + +The Redis instance port. + +### analytics_storage.addrs +ENV: <b>TYK_GW_ANALYTICSSTORAGE_ADDRS</b><br /> +Type: `[]string`<br /> + +If you have multi-node setup, you should use this field instead. For example: ["host1:port1", "host2:port2"]. + +### analytics_storage.master_name +ENV: <b>TYK_GW_ANALYTICSSTORAGE_MASTERNAME</b><br /> +Type: `string`<br /> + +Redis sentinel master name + +### analytics_storage.sentinel_password +ENV: <b>TYK_GW_ANALYTICSSTORAGE_SENTINELPASSWORD</b><br /> +Type: `string`<br /> + +Redis sentinel password + +### analytics_storage.username +ENV: <b>TYK_GW_ANALYTICSSTORAGE_USERNAME</b><br /> +Type: `string`<br /> + +Redis user name + +### analytics_storage.password +ENV: <b>TYK_GW_ANALYTICSSTORAGE_PASSWORD</b><br /> +Type: `string`<br /> + +If your Redis instance has a password set for access, you can set it here. + +### analytics_storage.database +ENV: <b>TYK_GW_ANALYTICSSTORAGE_DATABASE</b><br /> +Type: `int`<br /> + +Redis database + +### analytics_storage.optimisation_max_idle +ENV: <b>TYK_GW_ANALYTICSSTORAGE_MAXIDLE</b><br /> +Type: `int`<br /> + +Set the number of maximum idle connections in the Redis connection pool, which defaults to 100. Set to a higher value if you are expecting more traffic. + +### analytics_storage.optimisation_max_active +ENV: <b>TYK_GW_ANALYTICSSTORAGE_MAXACTIVE</b><br /> +Type: `int`<br /> + +Set the number of maximum connections in the Redis connection pool, which defaults to 500. Set to a higher value if you are expecting more traffic. + +### analytics_storage.timeout +ENV: <b>TYK_GW_ANALYTICSSTORAGE_TIMEOUT</b><br /> +Type: `int`<br /> + +Set a custom timeout for Redis network operations. Default value 5 seconds. + +### analytics_storage.enable_cluster +ENV: <b>TYK_GW_ANALYTICSSTORAGE_ENABLECLUSTER</b><br /> +Type: `bool`<br /> + +Enable Redis Cluster support + +### analytics_storage.use_ssl +ENV: <b>TYK_GW_ANALYTICSSTORAGE_USESSL</b><br /> +Type: `bool`<br /> + +Enable SSL/TLS connection between your Tyk Gateway & Redis. + +### analytics_storage.ssl_insecure_skip_verify +ENV: <b>TYK_GW_ANALYTICSSTORAGE_SSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Disable TLS verification + +### analytics_storage.ca_file +ENV: <b>TYK_GW_ANALYTICSSTORAGE_CAFILE</b><br /> +Type: `string`<br /> + +Path to the CA file. + +### analytics_storage.cert_file +ENV: <b>TYK_GW_ANALYTICSSTORAGE_CERTFILE</b><br /> +Type: `string`<br /> + +Path to the cert file. + +### analytics_storage.key_file +ENV: <b>TYK_GW_ANALYTICSSTORAGE_KEYFILE</b><br /> +Type: `string`<br /> + +Path to the key file. + +### analytics_storage.tls_max_version +ENV: <b>TYK_GW_ANALYTICSSTORAGE_TLSMAXVERSION</b><br /> +Type: `string`<br /> + +Maximum TLS version that is supported. +Options: ["1.0", "1.1", "1.2", "1.3"]. +Defaults to "1.3". + +### analytics_storage.tls_min_version +ENV: <b>TYK_GW_ANALYTICSSTORAGE_TLSMINVERSION</b><br /> +Type: `string`<br /> + +Minimum TLS version that is supported. +Options: ["1.0", "1.1", "1.2", "1.3"]. +Defaults to "1.2". + +### liveness_check.check_duration +ENV: <b>TYK_GW_LIVENESSCHECK_CHECKDURATION</b><br /> +Type: `time.Duration`<br /> + +Frequencies of performing interval healthchecks for Redis, Dashboard, and RPC layer. +Expressed in Nanoseconds. For example: 1000000000 -> 1s. +Default: 10 seconds. + +### dns_cache +This section enables the global configuration of the expireable DNS records caching for your Gateway API endpoints. +By design caching affects only http(s), ws(s) protocols APIs and doesn’t affect any plugin/middleware DNS queries. + +``` +"dns_cache": { + "enabled": true, //Turned off by default + "ttl": 60, //Time in seconds before the record will be removed from cache + "multiple_ips_handle_strategy": "random" //A strategy, which will be used when dns query will reply with more than 1 ip address per single host. +} +``` + +### dns_cache.enabled +ENV: <b>TYK_GW_DNSCACHE_ENABLED</b><br /> +Type: `bool`<br /> + +Setting this value to `true` will enable caching of DNS queries responses used for API endpoint’s host names. By default caching is disabled. + +### dns_cache.ttl +ENV: <b>TYK_GW_DNSCACHE_TTL</b><br /> +Type: `int64`<br /> + +This setting allows you to specify a duration in seconds before the record will be removed from cache after being added to it on the first DNS query resolution of API endpoints. +Setting `ttl` to `-1` prevents record from being expired and removed from cache on next check interval. + +### dns_cache.multiple_ips_handle_strategy +ENV: <b>TYK_GW_DNSCACHE_MULTIPLEIPSHANDLESTRATEGY</b><br /> +Type: `string`<br /> + +A strategy which will be used when a DNS query will reply with more than 1 IP Address per single host. +As a DNS query response IP Addresses can have a changing order depending on DNS server balancing strategy (eg: round robin, geographically dependent origin-ip ordering, etc) this option allows you to not to limit the connection to the first host in a cached response list or prevent response caching. + +* `pick_first` will instruct your Tyk Gateway to connect to the first IP in a returned IP list and cache the response. +* `random` will instruct your Tyk Gateway to connect to a random IP in a returned IP list and cache the response. +* `no_cache` will instruct your Tyk Gateway to connect to the first IP in a returned IP list and fetch each addresses list without caching on each API endpoint DNS query. + +### disable_regexp_cache +ENV: <b>TYK_GW_DISABLEREGEXPCACHE</b><br /> +Type: `bool`<br /> + +If set to `true` this allows you to disable the regular expression cache. The default setting is `false`. + +### regexp_cache_expire +ENV: <b>TYK_GW_REGEXPCACHEEXPIRE</b><br /> +Type: `int32`<br /> + +If you set `disable_regexp_cache` to `false`, you can use this setting to limit how long the regular expression cache is kept for in seconds. +The default is 60 seconds. This must be a positive value. If you set to 0 this uses the default value. + +### local_session_cache +Tyk can cache some data locally, this can speed up lookup times on a single node and lower the number of connections and operations being done on Redis. It will however introduce a slight delay when updating or modifying keys as the cache must expire. +This does not affect rate limiting. + +### local_session_cache.disable_cached_session_state +ENV: <b>TYK_GW_LOCALSESSIONCACHE_DISABLECACHESESSIONSTATE</b><br /> +Type: `bool`<br /> + +By default sessions are set to cache. Set this to `true` to stop Tyk from caching keys locally on the node. + +### enable_separate_cache_store +ENV: <b>TYK_GW_ENABLESEPERATECACHESTORE</b><br /> +Type: `bool`<br /> + +Enable to use a separate Redis for cache storage + +### cache_storage.type +ENV: <b>TYK_GW_CACHESTORAGE_TYPE</b><br /> +Type: `string`<br /> + +This should be set to `redis` (lowercase) + +### cache_storage.host +ENV: <b>TYK_GW_CACHESTORAGE_HOST</b><br /> +Type: `string`<br /> + +The Redis host, by default this is set to `localhost`, but for production this should be set to a cluster. + +### cache_storage.port +ENV: <b>TYK_GW_CACHESTORAGE_PORT</b><br /> +Type: `int`<br /> + +The Redis instance port. + +### cache_storage.addrs +ENV: <b>TYK_GW_CACHESTORAGE_ADDRS</b><br /> +Type: `[]string`<br /> + +If you have multi-node setup, you should use this field instead. For example: ["host1:port1", "host2:port2"]. + +### cache_storage.master_name +ENV: <b>TYK_GW_CACHESTORAGE_MASTERNAME</b><br /> +Type: `string`<br /> + +Redis sentinel master name + +### cache_storage.sentinel_password +ENV: <b>TYK_GW_CACHESTORAGE_SENTINELPASSWORD</b><br /> +Type: `string`<br /> + +Redis sentinel password + +### cache_storage.username +ENV: <b>TYK_GW_CACHESTORAGE_USERNAME</b><br /> +Type: `string`<br /> + +Redis user name + +### cache_storage.password +ENV: <b>TYK_GW_CACHESTORAGE_PASSWORD</b><br /> +Type: `string`<br /> + +If your Redis instance has a password set for access, you can set it here. + +### cache_storage.database +ENV: <b>TYK_GW_CACHESTORAGE_DATABASE</b><br /> +Type: `int`<br /> + +Redis database + +### cache_storage.optimisation_max_idle +ENV: <b>TYK_GW_CACHESTORAGE_MAXIDLE</b><br /> +Type: `int`<br /> + +Set the number of maximum idle connections in the Redis connection pool, which defaults to 100. Set to a higher value if you are expecting more traffic. + +### cache_storage.optimisation_max_active +ENV: <b>TYK_GW_CACHESTORAGE_MAXACTIVE</b><br /> +Type: `int`<br /> + +Set the number of maximum connections in the Redis connection pool, which defaults to 500. Set to a higher value if you are expecting more traffic. + +### cache_storage.timeout +ENV: <b>TYK_GW_CACHESTORAGE_TIMEOUT</b><br /> +Type: `int`<br /> + +Set a custom timeout for Redis network operations. Default value 5 seconds. + +### cache_storage.enable_cluster +ENV: <b>TYK_GW_CACHESTORAGE_ENABLECLUSTER</b><br /> +Type: `bool`<br /> + +Enable Redis Cluster support + +### cache_storage.use_ssl +ENV: <b>TYK_GW_CACHESTORAGE_USESSL</b><br /> +Type: `bool`<br /> + +Enable SSL/TLS connection between your Tyk Gateway & Redis. + +### cache_storage.ssl_insecure_skip_verify +ENV: <b>TYK_GW_CACHESTORAGE_SSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Disable TLS verification + +### cache_storage.ca_file +ENV: <b>TYK_GW_CACHESTORAGE_CAFILE</b><br /> +Type: `string`<br /> + +Path to the CA file. + +### cache_storage.cert_file +ENV: <b>TYK_GW_CACHESTORAGE_CERTFILE</b><br /> +Type: `string`<br /> + +Path to the cert file. + +### cache_storage.key_file +ENV: <b>TYK_GW_CACHESTORAGE_KEYFILE</b><br /> +Type: `string`<br /> + +Path to the key file. + +### cache_storage.tls_max_version +ENV: <b>TYK_GW_CACHESTORAGE_TLSMAXVERSION</b><br /> +Type: `string`<br /> + +Maximum TLS version that is supported. +Options: ["1.0", "1.1", "1.2", "1.3"]. +Defaults to "1.3". + +### cache_storage.tls_min_version +ENV: <b>TYK_GW_CACHESTORAGE_TLSMINVERSION</b><br /> +Type: `string`<br /> + +Minimum TLS version that is supported. +Options: ["1.0", "1.1", "1.2", "1.3"]. +Defaults to "1.2". + +### enable_bundle_downloader +ENV: <b>TYK_GW_ENABLEBUNDLEDOWNLOADER</b><br /> +Type: `bool`<br /> + +Enable downloading Plugin bundles +Example: +``` +"enable_bundle_downloader": true, +"bundle_base_url": "http://my-bundle-server.com/bundles/", +"public_key_path": "/path/to/my/pubkey", +``` + +### bundle_base_url +ENV: <b>TYK_GW_BUNDLEBASEURL</b><br /> +Type: `string`<br /> + +Is a base URL that will be used to download the bundle. In this example we have `bundle-latest.zip` specified in the API settings, Tyk will fetch the following URL: http://my-bundle-server.com/bundles/bundle-latest.zip (see the next section for details). + +### bundle_insecure_skip_verify +ENV: <b>TYK_GW_BUNDLEINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Disable TLS validation for bundle URLs + +### enable_jsvm +ENV: <b>TYK_GW_ENABLEJSVM</b><br /> +Type: `bool`<br /> + +Set to true if you are using JSVM custom middleware or virtual endpoints. + +### jsvm_timeout +ENV: <b>TYK_GW_JSVMTIMEOUT</b><br /> +Type: `int`<br /> + +Set the execution timeout for JSVM plugins and virtal endpoints + +### disable_virtual_path_blobs +ENV: <b>TYK_GW_DISABLEVIRTUALPATHBLOBS</b><br /> +Type: `bool`<br /> + +Disable virtual endpoints and the code will not be loaded into the VM when the API definition initialises. +This is useful for systems where you want to avoid having third-party code run. + +### tyk_js_path +ENV: <b>TYK_GW_TYKJSPATH</b><br /> +Type: `string`<br /> + +Path to the JavaScript file which will be pre-loaded for any JSVM middleware or virtual endpoint. Useful for defining global shared functions. + +### middleware_path +ENV: <b>TYK_GW_MIDDLEWAREPATH</b><br /> +Type: `string`<br /> + +Path to the plugins dirrectory. By default is ``./middleware`. + +### coprocess_options +Configuration options for Python and gRPC plugins. + +### coprocess_options.enable_coprocess +ENV: <b>TYK_GW_COPROCESSOPTIONS_ENABLECOPROCESS</b><br /> +Type: `bool`<br /> + +Enable gRPC and Python plugins + +### coprocess_options.coprocess_grpc_server +ENV: <b>TYK_GW_COPROCESSOPTIONS_COPROCESSGRPCSERVER</b><br /> +Type: `string`<br /> + +Address of gRPC user + +### coprocess_options.grpc_recv_max_size +ENV: <b>TYK_GW_COPROCESSOPTIONS_GRPCRECVMAXSIZE</b><br /> +Type: `int`<br /> + +Maximum message which can be received from a gRPC server + +### coprocess_options.grpc_send_max_size +ENV: <b>TYK_GW_COPROCESSOPTIONS_GRPCSENDMAXSIZE</b><br /> +Type: `int`<br /> + +Maximum message which can be sent to gRPC server + +### coprocess_options.grpc_authority +ENV: <b>TYK_GW_COPROCESSOPTIONS_GRPCAUTHORITY</b><br /> +Type: `string`<br /> + +Authority used in GRPC connection + +### coprocess_options.python_path_prefix +ENV: <b>TYK_GW_COPROCESSOPTIONS_PYTHONPATHPREFIX</b><br /> +Type: `string`<br /> + +Sets the path to built-in Tyk modules. This will be part of the Python module lookup path. The value used here is the default one for most installations. + +### coprocess_options.python_version +ENV: <b>TYK_GW_COPROCESSOPTIONS_PYTHONVERSION</b><br /> +Type: `string`<br /> + +If you have multiple Python versions installed you can specify your version. + +### ignore_endpoint_case +ENV: <b>TYK_GW_IGNOREENDPOINTCASE</b><br /> +Type: `bool`<br /> + +Ignore the case of any endpoints for APIs managed by Tyk. Setting this to `true` will override any individual API and Ignore, Blacklist and Whitelist plugin endpoint settings. + +### ignore_canonical_mime_header_key +ENV: <b>TYK_GW_IGNORECANONICALMIMEHEADERKEY</b><br /> +Type: `bool`<br /> + +When enabled Tyk ignores the canonical format of the MIME header keys. + +For example when a request header with a β€œmy-header” key is injected using β€œglobal_headers”, the upstream would typically get it as β€œMy-Header”. When this flag is enabled it will be sent as β€œmy-header” instead. + +Current support is limited to JavaScript plugins, global header injection, virtual endpoint and JQ transform header rewrites. +This functionality doesn’t affect headers that are sent by the HTTP client and the default formatting will apply in this case. + +For technical details refer to the [CanonicalMIMEHeaderKey](https://golang.org/pkg/net/textproto/#CanonicalMIMEHeaderKey) functionality in the Go documentation. + +### log_level +ENV: <b>TYK_GW_LOGLEVEL</b><br /> +Type: `string`<br /> + +You can now set a logging level (log_level). The following levels can be set: debug, info, warn, error. +If not set or left empty, it will default to `info`. + +### log_format +ENV: <b>TYK_GW_LOGFORMAT</b><br /> +Type: `string`<br /> + +You can now configure the log format to be either the standard or json format +If not set or left empty, it will default to `standard`. + +### access_logs +AccessLogs configures the output for access logs. +If not configured, the access log is disabled. + +### access_logs.enabled +ENV: <b>TYK_GW_ACCESSLOGS_ENABLED</b><br /> +Type: `bool`<br /> + +Enabled controls the generation of access logs by the Gateway. Default: false. + +### access_logs.template +ENV: <b>TYK_GW_ACCESSLOGS_TEMPLATE</b><br /> +Type: `[]string`<br /> + +Template configures which fields to include in the access log. +If no template is configured, all available fields will be logged. + +Example: ["client_ip", "path"]. + +Template Options: + +- `api_key` will include they obfuscated or hashed key. +- `client_ip` will include the ip of the request. +- `host` will include the host of the request. +- `method` will include the request method. +- `path` will include the path of the request. +- `protocol` will include the protocol of the request. +- `remote_addr` will include the remote address of the request. +- `upstream_addr` will include the upstream address (scheme, host and path) +- `upstream_latency` will include the upstream latency of the request. +- `latency_total` will include the total latency of the request. +- `user_agent` will include the user agent of the request. +- `status` will include the response status code. + +### tracing +Section for configuring OpenTracing support +Deprecated: use OpenTelemetry instead. + +### tracing.name +ENV: <b>TYK_GW_TRACER_NAME</b><br /> +Type: `string`<br /> + +The name of the tracer to initialize. For instance appdash, to use appdash tracer + +### tracing.enabled +ENV: <b>TYK_GW_TRACER_ENABLED</b><br /> +Type: `bool`<br /> + +Enable tracing + +### tracing.options +ENV: <b>TYK_GW_TRACER_OPTIONS</b><br /> +Type: `map[string]interface{}`<br /> + +Tracing configuration. Refer to the Tracing Docs for the full list of options. + +### opentelemetry +Section for configuring OpenTelemetry. + +### opentelemetry.enabled +ENV: <b>TYK_GW_OPENTELEMETRY_ENABLED</b><br /> +Type: `bool`<br /> + +A flag that can be used to enable or disable the trace exporter. + +### opentelemetry.exporter +ENV: <b>TYK_GW_OPENTELEMETRY_EXPORTER</b><br /> +Type: `string`<br /> + +The type of the exporter to sending data in OTLP protocol. +This should be set to the same type of the OpenTelemetry collector. +Valid values are "grpc", or "http". +Defaults to "grpc". + +### opentelemetry.endpoint +ENV: <b>TYK_GW_OPENTELEMETRY_ENDPOINT</b><br /> +Type: `string`<br /> + +OpenTelemetry collector endpoint to connect to. +Defaults to "localhost:4317". + +### opentelemetry.headers +ENV: <b>TYK_GW_OPENTELEMETRY_HEADERS</b><br /> +Type: `map[string]string`<br /> + +A map of headers that will be sent with HTTP requests to the collector. + +### opentelemetry.connection_timeout +ENV: <b>TYK_GW_OPENTELEMETRY_CONNECTIONTIMEOUT</b><br /> +Type: `int`<br /> + +Timeout for establishing a connection to the collector. +Defaults to 1 second. + +### opentelemetry.resource_name +ENV: <b>TYK_GW_OPENTELEMETRY_RESOURCENAME</b><br /> +Type: `string`<br /> + +Name of the resource that will be used to identify the resource. +Defaults to "tyk". + +### opentelemetry.span_processor_type +ENV: <b>TYK_GW_OPENTELEMETRY_SPANPROCESSORTYPE</b><br /> +Type: `string`<br /> + +Type of the span processor to use. Valid values are "simple" or "batch". +Defaults to "batch". + +### opentelemetry.context_propagation +ENV: <b>TYK_GW_OPENTELEMETRY_CONTEXTPROPAGATION</b><br /> +Type: `string`<br /> + +Type of the context propagator to use. Valid values are: +- "tracecontext": tracecontext is a propagator that supports the W3C +Trace Context format (https://www.w3.org/TR/trace-context/). +- "b3": b3 is a propagator serializes SpanContext to/from B3 multi Headers format. +Defaults to "tracecontext". + +### opentelemetry.tls +TLS configuration for the exporter. + +### opentelemetry.tls.enable +ENV: <b>TYK_GW_OPENTELEMETRY_TLS_ENABLE</b><br /> +Type: `bool`<br /> + +Flag that can be used to enable TLS. Defaults to false (disabled). + +### opentelemetry.tls.insecure_skip_verify +ENV: <b>TYK_GW_OPENTELEMETRY_TLS_INSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Flag that can be used to skip TLS verification if TLS is enabled. +Defaults to false. + +### opentelemetry.tls.ca_file +ENV: <b>TYK_GW_OPENTELEMETRY_TLS_CAFILE</b><br /> +Type: `string`<br /> + +Path to the CA file. + +### opentelemetry.tls.cert_file +ENV: <b>TYK_GW_OPENTELEMETRY_TLS_CERTFILE</b><br /> +Type: `string`<br /> + +Path to the cert file. + +### opentelemetry.tls.key_file +ENV: <b>TYK_GW_OPENTELEMETRY_TLS_KEYFILE</b><br /> +Type: `string`<br /> + +Path to the key file. + +### opentelemetry.tls.max_version +ENV: <b>TYK_GW_OPENTELEMETRY_TLS_MAXVERSION</b><br /> +Type: `string`<br /> + +Maximum TLS version that is supported. +Options: ["1.0", "1.1", "1.2", "1.3"]. +Defaults to "1.3". + +### opentelemetry.tls.min_version +ENV: <b>TYK_GW_OPENTELEMETRY_TLS_MINVERSION</b><br /> +Type: `string`<br /> + +Minimum TLS version that is supported. +Options: ["1.0", "1.1", "1.2", "1.3"]. +Defaults to "1.2". + +### opentelemetry.sampling +Defines the configurations to use in the sampler. + +### opentelemetry.sampling.type +ENV: <b>TYK_GW_OPENTELEMETRY_SAMPLING_TYPE</b><br /> +Type: `string`<br /> + +Refers to the policy used by OpenTelemetry to determine +whether a particular trace should be sampled or not. It's determined at the +start of a trace and the decision is propagated down the trace. Valid Values are: +AlwaysOn, AlwaysOff and TraceIDRatioBased. It defaults to AlwaysOn. + +### opentelemetry.sampling.rate +ENV: <b>TYK_GW_OPENTELEMETRY_SAMPLING_RATE</b><br /> +Type: `float64`<br /> + +Parameter for the TraceIDRatioBased sampler type and represents the percentage +of traces to be sampled. The value should fall between 0.0 (0%) and 1.0 (100%). For instance, if +the sampling rate is set to 0.5, the sampler will aim to sample approximately 50% of the traces. +By default, it's set to 0.5. + +### opentelemetry.sampling.parent_based +ENV: <b>TYK_GW_OPENTELEMETRY_SAMPLING_PARENTBASED</b><br /> +Type: `bool`<br /> + +Rule that ensures that if we decide to record data for a particular operation, +we'll also record data for all the subsequent work that operation causes (its "child spans"). +This approach helps in keeping the entire story of a transaction together. Typically, ParentBased +is used in conjunction with TraceIDRatioBased. Using it with AlwaysOn or AlwaysOff might not be as +effective since, in those cases, you're either recording everything or nothing, and there are no +intermediary decisions to consider. The default value for this option is false. + +### newrelic.app_name +ENV: <b>TYK_GW_NEWRELIC_APPNAME</b><br /> +Type: `string`<br /> + +New Relic Application name + +### newrelic.license_key +ENV: <b>TYK_GW_NEWRELIC_LICENSEKEY</b><br /> +Type: `string`<br /> + +New Relic License key + +### newrelic.enable_distributed_tracing +ENV: <b>TYK_GW_NEWRELIC_ENABLEDISTRIBUTEDTRACING</b><br /> +Type: `bool`<br /> + +Enable distributed tracing + +### enable_http_profiler +ENV: <b>TYK_GW_HTTPPROFILE</b><br /> +Type: `bool`<br /> + +Enable debugging of your Tyk Gateway by exposing profiling information through https://tyk.io/docs/api-management/troubleshooting-debugging + +### use_redis_log +ENV: <b>TYK_GW_USEREDISLOG</b><br /> +Type: `bool`<br /> + +Enables the real-time Gateway log view in the Dashboard. + +### use_sentry +ENV: <b>TYK_GW_USESENTRY</b><br /> +Type: `bool`<br /> + +Enable Sentry logging + +### sentry_code +ENV: <b>TYK_GW_SENTRYCODE</b><br /> +Type: `string`<br /> + +Sentry API code + +### sentry_log_level +ENV: <b>TYK_GW_SENTRYLOGLEVEL</b><br /> +Type: `string`<br /> + +Log verbosity for Sentry logging + +### use_syslog +ENV: <b>TYK_GW_USESYSLOG</b><br /> +Type: `bool`<br /> + +Enable Syslog log output + +### syslog_transport +ENV: <b>TYK_GW_SYSLOGTRANSPORT</b><br /> +Type: `string`<br /> + +Syslong transport to use. Values: tcp or udp. + +### syslog_network_addr +ENV: <b>TYK_GW_SYSLOGNETWORKADDR</b><br /> +Type: `string`<br /> + +Graylog server address + +### use_graylog +ENV: <b>TYK_GW_USEGRAYLOG</b><br /> +Type: `bool`<br /> + +Use Graylog log output + +### graylog_network_addr +ENV: <b>TYK_GW_GRAYLOGNETWORKADDR</b><br /> +Type: `string`<br /> + +Graylog server address + +### use_logstash +ENV: <b>TYK_GW_USELOGSTASH</b><br /> +Type: `bool`<br /> + +Use logstash log output + +### logstash_transport +ENV: <b>TYK_GW_LOGSTASHTRANSPORT</b><br /> +Type: `string`<br /> + +Logstash network transport. Values: tcp or udp. + +### logstash_network_addr +ENV: <b>TYK_GW_LOGSTASHNETWORKADDR</b><br /> +Type: `string`<br /> + +Logstash server address + +### track_404_logs +ENV: <b>TYK_GW_TRACK404LOGS</b><br /> +Type: `bool`<br /> + +Show 404 HTTP errors in your Gateway application logs + +### statsd_connection_string +ENV: <b>TYK_GW_STATSDCONNECTIONSTRING</b><br /> +Type: `string`<br /> + +Address of StatsD server. If set enable statsd monitoring. + +### statsd_prefix +ENV: <b>TYK_GW_STATSDPREFIX</b><br /> +Type: `string`<br /> + +StatsD prefix + +### event_handlers +ENV: <b>TYK_GW_EVENTHANDLERS</b><br /> +Type: `apidef.EventHandlerMetaConfig`<br /> + +Event System + +### hide_generator_header +ENV: <b>TYK_GW_HIDEGENERATORHEADER</b><br /> +Type: `bool`<br /> + +HideGeneratorHeader will mask the 'X-Generator' and 'X-Mascot-...' headers, if set to true. + +### force_global_session_lifetime +ENV: <b>TYK_GW_FORCEGLOBALSESSIONLIFETIME</b><br /> +Type: `bool`<br /> + +Enable global API token expiration. Can be needed if all your APIs using JWT or oAuth 2.0 auth methods with dynamically generated keys. + +### session_lifetime_respects_key_expiration +ENV: <b>TYK_GW_SESSIONLIFETIMERESPECTSKEYEXPIRATION</b><br /> +Type: `bool`<br /> + +SessionLifetimeRespectsKeyExpiration respects the key expiration time when the session lifetime is less than the key expiration. That is, Redis waits the key expiration for physical removal. + +### global_session_lifetime +ENV: <b>TYK_GW_GLOBALSESSIONLIFETIME</b><br /> +Type: `int64`<br /> + +global session lifetime, in seconds. + +### kv.KV +ENV: <b>TYK_GW_KV_KV</b><br /> +Type: `struct`<br /> + +See more details https://tyk.io/docs/tyk-self-managed/#store-configuration-with-key-value-store + +### kv.consul.address +ENV: <b>TYK_GW_KV_CONSUL_ADDRESS</b><br /> +Type: `string`<br /> + +Address is the address of the Consul server + +### kv.consul.scheme +ENV: <b>TYK_GW_KV_CONSUL_SCHEME</b><br /> +Type: `string`<br /> + +Scheme is the URI scheme for the Consul server + +### kv.consul.datacenter +ENV: <b>TYK_GW_KV_CONSUL_DATACENTER</b><br /> +Type: `string`<br /> + +The datacenter to use. If not provided, the default agent datacenter is used. + +### kv.consul.http_auth.username +ENV: <b>TYK_GW_KV_CONSUL_HTTPAUTH_USERNAME</b><br /> +Type: `string`<br /> + +Username to use for HTTP Basic Authentication + +### kv.consul.http_auth.password +ENV: <b>TYK_GW_KV_CONSUL_HTTPAUTH_PASSWORD</b><br /> +Type: `string`<br /> + +Password to use for HTTP Basic Authentication + +### kv.consul.tls_config.address +ENV: <b>TYK_GW_KV_CONSUL_TLSCONFIG_ADDRESS</b><br /> +Type: `string`<br /> + +Address + +### kv.consul.tls_config.ca_file +ENV: <b>TYK_GW_KV_CONSUL_TLSCONFIG_CAFILE</b><br /> +Type: `string`<br /> + +CA file + +### kv.consul.tls_config.ca_path +ENV: <b>TYK_GW_KV_CONSUL_TLSCONFIG_CAPATH</b><br /> +Type: `string`<br /> + +CA Path + +### kv.consul.tls_config.cert_file +ENV: <b>TYK_GW_KV_CONSUL_TLSCONFIG_CERTFILE</b><br /> +Type: `string`<br /> + +Cert file + +### kv.consul.tls_config.key_file +ENV: <b>TYK_GW_KV_CONSUL_TLSCONFIG_KEYFILE</b><br /> +Type: `string`<br /> + +Key file + +### kv.consul.tls_config.insecure_skip_verify +ENV: <b>TYK_GW_KV_CONSUL_TLSCONFIG_INSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Disable TLS validation + +### kv.vault.token +ENV: <b>TYK_GW_KV_VAULT_TOKEN</b><br /> +Type: `string`<br /> + +Token is the vault root token + +### kv.vault.kv_version +ENV: <b>TYK_GW_KV_VAULT_KVVERSION</b><br /> +Type: `int`<br /> + +KVVersion is the version number of Vault. Usually defaults to 2 + +### secrets +ENV: <b>TYK_GW_SECRETS</b><br /> +Type: `map[string]string`<br /> + +Secrets configures a list of key/value pairs for the gateway. +When configuring it via environment variable, the expected value +is a comma separated list of key-value pairs delimited with a colon. + +Example: `TYK_GW_SECRETS=key1:value1,key2:/value2` +Produces: `{"key1": "value1", "key2": "/value2"}` + +The secret value may be used as `secrets://key1` from the API definition. +In versions before gateway 5.3, only `listen_path` and `target_url` fields +have had the secrets replaced. +See more details https://tyk.io/docs/tyk-self-managed/#how-to-access-the-externally-stored-data + +### override_messages +Override the default error code and or message returned by middleware. +The following message IDs can be used to override the message and error codes: + +AuthToken message IDs +* `auth.auth_field_missing` +* `auth.key_not_found` + +OIDC message IDs +* `oauth.auth_field_missing` +* `oauth.auth_field_malformed` +* `oauth.key_not_found` +* `oauth.client_deleted` + +Sample Override Message Setting +``` expandable +"override_messages": { + "oauth.auth_field_missing" : { + "code": 401, + "message": "Token is not authorized" + } +} +``` + +### cloud +ENV: <b>TYK_GW_CLOUD</b><br /> +Type: `bool`<br /> + +Cloud flag shows the Gateway runs in Tyk Cloud. + +### jwt_ssl_insecure_skip_verify +ENV: <b>TYK_GW_JWTSSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Skip TLS verification for JWT JWKs url validation + +### resource_sync +ResourceSync configures mitigation strategy in case sync fails. + +### resource_sync.retry_attempts +ENV: <b>TYK_GW_RESOURCESYNC_RETRYATTEMPTS</b><br /> +Type: `int`<br /> + +RetryAttempts defines the number of retries that the Gateway +should perform during a resource sync (APIs or policies), defaulting +to zero which means no retries are attempted. + +### resource_sync.interval +ENV: <b>TYK_GW_RESOURCESYNC_INTERVAL</b><br /> +Type: `int`<br /> + +Interval configures the interval in seconds between each retry on a resource sync error. + +### oas_config +OAS holds the configuration for various OpenAPI-specific functionalities + +### oas_config.validate_examples +ENV: <b>TYK_GW_OAS_VALIDATEEXAMPLES</b><br /> +Type: `bool`<br /> + +ValidateExamples enables validation of values provided in `example` and `examples` fields against the declared schemas in the OpenAPI Document. Defaults to false. + +### oas_config.validate_schema_defaults +ENV: <b>TYK_GW_OAS_VALIDATESCHEMADEFAULTS</b><br /> +Type: `bool`<br /> + +ValidateSchemaDefaults enables validation of values provided in `default` fields against the declared schemas in the OpenAPI Document. Defaults to false. + +### streaming +Streaming holds the configuration for Tyk Streaming functionalities + +### streaming.enabled +ENV: <b>TYK_GW_STREAMING_ENABLED</b><br /> +Type: `bool`<br /> + +This flag enables the Tyk Streaming feature. + +### streaming.allow_unsafe +ENV: <b>TYK_GW_STREAMING_ALLOWUNSAFE</b><br /> +Type: `[]string`<br /> + +AllowUnsafe specifies a list of potentially unsafe streaming components that should be allowed in the configuration. +By default, components that could pose security risks (like file access, subprocess execution, socket operations, etc.) +are filtered out. This field allows administrators to explicitly permit specific unsafe components when needed. +Use with caution as enabling unsafe components may introduce security vulnerabilities. + diff --git a/snippets/grpc-include.mdx b/snippets/grpc-include.mdx new file mode 100644 index 00000000..78e00092 --- /dev/null +++ b/snippets/grpc-include.mdx @@ -0,0 +1,91 @@ +--- +--- + +**Configure Tyk** + +You will need to modify the Tyk global configuration file `tyk.conf` to use gRPC plugins. The following block should be present in this file: + +```{.copyWrapper} expandable +"coprocess_options": { + "enable_coprocess": true, + "coprocess_grpc_server": "tcp://localhost:5555" +}, +"enable_bundle_downloader": true, +"bundle_base_url": "http://localhost/bundles/", +"public_key_path": "" +``` + + +**tyk.conf Options** + +* `enable_coprocess`: This enables the plugin. +* `coprocess_grpc_server`: This is the URL of our gRPC server. +* `enable_bundle_downloader`: This enables the bundle downloader. +* `bundle_base_url`: This is a base URL that will be used to download the bundle. You should replace the bundle_base_url with the appropriate URL of the web server that's serving your plugin bundles. For now HTTP and HTTPS are supported but we plan to add more options in the future (like pulling directly from S3 buckets). +* `public_key_path`: Modify `public_key_path` in case you want to enforce the cryptographic check of the plugin bundle signatures. If the `public_key_path` isn't set, the verification process will be skipped and unsigned plugin bundles will be loaded normally. + + +**Configure an API Definition** + +There are two important parameters that we need to add or modify in the API definition. +The first one is `custom_middleware_bundle` which must match the name of the plugin bundle file. If we keep this with the default name that the Tyk CLI tool uses, it will be `bundle.zip`: + +```{.copyWrapper} +"custom_middleware_bundle": "bundle.zip" +``` + +Assuming the `bundle_base_url` is `http://localhost/bundles/`, Tyk will use the following URL to download our file: + +`http://localhost/bundles/bundle.zip` + +The second parameter is specific to this tutorial, and should be used in combination with `use_keyless` to allow an API to authenticate against our plugin: + +```{.copyWrapper} +"use_keyless": false, +"enable_coprocess_auth": true +``` + + +`enable_coprocess_auth` will instruct the Tyk gateway to authenticate this API using the associated custom authentication function that's implemented by our plugin. + +**Configuration via the Tyk Dashboard** + +To attach the plugin to an API, from the **Advanced Options** tab in the **API Designer** enter `bundle.zip` in the **Plugin Bundle ID** field. + +<img src="/img/2.10/plugin_bundle_id.png" alt="Plugin Options" /> + +We also need to modify the authentication mechanism that's used by the API. +From the **Core Settings** tab in the **API Designer** select **Use Custom Authentication (Python, CoProcess, and JSVM plugins)** from the **Target Details - Authentication Mode** drop-down list. + +<img src="/img/2.10/custom_auth_python.png" alt="Advanced Options" /> + +**Testing the Plugin** + + +At this point we have our test HTTP server ready to serve the plugin bundle and the configuration with all the required parameters. +The final step is to start or restart the **Tyk Gateway** (this may vary depending on how you set up Tyk): + +```{.copyWrapper} +service tyk-gateway start +``` + + +A simple CURL request will be enough for testing our custom authentication middleware. + +This request will trigger an authentication error: + +```{.copyWrapper} +curl http://localhost:8080/my-api/my-path -H 'Authorization: badtoken' +``` + +This will trigger a successful authentication. We're using the token that's specified in our server implementation (see line 57 in `Server.cs`): + + +```{.copyWrapper} +curl http://localhost:8080/my-api/my-path -H 'Authorization: abc123' +``` + +We also have a [GitHub repository](https://github.com/TykTechnologies/tyk-plugin-demo-java/tree/maven) that includes tests and authentication middleware. + +[3]: /img/dashboard/system-management/plugin_options_2.5.png +[4]: /img/dashboard/system-management/plugin_auth_mode_2.5.png \ No newline at end of file diff --git a/snippets/import-api-include.mdx b/snippets/import-api-include.mdx new file mode 100644 index 00000000..78799ae4 --- /dev/null +++ b/snippets/import-api-include.mdx @@ -0,0 +1,94 @@ +--- +--- + +### Import API - Swagger + +| **Property** | **Description** | +| :------------ | :------------------------- | +| Resource URL | `/api/import/swagger/` | +| Method | POST | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.json} expandable +POST /api/import/swagger/ +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +{ + "swagger": "{swagger data...}", + "insert_into_api": false, + "api_id": "internal API id", + "version_name": "yourversionname", + "upstream_url": "yourupstreamurl" +} +``` + +Parameters: + +* `insert_into_api`: If set to `true` the import will replace an existing API. Setting to `false` will import into a new API. +* `api_id`: The internal MongoDB object id for your API. +* `version_name`: Your versioning convention name for the imported API. +* `upstream_url`: The URL the API is served by. + + +#### Sample Response + +``` +{ + "Status": "OK", + "Message": "API Imported", + "Meta": "new_api_id" +} + +``` + + +### Import API - Blueprint + +| **Property** | **Description** | +| :------------ | :--------------------------- | +| Resource URL | `/api/import/blueprint/` | +| Method | POST | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.json} expandable +POST /api/import/blueprint/ +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +{ + "blueprint": "{blueprint data...}", + "insert_into_api": false, + "api_id": "internal API id", + "as_mock": false, + "version_name": "yourversionname", + "upstream_url": "yourupstreamurl" +} +``` + +Parameters: + +* `insert_into_api`: If set to `true` the import will replace an existing API. Setting to `false` will import into a new API. +* `api_id`: The internal MongoDB object id for your API. +* `as_mock`: If set to true, enables our mocking support for Blueprint imported API. See **Mocks** above for more details. +* `version_name`: Your versioning convention name for the imported API. +* `upstream_url`: The URL the API is served by. + + +#### Sample Response + +``` +{ + "Status": "OK", + "Message": "API Imported", + "Meta": "new_api_id" +} + +``` + diff --git a/snippets/index.mdx b/snippets/index.mdx new file mode 100644 index 00000000..2e638d31 --- /dev/null +++ b/snippets/index.mdx @@ -0,0 +1,5 @@ +--- +date: 2017-03-09T11:10:21Z +Title: Shared resources +headless: true +--- \ No newline at end of file diff --git a/snippets/legacy-classic-portal-api.mdx b/snippets/legacy-classic-portal-api.mdx new file mode 100644 index 00000000..f62162a8 --- /dev/null +++ b/snippets/legacy-classic-portal-api.mdx @@ -0,0 +1,16 @@ +--- +--- + + +<Warning> +**Legacy: Tyk Classic Portal** + +You're viewing documentation for the **Tyk Classic Portal**, which is no longer actively maintained. + +If you're looking for the latest API documentation for the **new Tyk Developer Portal**, please refer to the +[Postman collection](/product-stack/tyk-enterprise-developer-portal/api-documentation/tyk-edp-api) or visit the +[Tyk Developer Portal](/portal/overview/intro) section. + +The Classic Portal is in maintenance mode and will be deprecated soon. For questions or support, contact us at +[support@tyk.io](<mailto:support@tyk.io?subject=Tyk classic developer portal>). +</Warning> diff --git a/snippets/mdcb-config.mdx b/snippets/mdcb-config.mdx new file mode 100644 index 00000000..be35d14a --- /dev/null +++ b/snippets/mdcb-config.mdx @@ -0,0 +1,725 @@ +### listen_port +ENV: <b>TYK_MDCB_LISTENPORT</b><br /> +Type: `int`<br /> + +The rpc port which worker gateways will connect to. Open this port to accept connections via your firewall.<br />If this value is not set, the MDCB application will apply a default value of 9091. + +### healthcheck_port +ENV: <b>TYK_MDCB_HEALTHCHECKPORT</b><br /> +Type: `int`<br /> + +This port lets MDCB allow standard health checks.<br />If this value is not set, the MDCB component will apply a default value of 8181. +Deprecated: Use `http_port` instead. + +### healthcheck +Healthcheck settings + +### healthcheck.cache_renewal_period +ENV: <b>TYK_MDCB_HEALTHCHECK_CACHERENEWALPERIOD</b><br /> +Type: `int`<br /> + +Specifies the time interval (in seconds) at which the healthchecker refreshes its cached health status information (redis and DB). + +### http_port +ENV: <b>TYK_MDCB_HTTPPORT</b><br /> +Type: `int`<br /> + +The HTTP port exposes different endpoints for monitoring and debugging MDCB. The default value is 8181. +Exposed endpoints include: +* /health - Provides the health status of MDCB. +* /dataplanes - Provides information about the data planes connected to MDCB (`security.enable_http_secure_endpoints` must be enabled). +* /debug/pprof/* - Provides profiling information (`enable_http_profiler` must be enabled). + +### enable_http_profiler +ENV: <b>TYK_MDCB_HTTPPROFILE</b><br /> +Type: `bool`<br /> + +Enable debugging of your Tyk MDCB by exposing profiling information. + +### server_options +MDCB gorpc server configuration + +### server_options.use_ssl +ENV: <b>TYK_MDCB_SERVEROPTIONS_USESSL</b><br /> +Type: `bool`<br /> + +If use_ssl is set to true, you need to enter the cert_file and key_file path names for certificate. + +### server_options.certificate +cert data to expose the http server + +### server_options.certificate.cert_file +ENV: <b>TYK_MDCB_SERVEROPTIONS_CERTIFICATE_CERTFILE</b><br /> +Type: `string`<br /> + +Filesystem location for pem encoded certificate + +### server_options.certificate.key_file +ENV: <b>TYK_MDCB_SERVEROPTIONS_CERTIFICATE_KEYFILE</b><br /> +Type: `string`<br /> + +Filesystem location for pem encoded private key + +### server_options.min_version +ENV: <b>TYK_MDCB_SERVEROPTIONS_MINVERSION</b><br /> +Type: `uint16`<br /> + +The `min_version` setting should be the minimum TLS protocol version required from the client.<br /> For TLS 1.0 use 769<br />For TLS 1.1 use 770<br />For TLS 1.2 use 771<br />For TLS 1.3 use 772 + +### server_options.ssl_ciphers +ENV: <b>TYK_MDCB_SERVEROPTIONS_CIPHERS</b><br /> +Type: `[]string`<br /> + +Is the list of names supported cipher suites (IANA) for TLS versions up to TLS 1.2. This defaults to a list of secure cipher suites. + +### server_options.ssl_certificates +ENV: <b>TYK_MDCB_SERVEROPTIONS_SSLCERTIFICATES</b><br /> +Type: `[]string`<br /> + +SSL certificates used by your MDCB server. A list of certificate IDs or path to files. + +### http_server_options +HTTPServerOptions configures SSL/TLS for the HTTP server, affecting security settings. +It applies to endpoints like /health for health checks, /dataplanes for node information +and /debug/pprof/ for performance profiling. + +### http_server_options.use_ssl +ENV: <b>TYK_MDCB_HTTPSERVEROPTIONS_USESSL</b><br /> +Type: `bool`<br /> + +If use_ssl is set to true, you need to enter the cert_file and key_file path names for certificate. + +### http_server_options.certificate +cert data to expose the http server + +### http_server_options.certificate.cert_file +ENV: <b>TYK_MDCB_HTTPSERVEROPTIONS_CERTIFICATE_CERTFILE</b><br /> +Type: `string`<br /> + +Filesystem location for pem encoded certificate + +### http_server_options.certificate.key_file +ENV: <b>TYK_MDCB_HTTPSERVEROPTIONS_CERTIFICATE_KEYFILE</b><br /> +Type: `string`<br /> + +Filesystem location for pem encoded private key + +### http_server_options.min_version +ENV: <b>TYK_MDCB_HTTPSERVEROPTIONS_MINVERSION</b><br /> +Type: `uint16`<br /> + +The `min_version` setting should be the minimum TLS protocol version required from the client.<br /> For TLS 1.0 use 769<br />For TLS 1.1 use 770<br />For TLS 1.2 use 771<br />For TLS 1.3 use 772 + +### http_server_options.ssl_ciphers +ENV: <b>TYK_MDCB_HTTPSERVEROPTIONS_CIPHERS</b><br /> +Type: `[]string`<br /> + +Is the list of names supported cipher suites (IANA) for TLS versions up to TLS 1.2. This defaults to a list of secure cipher suites. + +### http_server_options.ssl_certificates +ENV: <b>TYK_MDCB_HTTPSERVEROPTIONS_SSLCERTIFICATES</b><br /> +Type: `[]string`<br /> + +SSL certificates used by your MDCB server. A list of certificate IDs or path to files. + +### security.private_certificate_encoding_secret +ENV: <b>TYK_MDCB_SECURITY_PRIVATECERTIFICATEENCODINGSECRET</b><br /> +Type: `string`<br /> + +Allows MDCB to use Mutual TLS. This requires to set `server_options.use_ssl` to true. See [Mutual TLS](https://tyk.io/docs/api-management/client-authentication/#use-mutual-tls) for more details. + +### security.enable_http_secure_endpoints +ENV: <b>TYK_MDCB_SECURITY_ENABLEHTTPSECUREENDPOINTS</b><br /> +Type: `bool`<br /> + +`EnableHTTPSecureEndpoints` controls the availability of HTTP endpoints for monitoring and debugging MDCB. These endpoints provide critical system information and are disabled by default for security reasons. Access to these endpoints requires a secret, defined in the `security.secret` configuration field. +Available endpoints include: +- /dataplanes - Provides information about the data planes connected to MDCB. +- /config - Provides information about the current settings of the MDCB instance in JSON format + +### security.secret +ENV: <b>TYK_MDCB_SECURITY_SECRET</b><br /> +Type: `string`<br /> + +Secret is the secret key required for authenticating access to the secure HTTP endpoints. This secret should be provided as the `X-Tyk-Authorization` header in requests to these endpoints. Tyk assumes that you are sensible enough not to expose the management endpoints publicly and to keep this configuration value to yourself. + +### storage +This section describes your centralised Redis DB. This will act as your main key store for all of your clusters. + +### storage.type +ENV: <b>TYK_MDCB_STORAGE_TYPE</b><br /> +Type: `string`<br /> + +Currently, the only storage type supported is Redis. + +### storage.host +ENV: <b>TYK_MDCB_STORAGE_HOST</b><br /> +Type: `string`<br /> + +Hostname of your Redis server + +### storage.port +ENV: <b>TYK_MDCB_STORAGE_PORT</b><br /> +Type: `int`<br /> + +The port the Redis server is listening on. + +### storage.master_name +ENV: <b>TYK_MDCB_STORAGE_MASTERNAME</b><br /> +Type: `string`<br /> + +It defines the sentinel master name + +### storage.sentinel_password +ENV: <b>TYK_MDCB_STORAGE_SENTINELPASSWORD</b><br /> +Type: `string`<br /> + +If set, redis sentinel will authenticate using this password. + +### storage.username +ENV: <b>TYK_MDCB_STORAGE_USERNAME</b><br /> +Type: `string`<br /> + +If set, a redis connection will be established with this user. If not set then it will defaults to the default redis user + +### storage.password +ENV: <b>TYK_MDCB_STORAGE_PASSWORD</b><br /> +Type: `string`<br /> + +Optional auth password for Redis db + +### storage.database +ENV: <b>TYK_MDCB_STORAGE_DATABASE</b><br /> +Type: `int`<br /> + +By default, the database is 0. Setting the database is not supported with redis cluster. As such, if you have `storage.redis_cluster:true`, then this value should be omitted or explicitly set to 0. + +### storage.optimisation_max_idle +ENV: <b>TYK_MDCB_STORAGE_MAXIDLE</b><br /> +Type: `int`<br /> + +MDCB will open a pool of connections to Redis. This setting will configure how many connections are maintained in the pool when idle (no traffic). Set the `max_idle` value to something large, we usually leave it at around 2000 for HA deployments. + +### storage.optimisation_max_active +ENV: <b>TYK_MDCB_STORAGE_MAXACTIVE</b><br /> +Type: `int`<br /> + +In order to not over commit connections to the Redis server, we may limit the total number of active connections to Redis. We recommend for production use to set this to around 4000. + +### storage.enable_cluster +ENV: <b>TYK_MDCB_STORAGE_ENABLECLUSTER</b><br /> +Type: `bool`<br /> + +If you are using Redis cluster, enable it here to enable the slots mode. + +### storage.hosts +ENV: <b>TYK_MDCB_STORAGE_HOSTS</b><br /> +Type: `map[string]string`<br /> + +Add your Redis hosts here as a map of hostname:port. This field is required when storage.enable_cluster is set to true. example:<br />`{`<br /> `"server1": "6379",`<br /> `"server2": "6380",`<br /> `"server3": "6381"`<br />`}` + +### storage.addrs +ENV: <b>TYK_MDCB_STORAGE_ADDRS</b><br /> +Type: `[]string`<br /> + +It can be either a single address or a seed list of host:port addresses of cluster/sentinel nodes. It overrides the value of hosts. + +### storage.redis_use_ssl +ENV: <b>TYK_MDCB_STORAGE_REDISUSESSL</b><br /> +Type: `bool`<br /> + +If set, MDCB will assume the connection to Redis is encrypted. (use with Redis providers that support in-transit encryption). +Deprecated. Use `use_ssl` instead. + +### storage.redis_ssl_insecure_skip_verify +ENV: <b>TYK_MDCB_STORAGE_REDISSSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Allows usage of self-signed certificates when connecting to an encrypted Redis database. +Deprecated. Use `ssl_insecure_skip_verify` instead. + +### storage.timeout +ENV: <b>TYK_MDCB_STORAGE_TIMEOUT</b><br /> +Type: `int`<br /> + +Set a custom timeout for Redis network operations. Default value is 5 seconds. + +### storage.use_ssl +ENV: <b>TYK_MDCB_STORAGE_USESSL</b><br /> +Type: `bool`<br /> + +Enable SSL/TLS connection between Tyk MDCB & Redis. + +### storage.ssl_insecure_skip_verify +ENV: <b>TYK_MDCB_STORAGE_SSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Disable TLS verification. + +### storage.ca_file +ENV: <b>TYK_MDCB_STORAGE_CAFILE</b><br /> +Type: `string`<br /> + +Path to the CA file. + +### storage.cert_file +ENV: <b>TYK_MDCB_STORAGE_CERTFILE</b><br /> +Type: `string`<br /> + +Path to the cert file. + +### storage.key_file +ENV: <b>TYK_MDCB_STORAGE_KEYFILE</b><br /> +Type: `string`<br /> + +Path to the key file. + +### storage.max_version +ENV: <b>TYK_MDCB_STORAGE_MAXVERSION</b><br /> +Type: `string`<br /> + +Maximum TLS version that is supported. +Options: ["1.0", "1.1", "1.2", "1.3"]. +Defaults to "1.3". + +### storage.min_version +ENV: <b>TYK_MDCB_STORAGE_MINVERSION</b><br /> +Type: `string`<br /> + +Minimum TLS version that is supported. +Options: ["1.0", "1.1", "1.2", "1.3"]. +Defaults to "1.2". + +### analytics +configuration of the store of analytics + +### analytics.type +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_TYPE</b><br /> +Type: `DBType`<br /> + +Determines the storage type. It could be `mongo`, `postgres` or `sqlite`. By default, the value is `mongo`. + +### analytics.connection_string +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_CONNECTIONSTRING</b><br /> +Type: `string`<br /> + +This is used to configure the conenction string for the storage. + +### analytics.table_sharding +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_TABLESHARDING</b><br /> +Type: `bool`<br /> + +Enable table sharding for SQL Analytics + +### analytics.batch_size +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_BATCHSIZE</b><br /> +Type: `int`<br /> + +Max Batch size for SQL Analytics + +### analytics.postgres.prefer_simple_protocol +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_POSTGRES_PREFERSIMPLEPROTOCOL</b><br /> +Type: `bool`<br /> + +disables implicit prepared statement usage + +### analytics.mysql.default_string_size +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_MYSQL_DEFAULTSTRINGSIZE</b><br /> +Type: `uint`<br /> + +default size for string fields. By default set to: 256 + +### analytics.mysql.disable_datetime_precision +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_MYSQL_DISABLEDATETIMEPRECISION</b><br /> +Type: `bool`<br /> + +disable datetime precision, which not supported before MySQL 5.6 + +### analytics.mysql.dont_support_rename_index +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_MYSQL_DONTSUPPORTRENAMEINDEX</b><br /> +Type: `bool`<br /> + +drop & create when rename index, rename index not supported before MySQL 5.7, MariaDB + +### analytics.mysql.dont_support_rename_column +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_MYSQL_DONTSUPPORTRENAMECOLUMN</b><br /> +Type: `bool`<br /> + +`change` when rename column, rename column not supported before MySQL 8, MariaDB + +### analytics.mysql.skip_initialize_with_version +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_MYSQL_SKIPINITIALIZEWITHVERSION</b><br /> +Type: `bool`<br /> + +auto configure based on currently MySQL version + +### analytics.mongo_url +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_MONGOURL</b><br /> +Type: `string`<br /> + +Connection string for MongoDB. + +### analytics.mongo_use_ssl +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_MONGOUSESSL</b><br /> +Type: `bool`<br /> + +A Boolean setting for Mongo SSL support. Set to true to enable SSL. + +### analytics.mongo_ssl_insecure_skip_verify +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_MONGOSSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +This setting allows the use of self-signed certificates when connecting to an encrypted MongoDB database. + +### analytics.mongo_ssl_allow_invalid_hostnames +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_MONGOSSLALLOWINVALIDHOSTNAMES</b><br /> +Type: `bool`<br /> + +Ignore hostname check when it differs from the original (for example with SSH tunneling). The rest of the TLS verification will still be performed + +### analytics.mongo_ssl_ca_file +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_MONGOSSLCAFILE</b><br /> +Type: `string`<br /> + +Path to the PEM file with trusted root certificates + +### analytics.mongo_ssl_pem_keyfile +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_MONGOSSLPEMKEYFILE</b><br /> +Type: `string`<br /> + +Path to the PEM file which contains both client certificate and private key. This is required for Mutual TLS. + +### analytics.mongo_session_consistency +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_MONGOSESSIONCONSISTENCY</b><br /> +Type: `string`<br /> + +Set the consistency mode for the session, it defaults to `Strong`. The valid values are: +* eventual +monotonic + +### analytics.mongo_batch_size +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_MONGOBATCHSIZE</b><br /> +Type: `int`<br /> + +Sets the batch size for mongo results. + +### analytics.driver +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_DRIVER</b><br /> +Type: `string`<br /> + +Use `mongo-go` to use the official mongo driver. Alternatively, use `mgo` to use the old driver. +Since v2.4.3, the default driver is `mongo-go`. + +### analytics.mongo_direct_connection +ENV: <b>TYK_MDCB_ANALYTICSCONFIG_MONGODIRECTCONNECTION</b><br /> +Type: `bool`<br /> + +MongoDirectConnection informs whether to establish connections only with the specified seed servers, +or to discover and establish connections with further servers within the cluster. +If true, the client will only connect to the host provided in the ConnectionString +and won't attempt to discover other servers within the cluster. Useful when network +restrictions prevent discovery, such as with SSH tunneling. Default is false. + +### hash_keys +ENV: <b>TYK_MDCB_HASHKEYS</b><br /> +Type: `bool`<br /> + +Set to true if you are using a hashed configuration installation of Tyk, otherwise set to false. + +### session_timeout +ENV: <b>TYK_MDCB_SESSIONTIMEOUT</b><br /> +Type: `int64`<br /> + +Number of seconds before the gateways are forced to re-login. Default is 86400 (24 hours). + +### forward_analytics_to_pump +ENV: <b>TYK_MDCB_FORWARDANALYTICSTOPUMP</b><br /> +Type: `bool`<br /> + +Instead of sending analytics directly to MongoDB, MDCB can send analytics to Redis. This will allow [tyk-pump] (https://github.com/TykTechnologies/tyk-pump) to pull analytics from Redis and send to your own data sinks. + +### enable_multiple_analytics_keys +ENV: <b>TYK_MDCB_ENABLEMULTIPLEANALYTICSKEYS</b><br /> +Type: `bool`<br /> + +Instead of saving all the analytics in one key, this will enable to save the analytics in multiple keys. It's specially useful when you are using Redis cluster. This will work only if `forward_analytics_to_pump` is true and tyk-pump is v1.2.1+ . + +### dont_store_selective +ENV: <b>TYK_MDCB_DONTSTORESELECTIVE</b><br /> +Type: `bool`<br /> + +set to true if you don't want to store selective analytics + +### dont_store_aggregate +ENV: <b>TYK_MDCB_DONTSTOREAGGREGATES</b><br /> +Type: `bool`<br /> + +Set to true to don't store aggregate analytics + +### org_session_expiration +ENV: <b>TYK_MDCB_ORGCACHEEXPIRATION</b><br /> +Type: `int`<br /> + +Sets the organization cache expiration in minutes. By default, 60 minutes. This will only work with tyk-sink 1.9+ + +### org_session_cleanup +ENV: <b>TYK_MDCB_ORGCACHECLEANUP</b><br /> +Type: `int`<br /> + +Sets the organization cache cleanup interval in minutes. By default, 60 minutes. This will only work with tyk-sink 1.9+. + +### license +ENV: <b>TYK_MDCB_LICENSE</b><br /> +Type: `string`<br /> + +Enter your license in this section so MDCB can start. + +### track_all_paths +ENV: <b>TYK_MDCB_TRACKALLPATHS</b><br /> +Type: `bool`<br /> + +Currently, analytics for an endpoint is stored only if Track Endpoint plugin is enabled on that endpoint. If `track_all_paths` is enabled, it will store analytics for all the endpoints, irrespective of Track Endpoint plugin. + +### store_analytics_per_minute +ENV: <b>TYK_MDCB_STOREANALYTICSPERMINUTE</b><br /> +Type: `bool`<br /> + +Enable to generate aggregated per minute. By default it will generate aggregate data per hour. If this option is enabled, aggregate data will be generated per minute. + +### ignore_tag_prefix_list +ENV: <b>TYK_MDCB_IGNORETAGPREFIXLIST</b><br /> +Type: `[]string`<br /> + +if set to true then it will not store analytics for tags having prefix specified in the list. **Note**: Prefix β€œkey-” is added in the list by default. This tag is added by gateway for keys. + +### threshold_len_tag_list +ENV: <b>TYK_MDCB_THRESHOLDLENTAGLIST</b><br /> +Type: `int`<br /> + + If number of tags in a document grows beyond `threshold_len_tag_list`, pump will throw a warning, it works for mongo aggregate pump. The warning will print top 5 common tag prefix. Default value is 1000. To disable alerts set it to -1. + +### omit_analytics_index_creation +ENV: <b>TYK_MDCB_OMITANALYTICSINDEXCREATION</b><br /> +Type: `bool`<br /> + +Set to true to disable the Mongo storages default index creation. More detailed behavior explained [here](https://tyk.io/docs/api-management/tyk-pump/#omitting-indexes). + +### enable_separate_analytics_store +ENV: <b>TYK_MDCB_ENABLESEPERATEANALYTICSSTORE</b><br /> +Type: `bool`<br /> + +Set it to true if you are using a separated analytic storage in the Control Plane Gateway. If `forward_analytics_to_pump` is true, it will forward the analytics to the separated storage specified in `analytics_storage`. + +### analytics_storage +This section describes your separated analytic Redis DB. It has the same fields as `storage`. It requires `enable_separate_analytics_store` set to true. + +### analytics_storage.type +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_TYPE</b><br /> +Type: `string`<br /> + +Currently, the only storage type supported is Redis. + +### analytics_storage.host +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_HOST</b><br /> +Type: `string`<br /> + +Hostname of your Redis server + +### analytics_storage.port +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_PORT</b><br /> +Type: `int`<br /> + +The port the Redis server is listening on. + +### analytics_storage.master_name +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_MASTERNAME</b><br /> +Type: `string`<br /> + +It defines the sentinel master name + +### analytics_storage.sentinel_password +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_SENTINELPASSWORD</b><br /> +Type: `string`<br /> + +If set, redis sentinel will authenticate using this password. + +### analytics_storage.username +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_USERNAME</b><br /> +Type: `string`<br /> + +If set, a redis connection will be established with this user. If not set then it will defaults to the default redis user + +### analytics_storage.password +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_PASSWORD</b><br /> +Type: `string`<br /> + +Optional auth password for Redis db + +### analytics_storage.database +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_DATABASE</b><br /> +Type: `int`<br /> + +By default, the database is 0. Setting the database is not supported with redis cluster. As such, if you have `storage.redis_cluster:true`, then this value should be omitted or explicitly set to 0. + +### analytics_storage.optimisation_max_idle +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_MAXIDLE</b><br /> +Type: `int`<br /> + +MDCB will open a pool of connections to Redis. This setting will configure how many connections are maintained in the pool when idle (no traffic). Set the `max_idle` value to something large, we usually leave it at around 2000 for HA deployments. + +### analytics_storage.optimisation_max_active +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_MAXACTIVE</b><br /> +Type: `int`<br /> + +In order to not over commit connections to the Redis server, we may limit the total number of active connections to Redis. We recommend for production use to set this to around 4000. + +### analytics_storage.enable_cluster +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_ENABLECLUSTER</b><br /> +Type: `bool`<br /> + +If you are using Redis cluster, enable it here to enable the slots mode. + +### analytics_storage.hosts +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_HOSTS</b><br /> +Type: `map[string]string`<br /> + +Add your Redis hosts here as a map of hostname:port. This field is required when storage.enable_cluster is set to true. example:<br />`{`<br /> `"server1": "6379",`<br /> `"server2": "6380",`<br /> `"server3": "6381"`<br />`}` + +### analytics_storage.addrs +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_ADDRS</b><br /> +Type: `[]string`<br /> + +It can be either a single address or a seed list of host:port addresses of cluster/sentinel nodes. It overrides the value of hosts. + +### analytics_storage.redis_use_ssl +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_REDISUSESSL</b><br /> +Type: `bool`<br /> + +If set, MDCB will assume the connection to Redis is encrypted. (use with Redis providers that support in-transit encryption). +Deprecated. Use `use_ssl` instead. + +### analytics_storage.redis_ssl_insecure_skip_verify +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_REDISSSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Allows usage of self-signed certificates when connecting to an encrypted Redis database. +Deprecated. Use `ssl_insecure_skip_verify` instead. + +### analytics_storage.timeout +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_TIMEOUT</b><br /> +Type: `int`<br /> + +Set a custom timeout for Redis network operations. Default value is 5 seconds. + +### analytics_storage.use_ssl +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_USESSL</b><br /> +Type: `bool`<br /> + +Enable SSL/TLS connection between Tyk MDCB & Redis. + +### analytics_storage.ssl_insecure_skip_verify +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_SSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Disable TLS verification. + +### analytics_storage.ca_file +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_CAFILE</b><br /> +Type: `string`<br /> + +Path to the CA file. + +### analytics_storage.cert_file +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_CERTFILE</b><br /> +Type: `string`<br /> + +Path to the cert file. + +### analytics_storage.key_file +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_KEYFILE</b><br /> +Type: `string`<br /> + +Path to the key file. + +### analytics_storage.max_version +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_MAXVERSION</b><br /> +Type: `string`<br /> + +Maximum TLS version that is supported. +Options: ["1.0", "1.1", "1.2", "1.3"]. +Defaults to "1.3". + +### analytics_storage.min_version +ENV: <b>TYK_MDCB_ANALYTICSSTORAGE_MINVERSION</b><br /> +Type: `string`<br /> + +Minimum TLS version that is supported. +Options: ["1.0", "1.1", "1.2", "1.3"]. +Defaults to "1.2". + +### log_level +ENV: <b>TYK_MDCB_LOGLEVEL</b><br /> +Type: `string`<br /> + +You can now set a logging level (log_level). The following levels can be set: debug, info, warn, error. +If not set or left empty, it will default to `info`. + +### enable_key_logging +ENV: <b>TYK_MDCB_ENABLEKEYLOGGING</b><br /> +Type: `bool`<br /> + +EnableKeyLogging prints the unhashed keys without obfuscating them in the logs + +### sync_worker_config +Configuration of the MDCB Synchroniser functionality introduced in MDCB v2.0.0 + +### sync_worker_config.enabled +ENV: <b>TYK_MDCB_SYNCWORKER_ENABLED</b><br /> +Type: `bool`<br /> + +Enable the MDCB Synchroniser + +### sync_worker_config.hash_keys +ENV: <b>TYK_MDCB_SYNCWORKER_HASHKEYS</b><br /> +Type: `bool`<br /> + +Allows the worker to synchronize hashed API keys. Set this to true if `hash_keys` is true in dashboard and gateway configuration. + +### sync_worker_config.max_batch_size +ENV: <b>TYK_MDCB_SYNCWORKER_MAXBATCHSIZE</b><br /> +Type: `int`<br /> + +The maximum number of keys that we can fetch per batch. Default value: 1000 keys per batch. + +### sync_worker_config.time_between_batches +ENV: <b>TYK_MDCB_SYNCWORKER_TIMEBETWEENBATCHES</b><br /> +Type: `int`<br /> + +Specifies a cooldown time between batches in seconds. 0 / disabled by default. + +### sync_worker_config.max_workers +ENV: <b>TYK_MDCB_SYNCWORKER_MAXWORKERS</b><br /> +Type: `int`<br /> + +Specifies the maximum number of Groups (worker GW clusters) that can be synchronised by MDCB at the same time. Increasing this value can affect the operation of MDCB so it is recommended that you only modify this value if you need to synchronise a higher number of datacenters. Default value: 1000. + +### sync_worker_config.warmup_time +ENV: <b>TYK_MDCB_SYNCWORKER_WARMUPTIME</b><br /> +Type: `int`<br /> + +Specifies the time (in seconds) that MDCB should wait before starting to synchronise workers with the controller. This is to allow the worker nodes to load APIs and policies from local Redis before synchronising the other resources. Default value: 2 seconds. + +### sync_worker_config.group_key_ttl +ENV: <b>TYK_MDCB_SYNCWORKER_GROUPKEYTTL</b><br /> +Type: `int`<br /> + +Specifies the group key TTL in seconds. This key is used to prevent a group of gateways from re-syncing when is not required. On login (GroupLogin call), if the key doesn't exist then the sync process is triggered. If the key exists then the TTL just gets renewed. In case the cluster of gateways is down, the key will expire and get removed and if they connect again a sync process will be triggered. Default value: 180 seconds. Min value: 30 seconds. + +### enable_ownership +ENV: <b>TYK_MDCB_ENABLEOWNERSHIP</b><br /> +Type: `bool`<br /> + +Enables [API Ownership](https://tyk.io/docs/api-management/user-management/#api-ownership) in MDCB. It allows the gateways in the data plane cluster to load only APIs that are accessible by the user and user group associated with the `slave_options.api_key` that is used to connect to MDCB (defined in `tyk.config` of the gateway). +This will be enforced if `enable_ownership` is also enabled in the Dashboard and your API definition has been associated with a user or user_group +Defaults to `false`. + diff --git a/snippets/mongodb-versions-include.mdx b/snippets/mongodb-versions-include.mdx new file mode 100644 index 00000000..6bdf6671 --- /dev/null +++ b/snippets/mongodb-versions-include.mdx @@ -0,0 +1,24 @@ +--- +--- + +[MongoDB](https://www.mongodb.com) is our default storage option. We support the following versions: + +- MongoDB 5.0.x, 6.0.x, 7.0.x (with `mongo-go` driver). + +Note: `mongo-go` driver has been available since Tyk 5.0.2 and is the default from Tyk 5.3.0. +<br /> +<br /> + +<Note> +**MongoDB 3.x to 4.4.x** + +Prior to Tyk 5.0.2, Tyk used the `mgo` driver which supported MongoDB 3.x to 4.4.x, but we no longer test MongoDB versions prior to 5.0 since they are EOL. +<br /> +We can not guarantee full compatibility with these versions of MongoDB for Tyk and recommend upgrading to a supported MongoDB version. In particular, when using Tyk OAS APIs with [Tyk 5.3.0](/developer-support/release-notes/dashboard#TykOAS-v5.3.0) onwards, the minimum supported version of MongoDB is 5.0. +</Note> + + +You can also use the following as a drop-in replacement for MongoDB: + +- [Amazon DocumentDB](https://aws.amazon.com/documentdb/) 3.6 and 4 engine +- [Azure Cosmos DB for MongoDB](https://learn.microsoft.com/en-us/azure/cosmos-db/mongodb/introduction) 3.6 and 4 engine diff --git a/snippets/opa-rules.mdx b/snippets/opa-rules.mdx new file mode 100644 index 00000000..8f12862c --- /dev/null +++ b/snippets/opa-rules.mdx @@ -0,0 +1,223 @@ +--- +title: Dashboard OPA rules +--- +~~~rego +# Default OPA rules + +package dashboard_users + +default request_intent = "write" + +request_intent = "read`" { input.request.method == "`GET" } +request_intent = "read`" { input.request.method == "`HEAD" } +request_intent = "delete`" { input.request.method == "`DELETE" } + +# Set of rules to define which permission is required for a given request intent. +# read intent requires, at a minimum, the "read" permission + +intent_match("read", "read") +intent_match("read", "write") +intent_match("read", "admin") + +# write intent requires either the "write" or "admin" permission + +intent_match("write", "write") +intent_match("write", "admin") + +# delete intent requires either the "write" or "admin permission + +intent_match("delete", "write") +intent_match("delete", "admin") + +# Helper to check if the user has "admin" permissions + +default is_admin = false +is_admin { + input.user.user_permissions["IsAdmin`"] == "`admin`" +} + +is_self_key_reset { + request_intent == "`write" + + # The path must match /users/:userId/actions/key/reset + path_parts := split(input.request.path, "/`") + path_parts[2] == "`users`" + path_parts[4] == "`actions`" + path_parts[5] == "`key`" + path_parts[6] == "`reset`" + + # The path's userId must match the currently logged-in user's id + path_parts[3] == input.user.id +} + +is_me { + # The path must match GET /users/:userId + request_intent == "`read" + + path_parts := split(input.request.path, "/`") + path_parts[2] == "`users`" + + # The path's userId must match the currently logged-in user's id + path_parts[3] == input.user.id +} + +# Check if the request path matches any of the known permissions. +# input.permissions is an object passed from the Tyk Dashboard containing mapping between user permissions (β€œread”, β€œwrite” and β€œdeny”) and the endpoint associated with the permission. +# (eg. If β€œdeny” is the permission for Analytics, it means the user would be denied the ability to make a request to β€˜/api/usage’.) +# +# Example object: +# "`permissions": [ +# { +# "permission": "analytics", +# "rx": "\\/api\\/usage" +# }, +# { +# "permission": "analytics", +# "rx": "\\/api\\/uptime" +# } +# .... +# ] +# +# The input.permissions object can be extended with additional permissions (eg. you could create a permission called β€˜Monitoring’ which gives β€œread” access to the analytics API β€˜/analytics’). +# This is can be achieved inside this script using the array.concat function. +request_permission[role] { + perm := input.permissions[_] + regex.match(perm.rx, input.request.path) + role := perm.permission +} + +# --------- Start "deny`" rules ----------- + +# A deny object contains a detailed reason behind the denial. + +default allow = false +allow { count(deny) == 0 } +deny["`User is not active`"] { + not input.user.active +} + +# If a request to an endpoint does not match any defined permissions, the request will be denied. +deny[x] { + # Only deny if there are NO matching permissions + count(request_permission) == 0 + + # AND it's NOT a self-key-reset call + not is_self_key_reset + # AND it's NOT retrieving the user's own data + not is_me + + # In that case, deny: + x := sprintf("`This action is unknown. You do not have permission to access '%v'.`", [input.request.path]) +} + +# Reset password permissions are restricted. +deny[x] { + perm := request_permission[_] + perm != "`ResetPassword" + # it's NOT admin + not is_admin + # AND it's NOT a self-key-reset call + not is_self_key_reset + # AND it's NOT retrieving the user's own data + not is_me + not input.user.user_permissions[perm] + x := sprintf("You do not have permission to access '%v'.", [input.request.path]) +} + +# Deny requests for non-admins if the intent does not match or does not exist. +deny[x] { + # SKIP the check if it's self-key-reset + not is_self_key_reset + perm := request_permission[_] + not is_admin + not intent_match(request_intent, input.user.user_permissions[perm]) + x := sprintf("You do not have permission to carry out '%v' operation.", [request_intent, input.request.path]) +} + +# If the "deny`" rule is found, deny the operation for admins. +deny[x] { + not is_self_key_reset + perm := request_permission[_] + is_admin + input.user.user_permissions[perm] == "`deny" + x := sprintf("You do not have permission to carry out '%v' operation.", [request_intent, input.request.path]) +} + +# Do not allow users (excluding admin users) to reset the password of another user. +deny[x] { + request_permission[_] = "ResetPassword" + not is_admin + user_id := split(input.request.path, "/`")[3] + user_id != input.user.id + x := sprintf("`You do not have permission to reset the password for other users.`", [user_id]) +} + +# Do not allow admin users to reset passwords if it is not allowed in the global config +deny[x] { + request_permission[_] == "`ResetPassword" + is_admin + user_id := split(input.request.path, "/`")[3] + user_id != input.user.id + not input.config.security.allow_admin_reset_password + not input.user.user_permissions["`ResetPassword"] + x := "You do not have permission to reset the password for other users. As an admin user, this permission can be modified using OPA rules." +} + +# --------- End "deny" rules ---------- + +################################################################################################################## +# Demo Section: Examples of rule capabilities. # +# The rules below are not executed until additional permissions have been assigned to the user or user group. # +################################################################################################################## +# If you are testing using OPA playground, you can mock Tyk functions like this: +# +# TykAPIGet(path) = {} +# TykDiff(o1,o2) = {} +# +# You can use this pre-built playground: https://play.openpolicyagent.org/p/T1Rcz5Ugnb +# Example: Deny users the ability to change the API status with an additional permission. +# Note: This rule will not be executed unless the additional permission is set. +deny["You do not have permission to change the API status."] { + # Checks the additional user permission enabled with tyk_analytics config: `"additional_permissions":["test_disable_deploy"]` + input.user.user_permissions["test_disable_deploy`"] + # Checks the request intent is to update the API + request_permission[_] == "`apis`" + request_intent == "`write" + # Checks if the user is attempting to update the field for API status. + # TykAPIGet accepts API URL as an argument, e.g. to receive API object call: TykAPIGet("/api/apis/<api-id>") + api := TykAPIGet(input.request.path) + # TykDiff performs Object diff and returns JSON Merge Patch document https://tools.ietf.org/html/rfc7396 + # eg. If only the state has changed, the diff may look like: {"active": true} + diff := TykDiff(api, input.request.body) + # Checks if API state has changed. + not is_null(diff.api_definition.active) +} + +# Using the patch_request helper you can modify the content of the request +# You should respond with JSON merge patch. +# See https://tools.ietf.org/html/rfc7396 for more details +# +# Example: Modify data under a certain condition by enforcing http proxy configuration for all APIs with the #external category. +patch_request[x] { + # Enforce only for users with ["test_patch_request"] permissions. + # Remove the ["test_patch_request"] permission to enforce the proxy configuration for all users instead of those with the permission. + input.user.user_permissions["test_patch_request`"] + request_permission[_] == "`apis`" + request_intent == "`write" + contains(input.request.body.api_definition.name, "#external") + x := {"api_definition": {"proxy": {"transport": {"proxy_url": "http://company-proxy:8080"}}}} +} + +# You can create additional permissions for not only individual users, but also user groups in your rules. +deny["Only '%v' group has permission to access this API"] { + # Checks for the additional user permission enabled with tyk_analytics config: '"additional_permissions":["test_admin_usergroup"] + input.user.user_permissions["test_admin_usergroup`"] + # Checks that the request intent is to access the API. + request_permission[_] == "`apis" + api := TykAPIGet(input.request.path) + # Checks that the API being accessed has the category #admin-teamA + contains(input.request.body.api_definition.name, "#admin-teamA`") + # Checks for the user group name. + not input.user.group_name == "`TeamA-Admin" +} +~~~ diff --git a/snippets/oss-product-list-include.mdx b/snippets/oss-product-list-include.mdx new file mode 100644 index 00000000..f7660494 --- /dev/null +++ b/snippets/oss-product-list-include.mdx @@ -0,0 +1,15 @@ +--- +--- + +## Tyk Open Source + +The Tyk Team has created and maintains the following components, which are fully Open Source and available under Mozilla Public License 2.0 (MPL). Star the Tyk components you use by clicking the appropriate button: + +* [Tyk Gateway](/tyk-oss-gateway) - Fully fledged API Gateway (Start here!) - <a href="https://github.com/TykTechnologies/tyk" className="github-button" target="_blank" rel="noopener noreferrer">⭐ Star TykTechnologies/tyk</a> +* [Tyk Pump](/tyk-pump) - Send API analytics data to your chosen backend - <a href="https://github.com/TykTechnologies/tyk-pump" className="github-button" target="_blank" rel="noopener noreferrer">⭐ Star TykTechnologies/tyk-pump</a> +* [Tyk Identity Broker](/api-management/external-service-integration#what-is-tyk-identity-broker-tib) - Connect your third-party IdP systems - <a href="https://github.com/TykTechnologies/api-management/external-service-integration#what-is-tyk-identity-broker-tib" className="github-button" target="_blank" rel="noopener noreferrer">⭐ Star TykTechnologies/api-management/external-service-integration#what-is-tyk-identity-broker-tib</a> +* [Tyk Helm Chart](/product-stack/tyk-charts/overview) - Deploy Tyk in K8S - <a href="https://github.com/TykTechnologies/tyk-charts" className="github-button" target="_blank" rel="noopener noreferrer">⭐ Star TykTechnologies/tyk-charts</a> + +MPL is a copyleft license that is easy to comply with. You must make the source code for any of your changes available under MPL, but you can combine the MPL software with proprietary code, as long as you keep the MPL code in separate files. Version 2.0 is, by default, compatible with LGPL and GPL version 2 or greater. You can distribute binaries under a proprietary license, as long as you make the source available under MPL. This is just a brief overview of MPL, you should refer to the MPL license in full and if you are in any doubt about how this license impacts you, please contact us to discuss. + +You can find additional FAQs regarding the MPL license [here](https://www.mozilla.org/en-US/MPL/2.0/FAQ/). \ No newline at end of file diff --git a/snippets/portal-developer-analytics.mdx b/snippets/portal-developer-analytics.mdx new file mode 100644 index 00000000..72dabb22 --- /dev/null +++ b/snippets/portal-developer-analytics.mdx @@ -0,0 +1,38 @@ +--- +--- + +### Analytics + +You can get aggregate statistics for 1 key or all developer keys (need to specify a list of all keys). Also, you can group by day (hour or month), and by API (policy id). + +API Endpoint: `/api/activity/keys/aggregate/#{keys}/#{from}/#{to}?p=-1&res=day` + +* `keys` should be specified separated by ',' delimiter. +* `from` and `to` values must be in // format. +* resolution specified `res` attribute: 'day', 'hour' or 'month' +* `api_id` - policy id associated with developer portal API. If ommited return stats for all APIs. + +#### Request + +```{.copyWrapper} +curl "https://admin.cloud.tyk.io/api/activity/keys/aggregate/add2b342,5f1d9603,/5/8/2017/13/8/2017?api_id=8e4d983609c044984ecbb286b8d25cd9&api_version=Non+Versioned&p=-1&res=day" \ +-X GET \ +-H "authorization: $TYK_API_KEY" +``` + +#### Response + +``` expandable +{ "data":[ + { + "id":{"day":9,"month":8,"year":2017,"hour":0,"code":200}, + "hits":13, + "success":10, + "error":3, + "last_hit":"2017-08-09T12:31:02Z" + }, + ... +],"pages":0} +``` + +In example above `add2b342,5f1d9603`, is 2 users keys. Note that this example shows [hashed key values as described here](/api-management/policies#access-key-hashing). Key hashing is turned on for the Cloud, but for Multi-Cloud and Self-Managed installations you can also turn it off. Hashed keys mean that the API administrator does not have access to real user keys, but they can still use the hashed values for showing analytics. \ No newline at end of file diff --git a/snippets/pump-config.mdx b/snippets/pump-config.mdx new file mode 100644 index 00000000..6fc831c3 --- /dev/null +++ b/snippets/pump-config.mdx @@ -0,0 +1,5100 @@ +### purge_delay +ENV: <b>TYK_PMP_PURGEDELAY</b><br /> +Type: `int`<br /> + +The number of seconds the Pump waits between checking for analytics data and purge it from +Redis. + +### purge_chunk +ENV: <b>TYK_PMP_PURGECHUNK</b><br /> +Type: `int64`<br /> + +The maximum number of records to pull from Redis at a time. If it's unset or `0`, all the +analytics records in Redis are pulled. If it's set, `storage_expiration_time` is used to +reset the analytics record TTL. + +### storage_expiration_time +ENV: <b>TYK_PMP_STORAGEEXPIRATIONTIME</b><br /> +Type: `int64`<br /> + +The number of seconds for the analytics records TTL. It only works if `purge_chunk` is +enabled. Defaults to `60` seconds. + +### dont_purge_uptime_data +ENV: <b>TYK_PMP_DONTPURGEUPTIMEDATA</b><br /> +Type: `bool`<br /> + +Setting this to `false` will create a pump that pushes uptime data to Uptime Pump, so the +Dashboard can read it. Disable by setting to `true`. + +### uptime_pump_config +Example Uptime Pump configuration: +```{.json} expandable +"uptime_pump_config": { + "uptime_type": "mongo", + "mongo_url": "mongodb://localhost:27017", + "collection_name": "tyk_uptime_analytics" +}, + +### SQL Uptime Pump +*Supported in Tyk Pump v1.5.0+* + +In `uptime_pump_config` you can configure a SQL uptime pump. To do that, you need to add the +field `uptime_type` with `sql` value. You can also use different types of SQL Uptime pumps, +like `postgres` or `sqlite` using the `type` field. + +An example of a SQL Postgres uptime pump would be: +```{.json} +"uptime_pump_config": { + "uptime_type": "sql", + "type": "postgres", + "connection_string": "host=sql_host port=sql_port user=sql_usr dbname=dbname password=sql_pw", + "table_sharding": false +}, +``` expandable + +Take into account that you can also set `log_level` field into the `uptime_pump_config` to `debug`, +`info` or `warning`. By default, the SQL logger verbosity is `silent`. + +### uptime_pump_config.EnvPrefix +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_ENVPREFIX</b><br /> +Type: `string`<br /> + + +The prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_SQL_META` + +### uptime_pump_config.mongo_url +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_MONGOURL</b><br /> +Type: `string`<br /> + +The full URL to your MongoDB instance, this can be a clustered instance if necessary and +should include the database and username / password data. + +### uptime_pump_config.mongo_use_ssl +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_MONGOUSESSL</b><br /> +Type: `bool`<br /> + +Set to true to enable Mongo SSL connection. + +### uptime_pump_config.mongo_ssl_insecure_skip_verify +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_MONGOSSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Allows the use of self-signed certificates when connecting to an encrypted MongoDB database. + +### uptime_pump_config.mongo_ssl_allow_invalid_hostnames +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_MONGOSSLALLOWINVALIDHOSTNAMES</b><br /> +Type: `bool`<br /> + +Ignore hostname check when it differs from the original (for example with SSH tunneling). +The rest of the TLS verification will still be performed. + +### uptime_pump_config.mongo_ssl_ca_file +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_MONGOSSLCAFILE</b><br /> +Type: `string`<br /> + +Path to the PEM file with trusted root certificates + +### uptime_pump_config.mongo_ssl_pem_keyfile +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_MONGOSSLPEMKEYFILE</b><br /> +Type: `string`<br /> + +Path to the PEM file which contains both client certificate and private key. This is +required for Mutual TLS. + +### uptime_pump_config.mongo_db_type +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_MONGODBTYPE</b><br /> +Type: `int`<br /> + +Specifies the mongo DB Type. If it's 0, it means that you are using standard mongo db. If it's 1 it means you are using AWS Document DB. If it's 2, it means you are using CosmosDB. +Defaults to Standard mongo (0). + +### uptime_pump_config.omit_index_creation +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_OMITINDEXCREATION</b><br /> +Type: `bool`<br /> + +Set to true to disable the default tyk index creation. + +### uptime_pump_config.mongo_session_consistency +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_MONGOSESSIONCONSISTENCY</b><br /> +Type: `string`<br /> + +Set the consistency mode for the session, it defaults to `Strong`. The valid values are: strong, monotonic, eventual. + +### uptime_pump_config.driver +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_MONGODRIVERTYPE</b><br /> +Type: `string`<br /> + +MongoDriverType is the type of the driver (library) to use. The valid values are: β€œmongo-go” and β€œmgo”. +Since v1.9, the default driver is "mongo-go". Check out this guide to [learn about MongoDB drivers supported by Tyk Pump](https://github.com/TykTechnologies/tyk-pump#driver-type). + +### uptime_pump_config.mongo_direct_connection +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_MONGODIRECTCONNECTION</b><br /> +Type: `bool`<br /> + +MongoDirectConnection informs whether to establish connections only with the specified seed servers, +or to obtain information for the whole cluster and establish connections with further servers too. +If true, the client will only connect to the host provided in the ConnectionString +and won't attempt to discover other hosts in the cluster. Useful when network restrictions +prevent discovery, such as with SSH tunneling. Default is false. + +### uptime_pump_config.collection_name +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_COLLECTIONNAME</b><br /> +Type: `string`<br /> + +Specifies the mongo collection name. + +### uptime_pump_config.max_insert_batch_size_bytes +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_MAXINSERTBATCHSIZEBYTES</b><br /> +Type: `int`<br /> + +Maximum insert batch size for mongo selective pump. If the batch we are writing surpasses this value, it will be sent in multiple batches. +Defaults to 10Mb. + +### uptime_pump_config.max_document_size_bytes +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_MAXDOCUMENTSIZEBYTES</b><br /> +Type: `int`<br /> + +Maximum document size. If the document exceed this value, it will be skipped. +Defaults to 10Mb. + +### uptime_pump_config.collection_cap_max_size_bytes +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_COLLECTIONCAPMAXSIZEBYTES</b><br /> +Type: `int`<br /> + +Amount of bytes of the capped collection in 64bits architectures. +Defaults to 5GB. + +### uptime_pump_config.collection_cap_enable +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_COLLECTIONCAPENABLE</b><br /> +Type: `bool`<br /> + +Enable collection capping. It's used to set a maximum size of the collection. + +### uptime_pump_config.type +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_TYPE</b><br /> +Type: `string`<br /> + +The supported and tested types are `sqlite` and `postgres`. + +### uptime_pump_config.connection_string +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_CONNECTIONSTRING</b><br /> +Type: `string`<br /> + +Specifies the connection string to the database. + +### uptime_pump_config.postgres +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_POSTGRES</b><br /> +Type: `PostgresConfig`<br /> + +Postgres configurations. + +### uptime_pump_config.postgres.prefer_simple_protocol +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_POSTGRES_PREFERSIMPLEPROTOCOL</b><br /> +Type: `bool`<br /> + +Disables implicit prepared statement usage. + +### uptime_pump_config.mysql +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_MYSQL</b><br /> +Type: `MysqlConfig`<br /> + +Mysql configurations. + +### uptime_pump_config.mysql.default_string_size +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_MYSQL_DEFAULTSTRINGSIZE</b><br /> +Type: `uint`<br /> + +Default size for string fields. Defaults to `256`. + +### uptime_pump_config.mysql.disable_datetime_precision +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_MYSQL_DISABLEDATETIMEPRECISION</b><br /> +Type: `bool`<br /> + +Disable datetime precision, which not supported before MySQL 5.6. + +### uptime_pump_config.mysql.dont_support_rename_index +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_MYSQL_DONTSUPPORTRENAMEINDEX</b><br /> +Type: `bool`<br /> + +Drop & create when rename index, rename index not supported before MySQL 5.7, MariaDB. + +### uptime_pump_config.mysql.dont_support_rename_column +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_MYSQL_DONTSUPPORTRENAMECOLUMN</b><br /> +Type: `bool`<br /> + +`change` when rename column, rename column not supported before MySQL 8, MariaDB. + +### uptime_pump_config.mysql.skip_initialize_with_version +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_MYSQL_SKIPINITIALIZEWITHVERSION</b><br /> +Type: `bool`<br /> + +Auto configure based on currently MySQL version. + +### uptime_pump_config.table_sharding +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_TABLESHARDING</b><br /> +Type: `bool`<br /> + +Specifies if all the analytics records are going to be stored in one table or in multiple +tables (one per day). By default, `false`. If `false`, all the records are going to be +stored in `tyk_aggregated` table. Instead, if it's `true`, all the records of the day are +going to be stored in `tyk_aggregated_YYYYMMDD` table, where `YYYYMMDD` is going to change +depending on the date. + +### uptime_pump_config.log_level +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_LOGLEVEL</b><br /> +Type: `string`<br /> + +Specifies the SQL log verbosity. The possible values are: `info`,`error` and `warning`. By +default, the value is `silent`, which means that it won't log any SQL query. + +### uptime_pump_config.batch_size +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_BATCHSIZE</b><br /> +Type: `int`<br /> + +Specifies the amount of records that are going to be written each batch. Type int. By +default, it writes 1000 records max per batch. + +### uptime_pump_config.uptime_type +ENV: <b>TYK_PMP_UPTIMEPUMPCONFIG_UPTIMETYPE</b><br /> +Type: `string`<br /> + +Determines the uptime type. Options are `mongo` and `sql`. Defaults to `mongo`. + +### pumps +The default environment variable prefix for each pump follows this format: +`TYK_PMP_PUMPS_{PUMP-NAME}_`, for example `TYK_PMP_PUMPS_KAFKA_`. + +You can also set custom names for each pump specifying the pump type. For example, if you +want a Kafka pump which is called `PROD` you need to create `TYK_PMP_PUMPS_PROD_TYPE=kafka` +and configure it using the `TYK_PMP_PUMPS_PROD_` prefix. + +### pumps.csv.name +ENV: <b>TYK_PMP_PUMPS_CSV_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.csv.type +ENV: <b>TYK_PMP_PUMPS_CSV_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.csv.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.csv.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_CSV_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.csv.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_CSV_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.csv.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_CSV_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.csv.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_CSV_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.csv.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_CSV_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.csv.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_CSV_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.csv.timeout +ENV: <b>TYK_PMP_PUMPS_CSV_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.csv.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_CSV_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.csv.max_record_size +ENV: <b>TYK_PMP_PUMPS_CSV_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.csv.ignore_fields +ENV: <b>TYK_PMP_PUMPS_CSV_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.csv.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_CSV_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +The prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_CSV_META` + +### pumps.csv.meta.csv_dir +ENV: <b>TYK_PMP_PUMPS_CSV_META_CSVDIR</b><br /> +Type: `string`<br /> + +The directory and the filename where the CSV data will be stored. + +### pumps.dogstatsd.name +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.dogstatsd.type +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.dogstatsd.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.dogstatsd.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.dogstatsd.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.dogstatsd.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.dogstatsd.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.dogstatsd.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.dogstatsd.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.dogstatsd.timeout +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.dogstatsd.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.dogstatsd.max_record_size +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.dogstatsd.ignore_fields +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.dogstatsd.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +The prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_DOGSTATSD_META` + +### pumps.dogstatsd.meta.namespace +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_META_NAMESPACE</b><br /> +Type: `string`<br /> + +Prefix for your metrics to datadog. + +### pumps.dogstatsd.meta.address +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_META_ADDRESS</b><br /> +Type: `string`<br /> + +Address of the datadog agent including host & port. + +### pumps.dogstatsd.meta.sample_rate +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_META_SAMPLERATE</b><br /> +Type: `float64`<br /> + +Defaults to `1` which equates to `100%` of requests. To sample at `50%`, set to `0.5`. + +### pumps.dogstatsd.meta.async_uds +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_META_ASYNCUDS</b><br /> +Type: `bool`<br /> + +Enable async UDS over UDP https://github.com/Datadog/datadog-go#unix-domain-sockets-client. + +### pumps.dogstatsd.meta.async_uds_write_timeout_seconds +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_META_ASYNCUDSWRITETIMEOUT</b><br /> +Type: `int`<br /> + +Integer write timeout in seconds if `async_uds: true`. + +### pumps.dogstatsd.meta.buffered +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_META_BUFFERED</b><br /> +Type: `bool`<br /> + +Enable buffering of messages. + +### pumps.dogstatsd.meta.buffered_max_messages +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_META_BUFFEREDMAXMESSAGES</b><br /> +Type: `int`<br /> + +Max messages in single datagram if `buffered: true`. Default 16. + +### pumps.dogstatsd.meta.tags +ENV: <b>TYK_PMP_PUMPS_DOGSTATSD_META_TAGS</b><br /> +Type: `[]string`<br /> + +List of tags to be added to the metric. The possible options are listed in the below example. + +If no tag is specified the fallback behavior is to use the below tags: +- `path` +- `method` +- `response_code` +- `api_version` +- `api_name` +- `api_id` +- `org_id` +- `tracked` +- `oauth_id` + +Note that this configuration can generate significant charges due to the unbound nature of +the `path` tag. + +```{.json} +"dogstatsd": { + "type": "dogstatsd", + "meta": { + "address": "localhost:8125", + "namespace": "pump", + "async_uds": true, + "async_uds_write_timeout_seconds": 2, + "buffered": true, + "buffered_max_messages": 32, + "sample_rate": 0.5, + "tags": [ + "method", + "response_code", + "api_version", + "api_name", + "api_id", + "org_id", + "tracked", + "path", + "oauth_id" + ] + } +}, +``` + +On startup, you should see the loaded configs when initializing the dogstatsd pump +``` +[May 10 15:23:44] INFO dogstatsd: initializing pump +[May 10 15:23:44] INFO dogstatsd: namespace: pump. +[May 10 15:23:44] INFO dogstatsd: sample_rate: 50% +[May 10 15:23:44] INFO dogstatsd: buffered: true, max_messages: 32 +[May 10 15:23:44] INFO dogstatsd: async_uds: true, write_timeout: 2s +``` expandable + +### pumps.elasticsearch.name +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.elasticsearch.type +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.elasticsearch.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.elasticsearch.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.elasticsearch.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.elasticsearch.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.elasticsearch.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.elasticsearch.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.elasticsearch.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.elasticsearch.timeout +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.elasticsearch.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.elasticsearch.max_record_size +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.elasticsearch.ignore_fields +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.elasticsearch.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +The prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_ELASTICSEARCH_META` + +### pumps.elasticsearch.meta.index_name +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_INDEXNAME</b><br /> +Type: `string`<br /> + +The name of the index that all the analytics data will be placed in. Defaults to +"tyk_analytics". + +### pumps.elasticsearch.meta.elasticsearch_url +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_ELASTICSEARCHURL</b><br /> +Type: `string`<br /> + +If sniffing is disabled, the URL that all data will be sent to. Defaults to +"http://localhost:9200". + +### pumps.elasticsearch.meta.use_sniffing +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_ENABLESNIFFING</b><br /> +Type: `bool`<br /> + +If sniffing is enabled, the "elasticsearch_url" will be used to make a request to get a +list of all the nodes in the cluster, the returned addresses will then be used. Defaults to +`false`. + +### pumps.elasticsearch.meta.document_type +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_DOCUMENTTYPE</b><br /> +Type: `string`<br /> + +The type of the document that is created in ES. Defaults to "tyk_analytics". + +### pumps.elasticsearch.meta.rolling_index +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_ROLLINGINDEX</b><br /> +Type: `bool`<br /> + +Appends the date to the end of the index name, so each days data is split into a different +index name. E.g. tyk_analytics-2016.02.28. Defaults to `false`. + +### pumps.elasticsearch.meta.extended_stats +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_EXTENDEDSTATISTICS</b><br /> +Type: `bool`<br /> + +If set to `true` will include the following additional fields: Raw Request, Raw Response and +User Agent. + +### pumps.elasticsearch.meta.generate_id +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_GENERATEID</b><br /> +Type: `bool`<br /> + +When enabled, generate _id for outgoing records. This prevents duplicate records when +retrying ES. + +### pumps.elasticsearch.meta.decode_base64 +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_DECODEBASE64</b><br /> +Type: `bool`<br /> + +Allows for the base64 bits to be decode before being passed to ES. + +### pumps.elasticsearch.meta.version +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_VERSION</b><br /> +Type: `string`<br /> + +Specifies the ES version. Use "3" for ES 3.X, "5" for ES 5.X, "6" for ES 6.X, "7" for ES +7.X . Defaults to "3". + +### pumps.elasticsearch.meta.disable_bulk +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_DISABLEBULK</b><br /> +Type: `bool`<br /> + +Disable batch writing. Defaults to false. + +### pumps.elasticsearch.meta.bulk_config +Batch writing trigger configuration. Each option is an OR with eachother: + +### pumps.elasticsearch.meta.bulk_config.workers +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_BULKCONFIG_WORKERS</b><br /> +Type: `int`<br /> + +Number of workers. Defaults to 1. + +### pumps.elasticsearch.meta.bulk_config.flush_interval +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_BULKCONFIG_FLUSHINTERVAL</b><br /> +Type: `int`<br /> + +Specifies the time in seconds to flush the data and send it to ES. Default disabled. + +### pumps.elasticsearch.meta.bulk_config.bulk_actions +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_BULKCONFIG_BULKACTIONS</b><br /> +Type: `int`<br /> + +Specifies the number of requests needed to flush the data and send it to ES. Defaults to +1000 requests. If it is needed, can be disabled with -1. + +### pumps.elasticsearch.meta.bulk_config.bulk_size +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_BULKCONFIG_BULKSIZE</b><br /> +Type: `int`<br /> + +Specifies the size (in bytes) needed to flush the data and send it to ES. Defaults to 5MB. +If it is needed, can be disabled with -1. + +### pumps.elasticsearch.meta.auth_api_key_id +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_AUTHAPIKEYID</b><br /> +Type: `string`<br /> + +API Key ID used for APIKey auth in ES. It's send to ES in the Authorization header as ApiKey base64(auth_api_key_id:auth_api_key) + +### pumps.elasticsearch.meta.auth_api_key +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_AUTHAPIKEY</b><br /> +Type: `string`<br /> + +API Key used for APIKey auth in ES. It's send to ES in the Authorization header as ApiKey base64(auth_api_key_id:auth_api_key) + +### pumps.elasticsearch.meta.auth_basic_username +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_USERNAME</b><br /> +Type: `string`<br /> + +Basic auth username. It's send to ES in the Authorization header as username:password encoded in base64. + +### pumps.elasticsearch.meta.auth_basic_password +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_PASSWORD</b><br /> +Type: `string`<br /> + +Basic auth password. It's send to ES in the Authorization header as username:password encoded in base64. + +### pumps.elasticsearch.meta.use_ssl +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_USESSL</b><br /> +Type: `bool`<br /> + +Enables SSL connection. + +### pumps.elasticsearch.meta.ssl_insecure_skip_verify +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_SSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Controls whether the pump client verifies the Elastic Search server's certificate chain and hostname. + +### pumps.elasticsearch.meta.ssl_cert_file +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_SSLCERTFILE</b><br /> +Type: `string`<br /> + +Can be used to set custom certificate file for authentication with Elastic Search. + +### pumps.elasticsearch.meta.ssl_key_file +ENV: <b>TYK_PMP_PUMPS_ELASTICSEARCH_META_SSLKEYFILE</b><br /> +Type: `string`<br /> + +Can be used to set custom key file for authentication with Elastic Search. + +### pumps.graylog.name +ENV: <b>TYK_PMP_PUMPS_GRAYLOG_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.graylog.type +ENV: <b>TYK_PMP_PUMPS_GRAYLOG_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.graylog.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.graylog.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_GRAYLOG_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.graylog.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_GRAYLOG_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.graylog.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_GRAYLOG_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.graylog.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_GRAYLOG_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.graylog.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_GRAYLOG_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.graylog.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_GRAYLOG_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.graylog.timeout +ENV: <b>TYK_PMP_PUMPS_GRAYLOG_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.graylog.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_GRAYLOG_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.graylog.max_record_size +ENV: <b>TYK_PMP_PUMPS_GRAYLOG_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.graylog.ignore_fields +ENV: <b>TYK_PMP_PUMPS_GRAYLOG_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.graylog.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_GRAYLOG_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +The prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_GRAYLOG_META` + +### pumps.graylog.meta.host +ENV: <b>TYK_PMP_PUMPS_GRAYLOG_META_GRAYLOGHOST</b><br /> +Type: `string`<br /> + +Graylog host. + +### pumps.graylog.meta.port +ENV: <b>TYK_PMP_PUMPS_GRAYLOG_META_GRAYLOGPORT</b><br /> +Type: `int`<br /> + +Graylog port. + +### pumps.graylog.meta.tags +ENV: <b>TYK_PMP_PUMPS_GRAYLOG_META_TAGS</b><br /> +Type: `[]string`<br /> + +List of tags to be added to the metric. The possible options are listed in the below example. + +If no tag is specified the fallback behaviour is to don't send anything. +The possible values are: +- `path` +- `method` +- `response_code` +- `api_version` +- `api_name` +- `api_id` +- `org_id` +- `tracked` +- `oauth_id` +- `raw_request` +- `raw_response` +- `request_time` +- `ip_address` + +### pumps.hybrid.name +ENV: <b>TYK_PMP_PUMPS_HYBRID_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.hybrid.type +ENV: <b>TYK_PMP_PUMPS_HYBRID_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.hybrid.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.hybrid.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_HYBRID_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.hybrid.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_HYBRID_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.hybrid.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_HYBRID_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.hybrid.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_HYBRID_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.hybrid.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_HYBRID_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.hybrid.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_HYBRID_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.hybrid.timeout +ENV: <b>TYK_PMP_PUMPS_HYBRID_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.hybrid.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_HYBRID_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.hybrid.max_record_size +ENV: <b>TYK_PMP_PUMPS_HYBRID_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.hybrid.ignore_fields +ENV: <b>TYK_PMP_PUMPS_HYBRID_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.hybrid.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_HYBRID_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +The prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_HYBRID_META` + +### pumps.hybrid.meta.ConnectionString +ENV: <b>TYK_PMP_PUMPS_HYBRID_META_CONNECTIONSTRING</b><br /> +Type: `string`<br /> + +MDCB URL connection string + +### pumps.hybrid.meta.RPCKey +ENV: <b>TYK_PMP_PUMPS_HYBRID_META_RPCKEY</b><br /> +Type: `string`<br /> + +Your organization ID to connect to the MDCB installation. + +### pumps.hybrid.meta.APIKey +ENV: <b>TYK_PMP_PUMPS_HYBRID_META_APIKEY</b><br /> +Type: `string`<br /> + +This the API key of a user used to authenticate and authorize the Hybrid Pump access through MDCB. +The user should be a standard Dashboard user with minimal privileges so as to reduce any risk if the user is compromised. + +### pumps.hybrid.meta.ignore_tag_prefix_list +ENV: <b>TYK_PMP_PUMPS_HYBRID_META_IGNORETAGPREFIXLIST</b><br /> +Type: `[]string`<br /> + +Specifies prefixes of tags that should be ignored if `aggregated` is set to `true`. + +### pumps.hybrid.meta.CallTimeout +ENV: <b>TYK_PMP_PUMPS_HYBRID_META_CALLTIMEOUT</b><br /> +Type: `int`<br /> + +Hybrid pump RPC calls timeout in seconds. Defaults to `10` seconds. + +### pumps.hybrid.meta.RPCPoolSize +ENV: <b>TYK_PMP_PUMPS_HYBRID_META_RPCPOOLSIZE</b><br /> +Type: `int`<br /> + +Hybrid pump connection pool size. Defaults to `5`. + +### pumps.hybrid.meta.aggregationTime +ENV: <b>TYK_PMP_PUMPS_HYBRID_META_AGGREGATIONTIME</b><br /> +Type: `int`<br /> + +aggregationTime is to specify the frequency of the aggregation in minutes if `aggregated` is set to `true`. + +### pumps.hybrid.meta.Aggregated +ENV: <b>TYK_PMP_PUMPS_HYBRID_META_AGGREGATED</b><br /> +Type: `bool`<br /> + +Send aggregated analytics data to Tyk MDCB + +### pumps.hybrid.meta.TrackAllPaths +ENV: <b>TYK_PMP_PUMPS_HYBRID_META_TRACKALLPATHS</b><br /> +Type: `bool`<br /> + +Specifies if it should store aggregated data for all the endpoints if `aggregated` is set to `true`. By default, `false` +which means that only store aggregated data for `tracked endpoints`. + +### pumps.hybrid.meta.store_analytics_per_minute +ENV: <b>TYK_PMP_PUMPS_HYBRID_META_STOREANALYTICSPERMINUTE</b><br /> +Type: `bool`<br /> + +Determines if the aggregations should be made per minute (true) or per hour (false) if `aggregated` is set to `true`. + +### pumps.hybrid.meta.UseSSL +ENV: <b>TYK_PMP_PUMPS_HYBRID_META_USESSL</b><br /> +Type: `bool`<br /> + +Use SSL to connect to Tyk MDCB + +### pumps.hybrid.meta.SSLInsecureSkipVerify +ENV: <b>TYK_PMP_PUMPS_HYBRID_META_SSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Skip SSL verification + +### pumps.influx.name +ENV: <b>TYK_PMP_PUMPS_INFLUX_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.influx.type +ENV: <b>TYK_PMP_PUMPS_INFLUX_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.influx.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.influx.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_INFLUX_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.influx.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_INFLUX_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.influx.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_INFLUX_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.influx.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_INFLUX_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.influx.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_INFLUX_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.influx.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_INFLUX_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.influx.timeout +ENV: <b>TYK_PMP_PUMPS_INFLUX_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.influx.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_INFLUX_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.influx.max_record_size +ENV: <b>TYK_PMP_PUMPS_INFLUX_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.influx.ignore_fields +ENV: <b>TYK_PMP_PUMPS_INFLUX_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.influx.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_INFLUX_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +The prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_INFLUX_META` + +### pumps.influx.meta.database_name +ENV: <b>TYK_PMP_PUMPS_INFLUX_META_DATABASENAME</b><br /> +Type: `string`<br /> + +InfluxDB pump database name. + +### pumps.influx.meta.address +ENV: <b>TYK_PMP_PUMPS_INFLUX_META_ADDR</b><br /> +Type: `string`<br /> + +InfluxDB pump host. + +### pumps.influx.meta.username +ENV: <b>TYK_PMP_PUMPS_INFLUX_META_USERNAME</b><br /> +Type: `string`<br /> + +InfluxDB pump database username. + +### pumps.influx.meta.password +ENV: <b>TYK_PMP_PUMPS_INFLUX_META_PASSWORD</b><br /> +Type: `string`<br /> + +InfluxDB pump database password. + +### pumps.influx.meta.fields +ENV: <b>TYK_PMP_PUMPS_INFLUX_META_FIELDS</b><br /> +Type: `[]string`<br /> + +Define which Analytics fields should be sent to InfluxDB. Check the available +fields in the example below. Default value is `["method", +"path", "response_code", "api_key", "time_stamp", "api_version", "api_name", "api_id", +"org_id", "oauth_id", "raw_request", "request_time", "raw_response", "ip_address"]`. + +### pumps.influx.meta.tags +ENV: <b>TYK_PMP_PUMPS_INFLUX_META_TAGS</b><br /> +Type: `[]string`<br /> + +List of tags to be added to the metric. + +### pumps.kafka.name +ENV: <b>TYK_PMP_PUMPS_KAFKA_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.kafka.type +ENV: <b>TYK_PMP_PUMPS_KAFKA_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.kafka.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.kafka.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_KAFKA_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.kafka.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_KAFKA_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.kafka.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_KAFKA_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.kafka.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_KAFKA_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.kafka.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_KAFKA_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.kafka.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_KAFKA_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.kafka.timeout +ENV: <b>TYK_PMP_PUMPS_KAFKA_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.kafka.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_KAFKA_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.kafka.max_record_size +ENV: <b>TYK_PMP_PUMPS_KAFKA_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.kafka.ignore_fields +ENV: <b>TYK_PMP_PUMPS_KAFKA_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.kafka.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_KAFKA_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +The prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_KAFKA_META` + +### pumps.kafka.meta.broker +ENV: <b>TYK_PMP_PUMPS_KAFKA_META_BROKER</b><br /> +Type: `[]string`<br /> + +The list of brokers used to discover the partitions available on the kafka cluster. E.g. +"localhost:9092". + +### pumps.kafka.meta.client_id +ENV: <b>TYK_PMP_PUMPS_KAFKA_META_CLIENTID</b><br /> +Type: `string`<br /> + +Unique identifier for client connections established with Kafka. + +### pumps.kafka.meta.topic +ENV: <b>TYK_PMP_PUMPS_KAFKA_META_TOPIC</b><br /> +Type: `string`<br /> + +The topic that the writer will produce messages to. + +### pumps.kafka.meta.timeout +ENV: <b>TYK_PMP_PUMPS_KAFKA_META_TIMEOUT</b><br /> +Type: `interface{}`<br /> + +Timeout is the maximum amount of seconds to wait for a connect or write to complete. + +### pumps.kafka.meta.compressed +ENV: <b>TYK_PMP_PUMPS_KAFKA_META_COMPRESSED</b><br /> +Type: `bool`<br /> + +Enable "github.com/golang/snappy" codec to be used to compress Kafka messages. By default +is `false`. + +### pumps.kafka.meta.meta_data +ENV: <b>TYK_PMP_PUMPS_KAFKA_META_METADATA</b><br /> +Type: `map[string]string`<br /> + +Can be used to set custom metadata inside the kafka message. + +### pumps.kafka.meta.use_ssl +ENV: <b>TYK_PMP_PUMPS_KAFKA_META_USESSL</b><br /> +Type: `bool`<br /> + +Enables SSL connection. + +### pumps.kafka.meta.ssl_insecure_skip_verify +ENV: <b>TYK_PMP_PUMPS_KAFKA_META_SSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Controls whether the pump client verifies the kafka server's certificate chain and host +name. + +### pumps.kafka.meta.ssl_cert_file +ENV: <b>TYK_PMP_PUMPS_KAFKA_META_SSLCERTFILE</b><br /> +Type: `string`<br /> + +Can be used to set custom certificate file for authentication with kafka. + +### pumps.kafka.meta.ssl_key_file +ENV: <b>TYK_PMP_PUMPS_KAFKA_META_SSLKEYFILE</b><br /> +Type: `string`<br /> + +Can be used to set custom key file for authentication with kafka. + +### pumps.kafka.meta.sasl_mechanism +ENV: <b>TYK_PMP_PUMPS_KAFKA_META_SASLMECHANISM</b><br /> +Type: `string`<br /> + +SASL mechanism configuration. Only "plain" and "scram" are supported. + +### pumps.kafka.meta.sasl_username +ENV: <b>TYK_PMP_PUMPS_KAFKA_META_USERNAME</b><br /> +Type: `string`<br /> + +SASL username. + +### pumps.kafka.meta.sasl_password +ENV: <b>TYK_PMP_PUMPS_KAFKA_META_PASSWORD</b><br /> +Type: `string`<br /> + +SASL password. + +### pumps.kafka.meta.sasl_algorithm +ENV: <b>TYK_PMP_PUMPS_KAFKA_META_ALGORITHM</b><br /> +Type: `string`<br /> + +SASL algorithm. It's the algorithm specified for scram mechanism. It could be sha-512 or sha-256. +Defaults to "sha-256". + +### pumps.kinesis.name +ENV: <b>TYK_PMP_PUMPS_KINESIS_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.kinesis.type +ENV: <b>TYK_PMP_PUMPS_KINESIS_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.kinesis.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.kinesis.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_KINESIS_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.kinesis.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_KINESIS_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.kinesis.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_KINESIS_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.kinesis.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_KINESIS_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.kinesis.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_KINESIS_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.kinesis.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_KINESIS_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.kinesis.timeout +ENV: <b>TYK_PMP_PUMPS_KINESIS_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.kinesis.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_KINESIS_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.kinesis.max_record_size +ENV: <b>TYK_PMP_PUMPS_KINESIS_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.kinesis.ignore_fields +ENV: <b>TYK_PMP_PUMPS_KINESIS_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.kinesis.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_KINESIS_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +The prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_KINESIS_META` + +### pumps.kinesis.meta.StreamName +ENV: <b>TYK_PMP_PUMPS_KINESIS_META_STREAMNAME</b><br /> +Type: `string`<br /> + +A name to identify the stream. The stream name is scoped to the AWS account used by the application +that creates the stream. It is also scoped by AWS Region. +That is, two streams in two different AWS accounts can have the same name. +Two streams in the same AWS account but in two different Regions can also have the same name. + +### pumps.kinesis.meta.Region +ENV: <b>TYK_PMP_PUMPS_KINESIS_META_REGION</b><br /> +Type: `string`<br /> + +AWS Region the Kinesis stream targets + +### pumps.kinesis.meta.BatchSize +ENV: <b>TYK_PMP_PUMPS_KINESIS_META_BATCHSIZE</b><br /> +Type: `int`<br /> + +Each PutRecords (the function used in this pump)request can support up to 500 records. +Each record in the request can be as large as 1 MiB, up to a limit of 5 MiB for the entire request, including partition keys. +Each shard can support writes up to 1,000 records per second, up to a maximum data write total of 1 MiB per second. + +### pumps.logzio.name +ENV: <b>TYK_PMP_PUMPS_LOGZIO_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.logzio.type +ENV: <b>TYK_PMP_PUMPS_LOGZIO_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.logzio.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.logzio.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_LOGZIO_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.logzio.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_LOGZIO_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.logzio.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_LOGZIO_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.logzio.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_LOGZIO_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.logzio.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_LOGZIO_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.logzio.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_LOGZIO_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.logzio.timeout +ENV: <b>TYK_PMP_PUMPS_LOGZIO_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.logzio.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_LOGZIO_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.logzio.max_record_size +ENV: <b>TYK_PMP_PUMPS_LOGZIO_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.logzio.ignore_fields +ENV: <b>TYK_PMP_PUMPS_LOGZIO_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.logzio.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_LOGZIO_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +The prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_LOGZIO_META` + +### pumps.logzio.meta.check_disk_space +ENV: <b>TYK_PMP_PUMPS_LOGZIO_META_CHECKDISKSPACE</b><br /> +Type: `bool`<br /> + +Set the sender to check if it crosses the maximum allowed disk usage. Default value is +`true`. + +### pumps.logzio.meta.disk_threshold +ENV: <b>TYK_PMP_PUMPS_LOGZIO_META_DISKTHRESHOLD</b><br /> +Type: `int`<br /> + +Set disk queue threshold, once the threshold is crossed the sender will not enqueue the +received logs. Default value is `98` (percentage of disk). + +### pumps.logzio.meta.drain_duration +ENV: <b>TYK_PMP_PUMPS_LOGZIO_META_DRAINDURATION</b><br /> +Type: `string`<br /> + +Set drain duration (flush logs on disk). Default value is `3s`. + +### pumps.logzio.meta.queue_dir +ENV: <b>TYK_PMP_PUMPS_LOGZIO_META_QUEUEDIR</b><br /> +Type: `string`<br /> + +The directory for the queue. + +### pumps.logzio.meta.token +ENV: <b>TYK_PMP_PUMPS_LOGZIO_META_TOKEN</b><br /> +Type: `string`<br /> + +Token for sending data to your logzio account. + +### pumps.logzio.meta.url +ENV: <b>TYK_PMP_PUMPS_LOGZIO_META_URL</b><br /> +Type: `string`<br /> + +If you do not want to use the default Logzio url i.e. when using a proxy. Default is +`https://listener.logz.io:8071`. + +### pumps.moesif.name +ENV: <b>TYK_PMP_PUMPS_MOESIF_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.moesif.type +ENV: <b>TYK_PMP_PUMPS_MOESIF_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.moesif.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.moesif.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_MOESIF_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.moesif.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_MOESIF_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.moesif.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_MOESIF_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.moesif.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_MOESIF_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.moesif.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_MOESIF_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.moesif.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_MOESIF_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.moesif.timeout +ENV: <b>TYK_PMP_PUMPS_MOESIF_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.moesif.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_MOESIF_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.moesif.max_record_size +ENV: <b>TYK_PMP_PUMPS_MOESIF_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.moesif.ignore_fields +ENV: <b>TYK_PMP_PUMPS_MOESIF_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.moesif.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_MOESIF_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +The prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_MOESIF_META` + +### pumps.moesif.meta.application_id +ENV: <b>TYK_PMP_PUMPS_MOESIF_META_APPLICATIONID</b><br /> +Type: `string`<br /> + +Moesif Application Id. You can find your Moesif Application Id from +[_Moesif Dashboard_](https://www.moesif.com/) -> _Top Right Menu_ -> _API Keys_ . Moesif +recommends creating separate Application Ids for each environment such as Production, +Staging, and Development to keep data isolated. + +### pumps.moesif.meta.request_header_masks +ENV: <b>TYK_PMP_PUMPS_MOESIF_META_REQUESTHEADERMASKS</b><br /> +Type: `[]string`<br /> + +An option to mask a specific request header field. + +### pumps.moesif.meta.response_header_masks +ENV: <b>TYK_PMP_PUMPS_MOESIF_META_RESPONSEHEADERMASKS</b><br /> +Type: `[]string`<br /> + +An option to mask a specific response header field. + +### pumps.moesif.meta.request_body_masks +ENV: <b>TYK_PMP_PUMPS_MOESIF_META_REQUESTBODYMASKS</b><br /> +Type: `[]string`<br /> + +An option to mask a specific - request body field. + +### pumps.moesif.meta.response_body_masks +ENV: <b>TYK_PMP_PUMPS_MOESIF_META_RESPONSEBODYMASKS</b><br /> +Type: `[]string`<br /> + +An option to mask a specific response body field. + +### pumps.moesif.meta.disable_capture_request_body +ENV: <b>TYK_PMP_PUMPS_MOESIF_META_DISABLECAPTUREREQUESTBODY</b><br /> +Type: `bool`<br /> + +An option to disable logging of request body. Default value is `false`. + +### pumps.moesif.meta.disable_capture_response_body +ENV: <b>TYK_PMP_PUMPS_MOESIF_META_DISABLECAPTURERESPONSEBODY</b><br /> +Type: `bool`<br /> + +An option to disable logging of response body. Default value is `false`. + +### pumps.moesif.meta.user_id_header +ENV: <b>TYK_PMP_PUMPS_MOESIF_META_USERIDHEADER</b><br /> +Type: `string`<br /> + +An optional field name to identify User from a request or response header. + +### pumps.moesif.meta.company_id_header +ENV: <b>TYK_PMP_PUMPS_MOESIF_META_COMPANYIDHEADER</b><br /> +Type: `string`<br /> + +An optional field name to identify Company (Account) from a request or response header. + +### pumps.moesif.meta.enable_bulk +ENV: <b>TYK_PMP_PUMPS_MOESIF_META_ENABLEBULK</b><br /> +Type: `bool`<br /> + +Set this to `true` to enable `bulk_config`. + +### pumps.moesif.meta.bulk_config +ENV: <b>TYK_PMP_PUMPS_MOESIF_META_BULKCONFIG</b><br /> +Type: `map[string]interface{}`<br /> + +Batch writing trigger configuration. + * `"event_queue_size"` - (optional) An optional field name which specify the maximum +number of events to hold in queue before sending to Moesif. In case of network issues when +not able to connect/send event to Moesif, skips adding new events to the queue to prevent +memory overflow. Type: int. Default value is `10000`. + * `"batch_size"` - (optional) An optional field name which specify the maximum batch size +when sending to Moesif. Type: int. Default value is `200`. + * `"timer_wake_up_seconds"` - (optional) An optional field which specifies a time (every n +seconds) how often background thread runs to send events to moesif. Type: int. Default value +is `2` seconds. + +### pumps.moesif.meta.authorization_header_name +ENV: <b>TYK_PMP_PUMPS_MOESIF_META_AUTHORIZATIONHEADERNAME</b><br /> +Type: `string`<br /> + +An optional request header field name to used to identify the User in Moesif. Default value +is `authorization`. + +### pumps.moesif.meta.authorization_user_id_field +ENV: <b>TYK_PMP_PUMPS_MOESIF_META_AUTHORIZATIONUSERIDFIELD</b><br /> +Type: `string`<br /> + +An optional field name use to parse the User from authorization header in Moesif. Default +value is `sub`. + +### pumps.mongo.name +ENV: <b>TYK_PMP_PUMPS_MONGO_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.mongo.type +ENV: <b>TYK_PMP_PUMPS_MONGO_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.mongo.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.mongo.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_MONGO_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.mongo.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_MONGO_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.mongo.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_MONGO_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.mongo.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_MONGO_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.mongo.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_MONGO_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.mongo.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_MONGO_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.mongo.timeout +ENV: <b>TYK_PMP_PUMPS_MONGO_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.mongo.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_MONGO_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.mongo.max_record_size +ENV: <b>TYK_PMP_PUMPS_MONGO_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.mongo.ignore_fields +ENV: <b>TYK_PMP_PUMPS_MONGO_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.mongo.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_MONGO_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +Prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_MONGO_META` for Mongo Pump +`TYK_PMP_PUMPS_UPTIME_META` for Uptime Pump +`TYK_PMP_PUMPS_MONGOAGGREGATE_META` for Mongo Aggregate Pump +`TYK_PMP_PUMPS_MONGOSELECTIVE_META` for Mongo Selective Pump +`TYK_PMP_PUMPS_MONGOGRAPH_META` for Mongo Graph Pump. + +### pumps.mongo.meta.mongo_url +ENV: <b>TYK_PMP_PUMPS_MONGO_META_MONGOURL</b><br /> +Type: `string`<br /> + +The full URL to your MongoDB instance, this can be a clustered instance if necessary and +should include the database and username / password data. + +### pumps.mongo.meta.mongo_use_ssl +ENV: <b>TYK_PMP_PUMPS_MONGO_META_MONGOUSESSL</b><br /> +Type: `bool`<br /> + +Set to true to enable Mongo SSL connection. + +### pumps.mongo.meta.mongo_ssl_insecure_skip_verify +ENV: <b>TYK_PMP_PUMPS_MONGO_META_MONGOSSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Allows the use of self-signed certificates when connecting to an encrypted MongoDB database. + +### pumps.mongo.meta.mongo_ssl_allow_invalid_hostnames +ENV: <b>TYK_PMP_PUMPS_MONGO_META_MONGOSSLALLOWINVALIDHOSTNAMES</b><br /> +Type: `bool`<br /> + +Ignore hostname check when it differs from the original (for example with SSH tunneling). +The rest of the TLS verification will still be performed. + +### pumps.mongo.meta.mongo_ssl_ca_file +ENV: <b>TYK_PMP_PUMPS_MONGO_META_MONGOSSLCAFILE</b><br /> +Type: `string`<br /> + +Path to the PEM file with trusted root certificates + +### pumps.mongo.meta.mongo_ssl_pem_keyfile +ENV: <b>TYK_PMP_PUMPS_MONGO_META_MONGOSSLPEMKEYFILE</b><br /> +Type: `string`<br /> + +Path to the PEM file which contains both client certificate and private key. This is +required for Mutual TLS. + +### pumps.mongo.meta.mongo_db_type +ENV: <b>TYK_PMP_PUMPS_MONGO_META_MONGODBTYPE</b><br /> +Type: `int`<br /> + +Specifies the mongo DB Type. If it's 0, it means that you are using standard mongo db. If it's 1 it means you are using AWS Document DB. If it's 2, it means you are using CosmosDB. +Defaults to Standard mongo (0). + +### pumps.mongo.meta.omit_index_creation +ENV: <b>TYK_PMP_PUMPS_MONGO_META_OMITINDEXCREATION</b><br /> +Type: `bool`<br /> + +Set to true to disable the default tyk index creation. + +### pumps.mongo.meta.mongo_session_consistency +ENV: <b>TYK_PMP_PUMPS_MONGO_META_MONGOSESSIONCONSISTENCY</b><br /> +Type: `string`<br /> + +Set the consistency mode for the session, it defaults to `Strong`. The valid values are: strong, monotonic, eventual. + +### pumps.mongo.meta.driver +ENV: <b>TYK_PMP_PUMPS_MONGO_META_MONGODRIVERTYPE</b><br /> +Type: `string`<br /> + +MongoDriverType is the type of the driver (library) to use. The valid values are: β€œmongo-go” and β€œmgo”. +Since v1.9, the default driver is "mongo-go". Check out this guide to [learn about MongoDB drivers supported by Tyk Pump](https://github.com/TykTechnologies/tyk-pump#driver-type). + +### pumps.mongo.meta.mongo_direct_connection +ENV: <b>TYK_PMP_PUMPS_MONGO_META_MONGODIRECTCONNECTION</b><br /> +Type: `bool`<br /> + +MongoDirectConnection informs whether to establish connections only with the specified seed servers, +or to obtain information for the whole cluster and establish connections with further servers too. +If true, the client will only connect to the host provided in the ConnectionString +and won't attempt to discover other hosts in the cluster. Useful when network restrictions +prevent discovery, such as with SSH tunneling. Default is false. + +### pumps.mongo.meta.collection_name +ENV: <b>TYK_PMP_PUMPS_MONGO_META_COLLECTIONNAME</b><br /> +Type: `string`<br /> + +Specifies the mongo collection name. + +### pumps.mongo.meta.max_insert_batch_size_bytes +ENV: <b>TYK_PMP_PUMPS_MONGO_META_MAXINSERTBATCHSIZEBYTES</b><br /> +Type: `int`<br /> + +Maximum insert batch size for mongo selective pump. If the batch we are writing surpasses this value, it will be sent in multiple batches. +Defaults to 10Mb. + +### pumps.mongo.meta.max_document_size_bytes +ENV: <b>TYK_PMP_PUMPS_MONGO_META_MAXDOCUMENTSIZEBYTES</b><br /> +Type: `int`<br /> + +Maximum document size. If the document exceed this value, it will be skipped. +Defaults to 10Mb. + +### pumps.mongo.meta.collection_cap_max_size_bytes +ENV: <b>TYK_PMP_PUMPS_MONGO_META_COLLECTIONCAPMAXSIZEBYTES</b><br /> +Type: `int`<br /> + +Amount of bytes of the capped collection in 64bits architectures. +Defaults to 5GB. + +### pumps.mongo.meta.collection_cap_enable +ENV: <b>TYK_PMP_PUMPS_MONGO_META_COLLECTIONCAPENABLE</b><br /> +Type: `bool`<br /> + +Enable collection capping. It's used to set a maximum size of the collection. + +### pumps.mongoaggregate.name +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.mongoaggregate.type +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.mongoaggregate.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.mongoaggregate.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.mongoaggregate.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.mongoaggregate.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.mongoaggregate.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.mongoaggregate.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.mongoaggregate.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.mongoaggregate.timeout +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.mongoaggregate.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.mongoaggregate.max_record_size +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.mongoaggregate.ignore_fields +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.mongoaggregate.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +Prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_MONGO_META` for Mongo Pump +`TYK_PMP_PUMPS_UPTIME_META` for Uptime Pump +`TYK_PMP_PUMPS_MONGOAGGREGATE_META` for Mongo Aggregate Pump +`TYK_PMP_PUMPS_MONGOSELECTIVE_META` for Mongo Selective Pump +`TYK_PMP_PUMPS_MONGOGRAPH_META` for Mongo Graph Pump. + +### pumps.mongoaggregate.meta.mongo_url +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGOURL</b><br /> +Type: `string`<br /> + +The full URL to your MongoDB instance, this can be a clustered instance if necessary and +should include the database and username / password data. + +### pumps.mongoaggregate.meta.mongo_use_ssl +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGOUSESSL</b><br /> +Type: `bool`<br /> + +Set to true to enable Mongo SSL connection. + +### pumps.mongoaggregate.meta.mongo_ssl_insecure_skip_verify +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGOSSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Allows the use of self-signed certificates when connecting to an encrypted MongoDB database. + +### pumps.mongoaggregate.meta.mongo_ssl_allow_invalid_hostnames +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGOSSLALLOWINVALIDHOSTNAMES</b><br /> +Type: `bool`<br /> + +Ignore hostname check when it differs from the original (for example with SSH tunneling). +The rest of the TLS verification will still be performed. + +### pumps.mongoaggregate.meta.mongo_ssl_ca_file +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGOSSLCAFILE</b><br /> +Type: `string`<br /> + +Path to the PEM file with trusted root certificates + +### pumps.mongoaggregate.meta.mongo_ssl_pem_keyfile +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGOSSLPEMKEYFILE</b><br /> +Type: `string`<br /> + +Path to the PEM file which contains both client certificate and private key. This is +required for Mutual TLS. + +### pumps.mongoaggregate.meta.mongo_db_type +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGODBTYPE</b><br /> +Type: `int`<br /> + +Specifies the mongo DB Type. If it's 0, it means that you are using standard mongo db. If it's 1 it means you are using AWS Document DB. If it's 2, it means you are using CosmosDB. +Defaults to Standard mongo (0). + +### pumps.mongoaggregate.meta.omit_index_creation +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_OMITINDEXCREATION</b><br /> +Type: `bool`<br /> + +Set to true to disable the default tyk index creation. + +### pumps.mongoaggregate.meta.mongo_session_consistency +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGOSESSIONCONSISTENCY</b><br /> +Type: `string`<br /> + +Set the consistency mode for the session, it defaults to `Strong`. The valid values are: strong, monotonic, eventual. + +### pumps.mongoaggregate.meta.driver +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGODRIVERTYPE</b><br /> +Type: `string`<br /> + +MongoDriverType is the type of the driver (library) to use. The valid values are: β€œmongo-go” and β€œmgo”. +Since v1.9, the default driver is "mongo-go". Check out this guide to [learn about MongoDB drivers supported by Tyk Pump](https://github.com/TykTechnologies/tyk-pump#driver-type). + +### pumps.mongoaggregate.meta.mongo_direct_connection +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_MONGODIRECTCONNECTION</b><br /> +Type: `bool`<br /> + +MongoDirectConnection informs whether to establish connections only with the specified seed servers, +or to obtain information for the whole cluster and establish connections with further servers too. +If true, the client will only connect to the host provided in the ConnectionString +and won't attempt to discover other hosts in the cluster. Useful when network restrictions +prevent discovery, such as with SSH tunneling. Default is false. + +### pumps.mongoaggregate.meta.use_mixed_collection +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_USEMIXEDCOLLECTION</b><br /> +Type: `bool`<br /> + +If set to `true` your pump will store analytics to both your organization defined +collections `z_tyk_analyticz_aggregate_{ORG ID}` and your org-less tyk_analytics_aggregates +collection. When set to 'false' your pump will only store analytics to your org defined +collection. + +### pumps.mongoaggregate.meta.track_all_paths +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_TRACKALLPATHS</b><br /> +Type: `bool`<br /> + +Specifies if it should store aggregated data for all the endpoints. By default, `false` +which means that only store aggregated data for `tracked endpoints`. + +### pumps.mongoaggregate.meta.ignore_tag_prefix_list +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_IGNORETAGPREFIXLIST</b><br /> +Type: `[]string`<br /> + +Specifies prefixes of tags that should be ignored. + +### pumps.mongoaggregate.meta.threshold_len_tag_list +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_THRESHOLDLENTAGLIST</b><br /> +Type: `int`<br /> + +Determines the threshold of amount of tags of an aggregation. If the amount of tags is superior to the threshold, +it will print an alert. +Defaults to 1000. + +### pumps.mongoaggregate.meta.store_analytics_per_minute +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_STOREANALYTICSPERMINUTE</b><br /> +Type: `bool`<br /> + +Determines if the aggregations should be made per minute (true) or per hour (false). + +### pumps.mongoaggregate.meta.aggregation_time +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_AGGREGATIONTIME</b><br /> +Type: `int`<br /> + +Determines the amount of time the aggregations should be made (in minutes). It defaults to the max value is 60 and the minimum is 1. +If StoreAnalyticsPerMinute is set to true, this field will be skipped. + +### pumps.mongoaggregate.meta.enable_aggregate_self_healing +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_ENABLEAGGREGATESELFHEALING</b><br /> +Type: `bool`<br /> + +Determines if the self healing will be activated or not. +Self Healing allows pump to handle Mongo document's max-size errors by creating a new document when the max-size is reached. +It also divide by 2 the AggregationTime field to avoid the same error in the future. + +### pumps.mongoaggregate.meta.ignore_aggregations +ENV: <b>TYK_PMP_PUMPS_MONGOAGGREGATE_META_IGNOREAGGREGATIONSLIST</b><br /> +Type: `[]string`<br /> + +This list determines which aggregations are going to be dropped and not stored in the collection. +Posible values are: "APIID","errors","versions","apikeys","oauthids","geo","tags","endpoints","keyendpoints", +"oauthendpoints", and "apiendpoints". + +### pumps.mongoselective.name +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.mongoselective.type +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.mongoselective.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.mongoselective.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.mongoselective.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.mongoselective.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.mongoselective.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.mongoselective.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.mongoselective.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.mongoselective.timeout +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.mongoselective.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.mongoselective.max_record_size +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.mongoselective.ignore_fields +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.mongoselective.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +Prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_MONGO_META` for Mongo Pump +`TYK_PMP_PUMPS_UPTIME_META` for Uptime Pump +`TYK_PMP_PUMPS_MONGOAGGREGATE_META` for Mongo Aggregate Pump +`TYK_PMP_PUMPS_MONGOSELECTIVE_META` for Mongo Selective Pump +`TYK_PMP_PUMPS_MONGOGRAPH_META` for Mongo Graph Pump. + +### pumps.mongoselective.meta.mongo_url +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_META_MONGOURL</b><br /> +Type: `string`<br /> + +The full URL to your MongoDB instance, this can be a clustered instance if necessary and +should include the database and username / password data. + +### pumps.mongoselective.meta.mongo_use_ssl +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_META_MONGOUSESSL</b><br /> +Type: `bool`<br /> + +Set to true to enable Mongo SSL connection. + +### pumps.mongoselective.meta.mongo_ssl_insecure_skip_verify +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_META_MONGOSSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Allows the use of self-signed certificates when connecting to an encrypted MongoDB database. + +### pumps.mongoselective.meta.mongo_ssl_allow_invalid_hostnames +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_META_MONGOSSLALLOWINVALIDHOSTNAMES</b><br /> +Type: `bool`<br /> + +Ignore hostname check when it differs from the original (for example with SSH tunneling). +The rest of the TLS verification will still be performed. + +### pumps.mongoselective.meta.mongo_ssl_ca_file +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_META_MONGOSSLCAFILE</b><br /> +Type: `string`<br /> + +Path to the PEM file with trusted root certificates + +### pumps.mongoselective.meta.mongo_ssl_pem_keyfile +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_META_MONGOSSLPEMKEYFILE</b><br /> +Type: `string`<br /> + +Path to the PEM file which contains both client certificate and private key. This is +required for Mutual TLS. + +### pumps.mongoselective.meta.mongo_db_type +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_META_MONGODBTYPE</b><br /> +Type: `int`<br /> + +Specifies the mongo DB Type. If it's 0, it means that you are using standard mongo db. If it's 1 it means you are using AWS Document DB. If it's 2, it means you are using CosmosDB. +Defaults to Standard mongo (0). + +### pumps.mongoselective.meta.omit_index_creation +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_META_OMITINDEXCREATION</b><br /> +Type: `bool`<br /> + +Set to true to disable the default tyk index creation. + +### pumps.mongoselective.meta.mongo_session_consistency +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_META_MONGOSESSIONCONSISTENCY</b><br /> +Type: `string`<br /> + +Set the consistency mode for the session, it defaults to `Strong`. The valid values are: strong, monotonic, eventual. + +### pumps.mongoselective.meta.driver +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_META_MONGODRIVERTYPE</b><br /> +Type: `string`<br /> + +MongoDriverType is the type of the driver (library) to use. The valid values are: β€œmongo-go” and β€œmgo”. +Since v1.9, the default driver is "mongo-go". Check out this guide to [learn about MongoDB drivers supported by Tyk Pump](https://github.com/TykTechnologies/tyk-pump#driver-type). + +### pumps.mongoselective.meta.mongo_direct_connection +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_META_MONGODIRECTCONNECTION</b><br /> +Type: `bool`<br /> + +MongoDirectConnection informs whether to establish connections only with the specified seed servers, +or to obtain information for the whole cluster and establish connections with further servers too. +If true, the client will only connect to the host provided in the ConnectionString +and won't attempt to discover other hosts in the cluster. Useful when network restrictions +prevent discovery, such as with SSH tunneling. Default is false. + +### pumps.mongoselective.meta.max_insert_batch_size_bytes +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_META_MAXINSERTBATCHSIZEBYTES</b><br /> +Type: `int`<br /> + +Maximum insert batch size for mongo selective pump. If the batch we are writing surpass this value, it will be send in multiple batchs. +Defaults to 10Mb. + +### pumps.mongoselective.meta.max_document_size_bytes +ENV: <b>TYK_PMP_PUMPS_MONGOSELECTIVE_META_MAXDOCUMENTSIZEBYTES</b><br /> +Type: `int`<br /> + +Maximum document size. If the document exceed this value, it will be skipped. +Defaults to 10Mb. + +### pumps.prometheus.name +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.prometheus.type +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.prometheus.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.prometheus.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.prometheus.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.prometheus.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.prometheus.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.prometheus.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.prometheus.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.prometheus.timeout +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.prometheus.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.prometheus.max_record_size +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.prometheus.ignore_fields +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.prometheus.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +Prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_PROMETHEUS_META` + +### pumps.prometheus.meta.listen_address +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_META_ADDR</b><br /> +Type: `string`<br /> + +The full URL to your Prometheus instance, {HOST}:{PORT}. For example `localhost:9090`. + +### pumps.prometheus.meta.path +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_META_PATH</b><br /> +Type: `string`<br /> + +The path to the Prometheus collection. For example `/metrics`. + +### pumps.prometheus.meta.aggregate_observations +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_META_AGGREGATEOBSERVATIONS</b><br /> +Type: `bool`<br /> + +This will enable an experimental feature that will aggregate the histogram metrics request time values before exposing them to prometheus. +Enabling this will reduce the CPU usage of your prometheus pump but you will loose histogram precision. Experimental. + +### pumps.prometheus.meta.disabled_metrics +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_META_DISABLEDMETRICS</b><br /> +Type: `[]string`<br /> + +Metrics to exclude from exposition. Currently, excludes only the base metrics. + +### pumps.prometheus.meta.track_all_paths +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_META_TRACKALLPATHS</b><br /> +Type: `bool`<br /> + +Specifies if it should expose aggregated metrics for all the endpoints. By default, `false` +which means that all APIs endpoints will be counted as 'unknown' unless the API uses the track endpoint plugin. + +### pumps.prometheus.meta.custom_metrics +ENV: <b>TYK_PMP_PUMPS_PROMETHEUS_META_CUSTOMMETRICS</b><br /> +Type: `CustomMetrics`<br /> + +Custom Prometheus metrics. + +### pumps.splunk.name +ENV: <b>TYK_PMP_PUMPS_SPLUNK_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.splunk.type +ENV: <b>TYK_PMP_PUMPS_SPLUNK_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.splunk.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.splunk.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_SPLUNK_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.splunk.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_SPLUNK_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.splunk.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_SPLUNK_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.splunk.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_SPLUNK_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.splunk.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_SPLUNK_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.splunk.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_SPLUNK_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.splunk.timeout +ENV: <b>TYK_PMP_PUMPS_SPLUNK_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.splunk.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_SPLUNK_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.splunk.max_record_size +ENV: <b>TYK_PMP_PUMPS_SPLUNK_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.splunk.ignore_fields +ENV: <b>TYK_PMP_PUMPS_SPLUNK_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.splunk.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_SPLUNK_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +The prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_SPLUNK_META` + +### pumps.splunk.meta.collector_token +ENV: <b>TYK_PMP_PUMPS_SPLUNK_META_COLLECTORTOKEN</b><br /> +Type: `string`<br /> + +Address of the datadog agent including host & port. + +### pumps.splunk.meta.collector_url +ENV: <b>TYK_PMP_PUMPS_SPLUNK_META_COLLECTORURL</b><br /> +Type: `string`<br /> + +Endpoint the Pump will send analytics too. Should look something like: +`https://splunk:8088/services/collector/event`. + +### pumps.splunk.meta.ssl_insecure_skip_verify +ENV: <b>TYK_PMP_PUMPS_SPLUNK_META_SSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Controls whether the pump client verifies the Splunk server's certificate chain and host name. + +### pumps.splunk.meta.ssl_cert_file +ENV: <b>TYK_PMP_PUMPS_SPLUNK_META_SSLCERTFILE</b><br /> +Type: `string`<br /> + +SSL cert file location. + +### pumps.splunk.meta.ssl_key_file +ENV: <b>TYK_PMP_PUMPS_SPLUNK_META_SSLKEYFILE</b><br /> +Type: `string`<br /> + +SSL cert key location. + +### pumps.splunk.meta.ssl_server_name +ENV: <b>TYK_PMP_PUMPS_SPLUNK_META_SSLSERVERNAME</b><br /> +Type: `string`<br /> + +SSL Server name used in the TLS connection. + +### pumps.splunk.meta.obfuscate_api_keys +ENV: <b>TYK_PMP_PUMPS_SPLUNK_META_OBFUSCATEAPIKEYS</b><br /> +Type: `bool`<br /> + +Controls whether the pump client should hide the API key. In case you still need substring +of the value, check the next option. Default value is `false`. + +### pumps.splunk.meta.obfuscate_api_keys_length +ENV: <b>TYK_PMP_PUMPS_SPLUNK_META_OBFUSCATEAPIKEYSLENGTH</b><br /> +Type: `int`<br /> + +Define the number of the characters from the end of the API key. The `obfuscate_api_keys` +should be set to `true`. Default value is `0`. + +### pumps.splunk.meta.fields +ENV: <b>TYK_PMP_PUMPS_SPLUNK_META_FIELDS</b><br /> +Type: `[]string`<br /> + +Define which Analytics fields should participate in the Splunk event. Check the available +fields in the example below. Default value is `["method", +"path", "response_code", "api_key", "time_stamp", "api_version", "api_name", "api_id", +"org_id", "oauth_id", "raw_request", "request_time", "raw_response", "ip_address"]`. + +### pumps.splunk.meta.ignore_tag_prefix_list +ENV: <b>TYK_PMP_PUMPS_SPLUNK_META_IGNORETAGPREFIXLIST</b><br /> +Type: `[]string`<br /> + +Choose which tags to be ignored by the Splunk Pump. Keep in mind that the tag name and value +are hyphenated. Default value is `[]`. + +### pumps.splunk.meta.enable_batch +ENV: <b>TYK_PMP_PUMPS_SPLUNK_META_ENABLEBATCH</b><br /> +Type: `bool`<br /> + +If this is set to `true`, pump is going to send the analytics records in batch to Splunk. +Default value is `false`. + +### pumps.splunk.meta.max_retries +ENV: <b>TYK_PMP_PUMPS_SPLUNK_META_MAXRETRIES</b><br /> +Type: `uint64`<br /> + +MaxRetries represents the maximum amount of retries to attempt if failed to send requests to splunk HEC. +Default value is `0` + +### pumps.sql.name +ENV: <b>TYK_PMP_PUMPS_SQL_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.sql.type +ENV: <b>TYK_PMP_PUMPS_SQL_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.sql.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.sql.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_SQL_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.sql.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_SQL_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.sql.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_SQL_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.sql.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_SQL_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.sql.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_SQL_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.sql.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_SQL_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.sql.timeout +ENV: <b>TYK_PMP_PUMPS_SQL_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.sql.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_SQL_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.sql.max_record_size +ENV: <b>TYK_PMP_PUMPS_SQL_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.sql.ignore_fields +ENV: <b>TYK_PMP_PUMPS_SQL_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.sql.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_SQL_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +The prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_SQL_META` + +### pumps.sql.meta.type +ENV: <b>TYK_PMP_PUMPS_SQL_META_TYPE</b><br /> +Type: `string`<br /> + +The supported and tested types are `sqlite` and `postgres`. + +### pumps.sql.meta.connection_string +ENV: <b>TYK_PMP_PUMPS_SQL_META_CONNECTIONSTRING</b><br /> +Type: `string`<br /> + +Specifies the connection string to the database. + +### pumps.sql.meta.postgres +Postgres configurations. + +### pumps.sql.meta.postgres.prefer_simple_protocol +ENV: <b>TYK_PMP_PUMPS_SQL_META_POSTGRES_PREFERSIMPLEPROTOCOL</b><br /> +Type: `bool`<br /> + +Disables implicit prepared statement usage. + +### pumps.sql.meta.mysql +Mysql configurations. + +### pumps.sql.meta.mysql.default_string_size +ENV: <b>TYK_PMP_PUMPS_SQL_META_MYSQL_DEFAULTSTRINGSIZE</b><br /> +Type: `uint`<br /> + +Default size for string fields. Defaults to `256`. + +### pumps.sql.meta.mysql.disable_datetime_precision +ENV: <b>TYK_PMP_PUMPS_SQL_META_MYSQL_DISABLEDATETIMEPRECISION</b><br /> +Type: `bool`<br /> + +Disable datetime precision, which not supported before MySQL 5.6. + +### pumps.sql.meta.mysql.dont_support_rename_index +ENV: <b>TYK_PMP_PUMPS_SQL_META_MYSQL_DONTSUPPORTRENAMEINDEX</b><br /> +Type: `bool`<br /> + +Drop & create when rename index, rename index not supported before MySQL 5.7, MariaDB. + +### pumps.sql.meta.mysql.dont_support_rename_column +ENV: <b>TYK_PMP_PUMPS_SQL_META_MYSQL_DONTSUPPORTRENAMECOLUMN</b><br /> +Type: `bool`<br /> + +`change` when rename column, rename column not supported before MySQL 8, MariaDB. + +### pumps.sql.meta.mysql.skip_initialize_with_version +ENV: <b>TYK_PMP_PUMPS_SQL_META_MYSQL_SKIPINITIALIZEWITHVERSION</b><br /> +Type: `bool`<br /> + +Auto configure based on currently MySQL version. + +### pumps.sql.meta.table_sharding +ENV: <b>TYK_PMP_PUMPS_SQL_META_TABLESHARDING</b><br /> +Type: `bool`<br /> + +Specifies if all the analytics records are going to be stored in one table or in multiple +tables (one per day). By default, `false`. If `false`, all the records are going to be +stored in `tyk_aggregated` table. Instead, if it's `true`, all the records of the day are +going to be stored in `tyk_aggregated_YYYYMMDD` table, where `YYYYMMDD` is going to change +depending on the date. + +### pumps.sql.meta.log_level +ENV: <b>TYK_PMP_PUMPS_SQL_META_LOGLEVEL</b><br /> +Type: `string`<br /> + +Specifies the SQL log verbosity. The possible values are: `info`,`error` and `warning`. By +default, the value is `silent`, which means that it won't log any SQL query. + +### pumps.sql.meta.batch_size +ENV: <b>TYK_PMP_PUMPS_SQL_META_BATCHSIZE</b><br /> +Type: `int`<br /> + +Specifies the amount of records that are going to be written each batch. Type int. By +default, it writes 1000 records max per batch. + +### pumps.sqlaggregate.name +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.sqlaggregate.type +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.sqlaggregate.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.sqlaggregate.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.sqlaggregate.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.sqlaggregate.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.sqlaggregate.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.sqlaggregate.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.sqlaggregate.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.sqlaggregate.timeout +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.sqlaggregate.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.sqlaggregate.max_record_size +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.sqlaggregate.ignore_fields +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.sqlaggregate.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +The prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_SQLAGGREGATE_META` + +### pumps.sqlaggregate.meta.type +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_META_TYPE</b><br /> +Type: `string`<br /> + +The supported and tested types are `sqlite` and `postgres`. + +### pumps.sqlaggregate.meta.connection_string +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_META_CONNECTIONSTRING</b><br /> +Type: `string`<br /> + +Specifies the connection string to the database. + +### pumps.sqlaggregate.meta.postgres +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_META_POSTGRES</b><br /> +Type: `PostgresConfig`<br /> + +Postgres configurations. + +### pumps.sqlaggregate.meta.postgres.prefer_simple_protocol +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_META_POSTGRES_PREFERSIMPLEPROTOCOL</b><br /> +Type: `bool`<br /> + +Disables implicit prepared statement usage. + +### pumps.sqlaggregate.meta.mysql +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_META_MYSQL</b><br /> +Type: `MysqlConfig`<br /> + +Mysql configurations. + +### pumps.sqlaggregate.meta.mysql.default_string_size +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_META_MYSQL_DEFAULTSTRINGSIZE</b><br /> +Type: `uint`<br /> + +Default size for string fields. Defaults to `256`. + +### pumps.sqlaggregate.meta.mysql.disable_datetime_precision +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_META_MYSQL_DISABLEDATETIMEPRECISION</b><br /> +Type: `bool`<br /> + +Disable datetime precision, which not supported before MySQL 5.6. + +### pumps.sqlaggregate.meta.mysql.dont_support_rename_index +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_META_MYSQL_DONTSUPPORTRENAMEINDEX</b><br /> +Type: `bool`<br /> + +Drop & create when rename index, rename index not supported before MySQL 5.7, MariaDB. + +### pumps.sqlaggregate.meta.mysql.dont_support_rename_column +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_META_MYSQL_DONTSUPPORTRENAMECOLUMN</b><br /> +Type: `bool`<br /> + +`change` when rename column, rename column not supported before MySQL 8, MariaDB. + +### pumps.sqlaggregate.meta.mysql.skip_initialize_with_version +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_META_MYSQL_SKIPINITIALIZEWITHVERSION</b><br /> +Type: `bool`<br /> + +Auto configure based on currently MySQL version. + +### pumps.sqlaggregate.meta.table_sharding +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_META_TABLESHARDING</b><br /> +Type: `bool`<br /> + +Specifies if all the analytics records are going to be stored in one table or in multiple +tables (one per day). By default, `false`. If `false`, all the records are going to be +stored in `tyk_aggregated` table. Instead, if it's `true`, all the records of the day are +going to be stored in `tyk_aggregated_YYYYMMDD` table, where `YYYYMMDD` is going to change +depending on the date. + +### pumps.sqlaggregate.meta.log_level +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_META_LOGLEVEL</b><br /> +Type: `string`<br /> + +Specifies the SQL log verbosity. The possible values are: `info`,`error` and `warning`. By +default, the value is `silent`, which means that it won't log any SQL query. + +### pumps.sqlaggregate.meta.batch_size +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_META_BATCHSIZE</b><br /> +Type: `int`<br /> + +Specifies the amount of records that are going to be written each batch. Type int. By +default, it writes 1000 records max per batch. + +### pumps.sqlaggregate.meta.track_all_paths +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_META_TRACKALLPATHS</b><br /> +Type: `bool`<br /> + +Specifies if it should store aggregated data for all the endpoints. By default, `false` +which means that only store aggregated data for `tracked endpoints`. + +### pumps.sqlaggregate.meta.ignore_tag_prefix_list +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_META_IGNORETAGPREFIXLIST</b><br /> +Type: `[]string`<br /> + +Specifies prefixes of tags that should be ignored. + +### pumps.sqlaggregate.meta.store_analytics_per_minute +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_META_STOREANALYTICSPERMINUTE</b><br /> +Type: `bool`<br /> + +Determines if the aggregations should be made per minute instead of per hour. + +### pumps.sqlaggregate.meta.omit_index_creation +ENV: <b>TYK_PMP_PUMPS_SQLAGGREGATE_META_OMITINDEXCREATION</b><br /> +Type: `bool`<br /> + +Set to true to disable the default tyk index creation. + +### pumps.statsd.name +ENV: <b>TYK_PMP_PUMPS_STATSD_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.statsd.type +ENV: <b>TYK_PMP_PUMPS_STATSD_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.statsd.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.statsd.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_STATSD_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.statsd.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_STATSD_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.statsd.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_STATSD_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.statsd.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_STATSD_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.statsd.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_STATSD_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.statsd.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_STATSD_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.statsd.timeout +ENV: <b>TYK_PMP_PUMPS_STATSD_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.statsd.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_STATSD_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.statsd.max_record_size +ENV: <b>TYK_PMP_PUMPS_STATSD_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.statsd.ignore_fields +ENV: <b>TYK_PMP_PUMPS_STATSD_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.statsd.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_STATSD_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +The prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_STATSD_META` + +### pumps.statsd.meta.address +ENV: <b>TYK_PMP_PUMPS_STATSD_META_ADDRESS</b><br /> +Type: `string`<br /> + +Address of statsd including host & port. + +### pumps.statsd.meta.fields +ENV: <b>TYK_PMP_PUMPS_STATSD_META_FIELDS</b><br /> +Type: `[]string`<br /> + +Define which Analytics fields should have its own metric calculation. + +### pumps.statsd.meta.tags +ENV: <b>TYK_PMP_PUMPS_STATSD_META_TAGS</b><br /> +Type: `[]string`<br /> + +List of tags to be added to the metric. + +### pumps.statsd.meta.separated_method +ENV: <b>TYK_PMP_PUMPS_STATSD_META_SEPARATEDMETHOD</b><br /> +Type: `bool`<br /> + +Allows to have a separated method field instead of having it embedded in the path field. + +### pumps.stdout.name +ENV: <b>TYK_PMP_PUMPS_STDOUT_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.stdout.type +ENV: <b>TYK_PMP_PUMPS_STDOUT_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.stdout.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.stdout.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_STDOUT_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.stdout.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_STDOUT_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.stdout.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_STDOUT_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.stdout.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_STDOUT_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.stdout.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_STDOUT_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.stdout.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_STDOUT_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.stdout.timeout +ENV: <b>TYK_PMP_PUMPS_STDOUT_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.stdout.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_STDOUT_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.stdout.max_record_size +ENV: <b>TYK_PMP_PUMPS_STDOUT_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.stdout.ignore_fields +ENV: <b>TYK_PMP_PUMPS_STDOUT_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.stdout.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_STDOUT_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +The prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_STDOUT_META` + +### pumps.stdout.meta.format +ENV: <b>TYK_PMP_PUMPS_STDOUT_META_FORMAT</b><br /> +Type: `string`<br /> + +Format of the analytics logs. Default is `text` if `json` is not explicitly specified. When +JSON logging is used all pump logs to stdout will be JSON. + +### pumps.stdout.meta.log_field_name +ENV: <b>TYK_PMP_PUMPS_STDOUT_META_LOGFIELDNAME</b><br /> +Type: `string`<br /> + +Root name of the JSON object the analytics record is nested in. + +### pumps.syslog.name +ENV: <b>TYK_PMP_PUMPS_SYSLOG_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.syslog.type +ENV: <b>TYK_PMP_PUMPS_SYSLOG_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.syslog.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.syslog.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_SYSLOG_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.syslog.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_SYSLOG_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.syslog.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_SYSLOG_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.syslog.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_SYSLOG_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.syslog.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_SYSLOG_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.syslog.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_SYSLOG_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.syslog.timeout +ENV: <b>TYK_PMP_PUMPS_SYSLOG_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.syslog.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_SYSLOG_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.syslog.max_record_size +ENV: <b>TYK_PMP_PUMPS_SYSLOG_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.syslog.ignore_fields +ENV: <b>TYK_PMP_PUMPS_SYSLOG_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.syslog.meta.meta_env_prefix +ENV: <b>TYK_PMP_PUMPS_SYSLOG_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +The prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_SYSLOG_META` + +### pumps.syslog.meta.transport +ENV: <b>TYK_PMP_PUMPS_SYSLOG_META_TRANSPORT</b><br /> +Type: `string`<br /> + +Possible values are `udp, tcp, tls` in string form. + +### pumps.syslog.meta.network_addr +ENV: <b>TYK_PMP_PUMPS_SYSLOG_META_NETWORKADDR</b><br /> +Type: `string`<br /> + +Host & Port combination of your syslog daemon ie: `"localhost:5140"`. + +### pumps.syslog.meta.log_level +ENV: <b>TYK_PMP_PUMPS_SYSLOG_META_LOGLEVEL</b><br /> +Type: `int`<br /> + +The severity level, an integer from 0-7, based off the Standard: +[Syslog Severity Levels](https://en.wikipedia.org/wiki/Syslog#Severity_level). + +### pumps.syslog.meta.tag +ENV: <b>TYK_PMP_PUMPS_SYSLOG_META_TAG</b><br /> +Type: `string`<br /> + +Prefix tag + +When working with FluentD, you should provide a +[FluentD Parser](https://docs.fluentd.org/input/syslog) based on the OS you are using so +that FluentD can correctly read the logs. + +```{.json} +"syslog": { + "name": "syslog", + "meta": { + "transport": "udp", + "network_addr": "localhost:5140", + "log_level": 6, + "tag": "syslog-pump" + } +``` expandable + +### pumps.timestream.name +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_NAME</b><br /> +Type: `string`<br /> + +The name of the pump. This is used to identify the pump in the logs. +Deprecated, use `type` instead. + +### pumps.timestream.type +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_TYPE</b><br /> +Type: `string`<br /> + +Sets the pump type. This is needed when the pump key does not equal to the pump name type. +Current valid types are: `mongo`, `mongo-pump-selective`, `mongo-pump-aggregate`, `csv`, +`elasticsearch`, `influx`, `influx2`, `moesif`, `statsd`, `segment`, `graylog`, `splunk`, `hybrid`, `prometheus`, +`logzio`, `dogstatsd`, `kafka`, `syslog`, `sql`, `sql_aggregate`, `stdout`, `timestream`, `mongo-graph`, +`sql-graph`, `sql-graph-aggregate`, `resurfaceio`. + +### pumps.timestream.filters +This feature adds a new configuration field in each pump called filters and its structure is +the following: +```{.json} +"filters":{ + "api_ids":[], + "org_ids":[], + "response_codes":[], + "skip_api_ids":[], + "skip_org_ids":[], + "skip_response_codes":[] +} +``` +The fields api_ids, org_ids and response_codes works as allow list (APIs and orgs where we +want to send the analytics records) and the fields skip_api_ids, skip_org_ids and +skip_response_codes works as block list. + +The priority is always block list configurations over allow list. + +An example of configuration would be: +```{.json} +"csv": { + "type": "csv", + "filters": { + "org_ids": ["org1","org2"] + }, + "meta": { + "csv_dir": "./bar" + } +} +``` expandable + +### pumps.timestream.filters.org_ids +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_FILTERS_ORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of org_ids. + +### pumps.timestream.filters.api_ids +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_FILTERS_APIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by an allow list of api_ids. + +### pumps.timestream.filters.response_codes +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_FILTERS_RESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by an allow list of response_codes. + +### pumps.timestream.filters.skip_org_ids +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_FILTERS_SKIPPEDORGSIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of org_ids. + +### pumps.timestream.filters.skip_api_ids +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_FILTERS_SKIPPEDAPIIDS</b><br /> +Type: `[]string`<br /> + +Filters pump data by a block list of api_ids. + +### pumps.timestream.filters.skip_response_codes +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_FILTERS_SKIPPEDRESPONSECODES</b><br /> +Type: `[]int`<br /> + +Filters pump data by a block list of response_codes. + +### pumps.timestream.timeout +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_TIMEOUT</b><br /> +Type: `int`<br /> + +By default, a pump will wait forever for each write operation to complete; you can configure an optional timeout by setting the configuration option `timeout`. +If you have deployed multiple pumps, then you can configure each timeout independently. The timeout is in seconds and defaults to 0. + +The timeout is configured within the main pump config as shown here; note that this example would configure a 5 second timeout: +```{.json} +"pump_name": { + ... + "timeout":5, + "meta": {...} +} +``` expandable + +Tyk will inform you if the pump's write operation is taking longer than the purging loop (configured via `purge_delay`) as this will mean that data is purged before being written to the target data sink. + +If there is no timeout configured and pump's write operation is taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try to set a timeout for this pump.` + +If there is a timeout configured, but pump's write operation is still taking longer than the purging loop, the following warning log will be generated: +`Pump {pump_name} is taking more time than the value configured of purge_delay. You should try lowering the timeout configured for this pump.`. + +### pumps.timestream.omit_detailed_recording +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to `false`. + +### pumps.timestream.max_record_size +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` expandable + +### pumps.timestream.ignore_fields +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_IGNOREFIELDS</b><br /> +Type: `[]string`<br /> + +IgnoreFields defines a list of analytics fields that will be ignored when writing to the pump. +This can be used to avoid writing sensitive information to the Database, or data that you don't really need to have. +The field names must be the same as the JSON tags of the analytics record fields. +For example: `["api_key", "api_version"]`. + +### pumps.timestream.meta.EnvPrefix +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_META_ENVPREFIX</b><br /> +Type: `string`<br /> + +The prefix for the environment variables that will be used to override the configuration. +Defaults to `TYK_PMP_PUMPS_TIMESTREAM_META` + +### pumps.timestream.meta.AWSRegion +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_META_AWSREGION</b><br /> +Type: `string`<br /> + +The aws region that contains the timestream database + +### pumps.timestream.meta.TableName +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_META_TABLENAME</b><br /> +Type: `string`<br /> + +The table name where the data is going to be written + +### pumps.timestream.meta.DatabaseName +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_META_DATABASENAME</b><br /> +Type: `string`<br /> + +The timestream database name that contains the table being written to + +### pumps.timestream.meta.Dimensions +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_META_DIMENSIONS</b><br /> +Type: `[]string`<br /> + +A filter of all the dimensions that will be written to the table. The possible options are +["Method","Host","Path","RawPath","APIKey","APIVersion","APIName","APIID","OrgID","OauthID"] + +### pumps.timestream.meta.Measures +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_META_MEASURES</b><br /> +Type: `[]string`<br /> + +A filter of all the measures that will be written to the table. The possible options are +["ContentLength","ResponseCode","RequestTime","NetworkStats.OpenConnections", +"NetworkStats.ClosedConnection","NetworkStats.BytesIn","NetworkStats.BytesOut", +"Latency.Total","Latency.Upstream","GeoData.City.GeoNameID","IPAddress", +"GeoData.Location.Latitude","GeoData.Location.Longitude","UserAgent","RawRequest","RawResponse", +"RateLimit.Limit","Ratelimit.Remaining","Ratelimit.Reset", +"GeoData.Country.ISOCode","GeoData.City.Names","GeoData.Location.TimeZone"] + +### pumps.timestream.meta.WriteRateLimit +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_META_WRITERATELIMIT</b><br /> +Type: `bool`<br /> + +Set to true in order to save any of the `RateLimit` measures. Default value is `false`. + +### pumps.timestream.meta.ReadGeoFromRequest +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_META_READGEOFROMREQUEST</b><br /> +Type: `bool`<br /> + +If set true, we will try to read geo information from the headers if +values aren't found on the analytic record . Default value is `false`. + +### pumps.timestream.meta.WriteZeroValues +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_META_WRITEZEROVALUES</b><br /> +Type: `bool`<br /> + +Set to true, in order to save numerical values with value zero. Default value is `false`. + +### pumps.timestream.meta.NameMappings +ENV: <b>TYK_PMP_PUMPS_TIMESTREAM_META_NAMEMAPPINGS</b><br /> +Type: `map[string]string`<br /> + +A name mapping for both Dimensions and Measures names. It's not required + +### analytics_storage_type +ENV: <b>TYK_PMP_ANALYTICSSTORAGETYPE</b><br /> +Type: `string`<br /> + +Sets the analytics storage type. Where the pump will be fetching data from. Currently, only +the `redis` option is supported. + +### analytics_storage_config +Example Temporal storage configuration: +```{.json} + "analytics_storage_config": { + "type": "redis", + "host": "localhost", + "port": 6379, + "hosts": null, + "username": "", + "password": "", + "database": 0, + "optimisation_max_idle": 100, + "optimisation_max_active": 0, + "enable_cluster": false, + "use_ssl": false, + "ssl_insecure_skip_verify": false + }, +``` expandable + +### analytics_storage_config.type +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_TYPE</b><br /> +Type: `string`<br /> + +Deprecated. + +### analytics_storage_config.host +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_HOST</b><br /> +Type: `string`<br /> + +Host value. For example: "localhost". + +### analytics_storage_config.port +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_PORT</b><br /> +Type: `int`<br /> + +Port value. For example: 6379. + +### analytics_storage_config.hosts +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_HOSTS</b><br /> +Type: `map[string]string`<br /> + +Deprecated: use Addrs instead. + +### analytics_storage_config.addrs +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_ADDRS</b><br /> +Type: `[]string`<br /> + +Use instead of the host value if you're running a Redis cluster with multiple instances. + +### analytics_storage_config.master_name +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_MASTERNAME</b><br /> +Type: `string`<br /> + +Sentinel master name. + +### analytics_storage_config.sentinel_password +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_SENTINELPASSWORD</b><br /> +Type: `string`<br /> + +Sentinel password. + +### analytics_storage_config.username +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_USERNAME</b><br /> +Type: `string`<br /> + +Database username. + +### analytics_storage_config.password +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_PASSWORD</b><br /> +Type: `string`<br /> + +Database password. + +### analytics_storage_config.database +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_DATABASE</b><br /> +Type: `int`<br /> + +Database name. + +### analytics_storage_config.timeout +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_TIMEOUT</b><br /> +Type: `int`<br /> + +How long to allow for new connections to be established (in milliseconds). Defaults to 5sec. + +### analytics_storage_config.optimisation_max_idle +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_MAXIDLE</b><br /> +Type: `int`<br /> + +Maximum number of idle connections in the pool. + +### analytics_storage_config.optimisation_max_active +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_MAXACTIVE</b><br /> +Type: `int`<br /> + +Maximum number of connections allocated by the pool at a given time. When zero, there is no +limit on the number of connections in the pool. Defaults to 500. + +### analytics_storage_config.enable_cluster +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_ENABLECLUSTER</b><br /> +Type: `bool`<br /> + +Enable this option if you are using a cluster instance. Default is `false`. + +### analytics_storage_config.redis_key_prefix +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_REDISKEYPREFIX</b><br /> +Type: `string`<br /> + +Prefix the key names. Defaults to "analytics-". +Deprecated: use KeyPrefix instead. + +### analytics_storage_config.key_prefix +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_KEYPREFIX</b><br /> +Type: `string`<br /> + +Prefix the key names. Defaults to "analytics-". + +### analytics_storage_config.use_ssl +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_USESSL</b><br /> +Type: `bool`<br /> + +Setting this to true to use SSL when connecting to the DB. + +### analytics_storage_config.ssl_insecure_skip_verify +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_SSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Set this to `true` to tell Pump to ignore database's cert validation. + +### analytics_storage_config.ssl_ca_file +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_SSLCAFILE</b><br /> +Type: `string`<br /> + +Path to the CA file. + +### analytics_storage_config.ssl_cert_file +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_SSLCERTFILE</b><br /> +Type: `string`<br /> + +Path to the cert file. + +### analytics_storage_config.ssl_key_file +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_SSLKEYFILE</b><br /> +Type: `string`<br /> + +Path to the key file. + +### analytics_storage_config.ssl_max_version +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_SSLMAXVERSION</b><br /> +Type: `string`<br /> + +Maximum supported TLS version. Defaults to TLS 1.3, valid values are TLS 1.0, 1.1, 1.2, 1.3. + +### analytics_storage_config.ssl_min_version +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_SSLMINVERSION</b><br /> +Type: `string`<br /> + +Minimum supported TLS version. Defaults to TLS 1.2, valid values are TLS 1.0, 1.1, 1.2, 1.3. + +### analytics_storage_config.redis_use_ssl +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_REDISUSESSL</b><br /> +Type: `bool`<br /> + +Setting this to true to use SSL when connecting to the DB. +Deprecated: use UseSSL instead. + +### analytics_storage_config.redis_ssl_insecure_skip_verify +ENV: <b>TYK_PMP_ANALYTICSSTORAGECONFIG_REDISSSLINSECURESKIPVERIFY</b><br /> +Type: `bool`<br /> + +Set this to `true` to tell Pump to ignore database's cert validation. +Deprecated: use SSLInsecureSkipVerify instead. + +### statsd_connection_string +ENV: <b>TYK_PMP_STATSDCONNECTIONSTRING</b><br /> +Type: `string`<br /> + +Connection string for StatsD monitoring for information please see the +[Instrumentation docs](https://tyk.io/docs/api-management/logs-metrics/#statsd-instrumentation). + +### statsd_prefix +ENV: <b>TYK_PMP_STATSDPREFIX</b><br /> +Type: `string`<br /> + +Custom prefix value. For example separate settings for production and staging. + +### log_level +ENV: <b>TYK_PMP_LOGLEVEL</b><br /> +Type: `string`<br /> + +Set the logger details for tyk-pump. The posible values are: `info`,`debug`,`error` and +`warn`. By default, the log level is `info`. + +### log_format +ENV: <b>TYK_PMP_LOGFORMAT</b><br /> +Type: `string`<br /> + +Set the logger format. The possible values are: `text` and `json`. By default, the log +format is `text`. + +### Health Check +From v2.9.4, we have introduced a `/health` endpoint to confirm the Pump is running. You +need to configure the following settings. This returns a HTTP 200 OK response if the Pump is +running. + +### health_check_endpoint_name +ENV: <b>TYK_PMP_HEALTHCHECKENDPOINTNAME</b><br /> +Type: `string`<br /> + + +The default is "health". + +### health_check_endpoint_port +ENV: <b>TYK_PMP_HEALTHCHECKENDPOINTPORT</b><br /> +Type: `int`<br /> + +The default port is 8083. + +### omit_detailed_recording +ENV: <b>TYK_PMP_OMITDETAILEDRECORDING</b><br /> +Type: `bool`<br /> + +Setting this to true will avoid writing raw_request and raw_response fields for each request +in pumps. Defaults to false. + +### max_record_size +ENV: <b>TYK_PMP_MAXRECORDSIZE</b><br /> +Type: `int`<br /> + +Defines maximum size (in bytes) for Raw Request and Raw Response logs, this value defaults +to 0. If it is not set then tyk-pump will not trim any data and will store the full +information. This can also be set at a pump level. For example: +```{.json} +"csv": { + "type": "csv", + "max_record_size":1000, + "meta": { + "csv_dir": "./" + } +} +``` + +### omit_config_file +ENV: <b>TYK_PMP_OMITCONFIGFILE</b><br /> +Type: `bool`<br /> + +Defines if tyk-pump should ignore all the values in configuration file. Specially useful when setting all configurations in environment variables. + +### enable_http_profiler +ENV: <b>TYK_PMP_HTTPPROFILE</b><br /> +Type: `bool`<br /> + +Enable debugging of Tyk Pump by exposing profiling information, the same as the [gateway](https://tyk.io/docs/api-management/troubleshooting-debugging) + +### raw_request_decoded +ENV: <b>TYK_PMP_DECODERAWREQUEST</b><br /> +Type: `bool`<br /> + +Setting this to true allows the Raw Request to be decoded from base 64 +for all pumps. This is set to false by default. + +### raw_response_decoded +ENV: <b>TYK_PMP_DECODERAWRESPONSE</b><br /> +Type: `bool`<br /> + +Setting this to true allows the Raw Response to be decoded from base 64 for all pumps. This is set to false by default. + diff --git a/snippets/self-managed-licensing-include.mdx b/snippets/self-managed-licensing-include.mdx new file mode 100644 index 00000000..bc2bfc9c --- /dev/null +++ b/snippets/self-managed-licensing-include.mdx @@ -0,0 +1,10 @@ +--- +--- + +Tyk Self-Managed is the easiest way to install Tyk Full Lifecycle API Management solution in your infrastructure. There is no calling home, and there are no usage limits. You have complete control. + +See [licensing](/getting-started/quick-start) and [deployment models](/tyk-self-managed/install) to learn about the different deployment options for Self-Managed. + +Contact us to get started with Tyk Self Managed. + +<ButtonLeft href="https://tyk.io/contact/" color="green" content="Contact us" /> \ No newline at end of file diff --git a/snippets/sql-versions-include.mdx b/snippets/sql-versions-include.mdx new file mode 100644 index 00000000..0953ac78 --- /dev/null +++ b/snippets/sql-versions-include.mdx @@ -0,0 +1,23 @@ +--- +--- + +From Tyk 4.0, you can use PostgreSQL as your datastore. We support the following versions: + +- [PostgreSQL](https://www.postgresql.org) version 12.x, 13.x, 14.x, 15.x, 16.x + +You can also use the following as a drop in replacement for PostgreSQL: + +- [Amazon RDS](https://aws.amazon.com/rds/) +- [Amazon Aurora PostgreSQL](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/Aurora.AuroraPostgreSQL.html) +- [Azure CosmosDB for PostgreSQL](https://learn.microsoft.com/en-us/azure/cosmos-db/postgresql/introduction) + + + + <Note> + In a production environment, we *only* support the PostgreSQL versions listed above. + </Note> + + +For POC, you can also use the following as replacement: + +- SQLite 3.x diff --git a/snippets/tyk-gateway-features-include.mdx b/snippets/tyk-gateway-features-include.mdx new file mode 100644 index 00000000..6b509ecf --- /dev/null +++ b/snippets/tyk-gateway-features-include.mdx @@ -0,0 +1,47 @@ +--- +--- +Tyk is an open source Enterprise API Gateway, supporting REST, GraphQL, TCP and gRPC protocols. + +Tyk Gateway is provided β€˜Batteries-included’, with no feature lockout. Enabling your organization to control who accesses your APIs, when they access, and how they access it. + +Tyk Technologies uses the same API Gateway for all its applications. Protecting, securing, and processing APIs for thousands of organizations and businesses around the world. Ideal for Open Banking, building software in the clouds as well as exposing APIs to teams, partners & consumers. + +Built from the ground up to be the fastest API gateway on the planet. It does not depend on a legacy proxy underneath. It has no 3rd party dependencies aside from Redis for distributed rate-limiting and token storage. Tyk Gateway can also be deployed as part of a larger Full Lifecycle API Management platform Tyk Self-Managed which also includes Management Control Plane, Dashboard GUI and Developer Portal. + +<img src="/img/diagrams/gateway4.png" alt="Tyk Open Source Gateway features" /> + +## Open Source API Gateway Features + +**Use any protocol:** REST, SOAP, GraphQL, gRPC, and TCP. + +**Industry Standard Authentication**: OIDC, JWT, bearer Tokens, Basic Auth, Client Certificates and more. + +**OpenAPI Standards**: Keep your OpenAPI 3.0 description as source of truth with [Tyk OAS APIs](/api-management/gateway-config-managing-oas#), import your Swagger and OAS2/3 documents to scaffold Tyk Classic APIs. + +**Ultra performant:** Low latency, and thousands of rps with just a single CPU, horizontally and vertically scalable. + +**Content mediation**: Transform all the things, from request or response headers to converting between SOAP and GraphQL. + +**Extensible Plugin Architecture**: Customize Tyk’s middleware chain by writing plugins in your language of choice - from Javascript to Go, or any language which supports gRPC. + +**Rate Limiting & Quotas:** Protect your upstreams from becoming overloaded and/or apply limits for each consumer. + +**API Versioning** - API Versions can be easily set and deprecated at a specific time and date. + +**Granular Access Control** - Grant access to one or more APIs on a per version and operation basis. + +**Blocklist/Allowlist/Ignored endpoint access** - Enforce strict security models on a version-by-version basis to your access points. + +**Analytics logging** - Record detailed usage data on who is using your API's (raw data only) + +**CORS** - Enable [CORS](/api-management/security-features#cross-origin-resource-sharing-cors) for certain APIs so users can make browser-based requests + +**Webhooks** - Trigger webhooks against events such as Quota Violations and Authentication failures + +**IP AllowListing** - Block access to non-trusted IP addresses for more secure interactions + +**Hitless reloads** - Tyk configurations can be altered dynamically and the service restarted without affecting any active request + +**Kubernetes native declarative API:** using Open Source [Tyk Operator](https://github.com/TykTechnologies/tyk-operator) (more info in OSS section) + +_...and everything else you expect from a Cloud Native API Gateway_ \ No newline at end of file diff --git a/snippets/x-tyk-gateway.mdx b/snippets/x-tyk-gateway.mdx new file mode 100644 index 00000000..2e2c40f5 --- /dev/null +++ b/snippets/x-tyk-gateway.mdx @@ -0,0 +1,3019 @@ +## Tyk vendor extension reference + +XTykAPIGateway contains custom Tyk API extensions for the OpenAPI definition. +The values for the extensions are stored inside the OpenAPI document, under +the key `x-tyk-api-gateway`. + +**Field: `info` ([Info](#info))** +Info contains the main metadata for the API definition. + +**Field: `upstream` ([Upstream](#upstream))** +Upstream contains the configurations related to the upstream. + +**Field: `server` ([Server](#server))** +Server contains the configurations related to the server. + +**Field: `middleware` ([Middleware](#middleware))** +Middleware contains the configurations related to the Tyk middleware. + +### **Info** + +Info contains the main metadata for the API definition. + +**Field: `id` (`string`)** +ID is the unique identifier of the API within Tyk. + +Tyk classic API definition: `api_id`. + +**Field: `dbId` (`string`)** +DBID is the unique identifier of the API within the Tyk database. + +Tyk classic API definition: `id`. + +**Field: `orgId` (`string`)** +OrgID is the ID of the organisation which the API belongs to. + +Tyk classic API definition: `org_id`. + +**Field: `name` (`string`)** +Name is the name of the API. + +Tyk classic API definition: `name`. + +**Field: `expiration` (`string`)** +Expiration date. + +Tyk classic API definition: `expiration`. + +**Field: `state` ([State](#state))** +State holds configuration for API definition states (active, internal). + +**Field: `versioning` ([Versioning](#versioning))** +Versioning holds configuration for API versioning. + +### **Upstream** + +Upstream holds configuration for the upstream server to which Tyk should proxy requests. + +**Field: `url` (`string`)** +URL defines the upstream address (or target URL) to which requests should be proxied. + +Tyk classic API definition: `proxy.target_url`. + +**Field: `serviceDiscovery` ([ServiceDiscovery](#servicediscovery))** +ServiceDiscovery contains the configuration related to Service Discovery. + +Tyk classic API definition: `proxy.service_discovery`. + +**Field: `uptimeTests` ([UptimeTests](#uptimetests))** +UptimeTests contains the configuration related to uptime tests. + +Tyk classic API definition: `uptime_tests` and `check_host_against_uptime_tests`. + +**Field: `mutualTLS` ([MutualTLS](#mutualtls))** +MutualTLS contains the configuration for establishing a mutual TLS connection between Tyk and the upstream server. + +Tyk classic API definition: `upstream_certificates_disabled` and `upstream_certificates`. + +**Field: `certificatePinning` ([CertificatePinning](#certificatepinning))** +CertificatePinning contains the configuration related to certificate pinning. + +Tyk classic API definition: `certificate_pinning_disabled` and `pinned_public_keys`. + +**Field: `rateLimit` ([RateLimit](#ratelimit))** +RateLimit contains the configuration related to API level rate limit. + +Tyk classic API definition: `global_rate_limit`. + +**Field: `authentication` ([UpstreamAuth](#upstreamauth))** +Authentication contains the configuration related to upstream authentication. + +Tyk classic API definition: `upstream_auth`. + +**Field: `loadBalancing` ([LoadBalancing](#loadbalancing))** +LoadBalancing contains configuration for load balancing between multiple upstream targets. + +Tyk classic API definition: `proxy.enable_load_balancing` and `proxy.targets`. + +**Field: `preserveHostHeader` ([PreserveHostHeader](#preservehostheader))** +PreserveHostHeader contains the configuration for preserving the host header. + +Tyk classic API definition: `proxy.preserve_host_header`. + +**Field: `preserveTrailingSlash` ([PreserveTrailingSlash](#preservetrailingslash))** +PreserveTrailingSlash controls whether Tyk preserves trailing slashes in URLs when proxying +requests to upstream services. When enabled, URLs like "/users/" will retain the trailing slash. + +Tyk classic API definition: `proxy.disable_strip_slash`. + +**Field: `tlsTransport` ([TLSTransport](#tlstransport))** +TLSTransport contains the configuration for TLS transport settings. + +Tyk classic API definition: `proxy.transport`. + +**Field: `proxy` ([Proxy](#proxy))** +Proxy contains the configuration for an internal proxy. + +Tyk classic API definition: `proxy.proxy_url`. + +### **Server** + +Server contains the configuration that sets Tyk up to receive requests from the client applications. + +**Field: `listenPath` ([ListenPath](#listenpath))** +ListenPath is the base path on Tyk to which requests for this API should +be sent. Tyk listens for any requests coming into the host at this +path, on the port that Tyk is configured to run on and processes these +accordingly. + +**Field: `authentication` ([Authentication](#authentication))** +Authentication contains the configurations that manage how clients can authenticate with Tyk to access the API. + +**Field: `clientCertificates` ([ClientCertificates](#clientcertificates))** +ClientCertificates contains the configurations related to establishing static mutual TLS between the client and Tyk. + +**Field: `gatewayTags` ([GatewayTags](#gatewaytags))** +GatewayTags contain segment tags to indicate which Gateways your upstream service is connected to (and hence where to deploy the API). + +**Field: `customDomain` ([Domain](#domain))** +CustomDomain is the domain to bind this API to. This enforces domain matching for client requests. + + +Tyk classic API definition: `domain`. + +**Field: `detailedActivityLogs` ([DetailedActivityLogs](#detailedactivitylogs))** +DetailedActivityLogs configures detailed analytics recording. + +**Field: `detailedTracing` ([DetailedTracing](#detailedtracing))** +DetailedTracing enables OpenTelemetry's detailed tracing for this API. + + +Tyk classic API definition: `detailed_tracing`. + +**Field: `eventHandlers` ([EventHandlers](#eventhandlers))** +EventHandlers contains the configuration related to Tyk Events. + + +Tyk classic API definition: `event_handlers`. + +**Field: `ipAccessControl` ([IPAccessControl](#ipaccesscontrol))** +IPAccessControl configures IP access control for this API. + + +Tyk classic API definition: `allowed_ips` and `blacklisted_ips`. + +**Field: `batchProcessing` ([BatchProcessing](#batchprocessing))** +BatchProcessing contains configuration settings to enable or disable batch request support for the API. + + +Tyk classic API definition: `enable_batch_request_support`. + +**Field: `protocol` (`string`)** +Protocol configures the HTTP protocol used by the API. +Possible values are: +- "http": Standard HTTP/1.1 protocol +- "http2": HTTP/2 protocol with TLS +- "h2c": HTTP/2 protocol without TLS (cleartext). + + +Tyk classic API definition: `protocol`. + +**Field: `port` (`int`)** +Port Setting this value will change the port that Tyk listens on. Default: 8080. + + +Tyk classic API definition: `listen_port`. + +### **Middleware** + +Middleware holds configuration for Tyk's native middleware. + +**Field: `global` ([Global](#global))** +Global contains configuration for middleware that affects the whole API (all endpoints). + +**Field: `operations` ([Operations](#operations))** +Operations contains configuration for middleware that can be applied to individual endpoints within the API (per-endpoint). + +### **State** + +State holds configuration for the status of the API within Tyk - if it is currently active and if it is exposed externally. + +**Field: `active` (`boolean`)** +Active enables the API so that Tyk will listen for and process requests made to the listenPath. + +Tyk classic API definition: `active`. + +**Field: `internal` (`boolean`)** +Internal makes the API accessible only internally. + +Tyk classic API definition: `internal`. + +### **Versioning** + +Versioning holds configuration for API versioning. + +Tyk classic API definition: `version_data`. + +**Field: `enabled` (`boolean`)** +Enabled is a boolean flag, if set to `true` it will enable versioning of the API. + +**Field: `name` (`string`)** +Name contains the name of the version as entered by the user ("v1" or similar). + +**Field: `default` (`string`)** +Default contains the default version name if a request is issued without a version. + +**Field: `location` (`string`)** +Location contains versioning location information. It can be one of the following: + +- `header`, +- `url-param`, +- `url`. + +**Field: `key` (`string`)** +Key contains the name of the key to check for versioning information. + +**Field: `versions` ([[]VersionToID](#versiontoid))** +Versions contains a list of versions that map to individual API IDs. + +**Field: `stripVersioningData` (`boolean`)** +StripVersioningData is a boolean flag, if set to `true`, the API responses will be stripped of versioning data. + +**Field: `urlVersioningPattern` (`string`)** +UrlVersioningPattern is a string that contains the pattern that if matched will remove the version from the URL. + +**Field: `fallbackToDefault` (`boolean`)** +FallbackToDefault controls the behaviour of Tyk when a versioned API is called with a nonexistent version name. +If set to `true` then the default API version will be invoked; if set to `false` Tyk will return an HTTP 404 +`This API version does not seem to exist` error in this scenario. + +### **ServiceDiscovery** + +ServiceDiscovery holds configuration required for service discovery. + +**Field: `enabled` (`boolean`)** +Enabled activates Service Discovery. + + +Tyk classic API definition: `service_discovery.use_discovery_service`. + +**Field: `queryEndpoint` (`string`)** +QueryEndpoint is the endpoint to call, this would usually be Consul, etcd or Eureka K/V store. + +Tyk classic API definition: `service_discovery.query_endpoint`. + +**Field: `dataPath` (`string`)** +DataPath is the namespace of the data path - where exactly in your service response the namespace can be found. +For example, if your service responds with: + +``` expandable +{ + "action": "get", + "node": { + "key": "/services/single", + "value": "http://httpbin.org:6000", + "modifiedIndex": 6, + "createdIndex": 6 + } +} +``` + +then your namespace would be `node.value`. + + +Tyk classic API definition: `service_discovery.data_path`. + +**Field: `useNestedQuery` (`boolean`)** +UseNestedQuery enables the use of a combination of `dataPath` and `parentDataPath`. +It is necessary when the data lives within this string-encoded JSON object. + +``` expandable +{ + "action": "get", + "node": { + "key": "/services/single", + "value": "{"hostname": "http://httpbin.org", "port": "80"}", + "modifiedIndex": 6, + "createdIndex": 6 + } +} +``` + + +Tyk classic API definition: `service_discovery.use_nested_query`. + +**Field: `parentDataPath` (`string`)** +ParentDataPath is the namespace of the where to find the nested +value if `useNestedQuery` is `true`. In the above example, it +would be `node.value`. You would change the `dataPath` setting +to be `hostname`, since this is where the host name data +resides in the JSON string. Tyk automatically assumes that +`dataPath` in this case is in a string-encoded JSON object and +will try to deserialize it. + + +Tyk classic API definition: `service_discovery.parent_data_path`. + +**Field: `portDataPath` (`string`)** +PortDataPath is the port of the data path. In the above nested example, we can see that there is a separate `port` value +for the service in the nested JSON. In this case, you can set the `portDataPath` value and Tyk will treat `dataPath` as +the hostname and zip them together (this assumes that the hostname element does not end in a slash or resource identifier +such as `/widgets/`). In the above example, the `portDataPath` would be `port`. + + +Tyk classic API definition: `service_discovery.port_data_path`. + +**Field: `useTargetList` (`boolean`)** +UseTargetList should be set to `true` if you are using load balancing. Tyk will treat the data path as a list and +inject it into the target list of your API definition. + + +Tyk classic API definition: `service_discovery.use_target_list`. + +**Field: `cacheTimeout` (`int64`)** +CacheTimeout is the timeout of a cache value when a new data is loaded from a discovery service. +Setting it too low will cause Tyk to call the SD service too often, setting it too high could mean that +failures are not recovered from quickly enough. + +Deprecated: The field is deprecated. Use `service_discovery` to configure service discovery cache options. + + +Tyk classic API definition: `service_discovery.cache_timeout`. + +**Field: `cache` ([ServiceDiscoveryCache](#servicediscoverycache))** +Cache holds cache related flags. + + +Tyk classic API definition:. +- `service_discovery.cache_disabled` +- `service_discovery.cache_timeout` + +**Field: `targetPath` (`string`)** +TargetPath is used to set a target path that will be appended to the +discovered endpoint, since many service discovery services only provide +host and port data. It is important to be able to target a specific +resource on that host. Setting this value will enable that. + + +Tyk classic API definition: `service_discovery.target_path`. + +**Field: `endpointReturnsList` (`boolean`)** +EndpointReturnsList is set `true` when the response type is a list instead of an object. + + +Tyk classic API definition: `service_discovery.endpoint_returns_list`. + +### **UptimeTests** + +UptimeTests configures uptime tests. + +**Field: `enabled` (`boolean`)** +Enabled specifies whether the uptime tests are active or not. + +Tyk classic API definition: `uptime_tests.disabled`. + +**Field: `serviceDiscovery` ([ServiceDiscovery](#servicediscovery))** +ServiceDiscovery contains the configuration related to test Service Discovery. + +Tyk classic API definition: `proxy.service_discovery`. + +**Field: `tests` ([[]UptimeTest](#uptimetest))** +Tests contains individual connectivity tests defined for checking if a service is online. + +**Field: `hostDownRetestPeriod` (`time.ReadableDuration`)** +HostDownRetestPeriod is the time to wait until rechecking a failed test. +If undefined, the default testing interval (10s) is in use. +Setting this to a lower value would result in quicker recovery on failed checks. + +**Field: `logRetentionPeriod` (`time.ReadableDuration`)** +LogRetentionPeriod holds a time to live for the uptime test results. +If unset, a value of 100 years is the default. + +### **MutualTLS** + +MutualTLS contains the configuration for establishing a mutual TLS connection between Tyk and the upstream server. + +**Field: `enabled` (`boolean`)** +Enabled activates upstream mutual TLS for the API. + +Tyk classic API definition: `upstream_certificates_disabled`. + +**Field: `domainToCertificateMapping` ([[]DomainToCertificate](#domaintocertificate))** +DomainToCertificates maintains the mapping of domain to certificate. + +Tyk classic API definition: `upstream_certificates`. + +### **CertificatePinning** + +CertificatePinning holds the configuration about mapping of domains to pinned public keys. + +**Field: `enabled` (`boolean`)** +Enabled is a boolean flag, if set to `true`, it enables certificate pinning for the API. + + +Tyk classic API definition: `certificate_pinning_disabled`. + +**Field: `domainToPublicKeysMapping` ([PinnedPublicKeys](#pinnedpublickeys))** +DomainToPublicKeysMapping maintains the mapping of domain to pinned public keys. + + +Tyk classic API definition: `pinned_public_keys`. + +### **RateLimit** + +RateLimit holds the configurations related to rate limit. +The API-level rate limit applies a base-line limit on the frequency of requests to the upstream service for all endpoints. The frequency of requests is configured in two parts: the time interval and the number of requests that can be made during each interval. +Tyk classic API definition: `global_rate_limit`. + +**Field: `enabled` (`boolean`)** +Enabled activates API level rate limiting for this API. + + +Tyk classic API definition: `!disable_rate_limit`. + +**Field: `rate` (`int`)** +Rate specifies the number of requests that can be passed to the upstream in each time interval (`per`). +This field sets the limit on the frequency of requests to ensure controlled +resource access or to prevent abuse. The rate is defined as an integer value. + +A higher value indicates a higher number of allowed requests in the given +time frame. For instance, if `Per` is set to `1m` (one minute), a Rate of `100` +means up to 100 requests can be made per minute. + + +Tyk classic API definition: `global_rate_limit.rate`. + +**Field: `per` ([ReadableDuration](#readableduration))** +Per defines the time interval for rate limiting using shorthand notation. +The value of Per is a string that specifies the interval in a compact form, +where hours, minutes and seconds are denoted by 'h', 'm' and 's' respectively. +Multiple units can be combined to represent the duration. + +Examples of valid shorthand notations: +- "1h" : one hour +- "20m" : twenty minutes +- "30s" : thirty seconds +- "1m29s": one minute and twenty-nine seconds +- "1h30m" : one hour and thirty minutes + +An empty value is interpreted as "0s", implying no rate limiting interval, which disables the API-level rate limit. +It's important to format the string correctly, as invalid formats will +be considered as 0s/empty. + + +Tyk classic API definition: `global_rate_limit.per`. + +### **UpstreamAuth** + +UpstreamAuth holds the configurations related to upstream API authentication. + +**Field: `enabled` (`boolean`)** +Enabled enables upstream API authentication. + +**Field: `basicAuth` ([UpstreamBasicAuth](#upstreambasicauth))** +BasicAuth holds the basic authentication configuration for upstream API authentication. + +**Field: `oauth` ([UpstreamOAuth](#upstreamoauth))** +OAuth contains the configuration for OAuth2 Client Credentials flow. + +**Field: `requestSigning` ([UpstreamRequestSigning](#upstreamrequestsigning))** +RequestSigning holds the configuration for generating signed requests to an upstream API. + +### **LoadBalancing** + +LoadBalancing represents the configuration for load balancing between multiple upstream targets. + +**Field: `enabled` (`boolean`)** +Enabled determines if load balancing is active. + +**Field: `skipUnavailableHosts` (`boolean`)** +SkipUnavailableHosts determines whether to skip unavailable hosts during load balancing based on uptime tests. +Tyk classic field: `proxy.check_host_against_uptime_tests` + +**Field: `targets` ([[]LoadBalancingTarget](#loadbalancingtarget))** +Targets defines the list of targets with their respective weights for load balancing. + +### **PreserveHostHeader** + +PreserveHostHeader holds the configuration for preserving the host header. + +**Field: `enabled` (`boolean`)** +Enabled activates preserving the host header. + +### **PreserveTrailingSlash** + +PreserveTrailingSlash holds the configuration for preserving the +trailing slash when routed to upstream services. + +The default behaviour of Tyk is to strip any trailing slash (/) from +the target URL when proxying the request upstream. In some use cases the +upstream might expect the trailing slash - or might consider /users/ to +be a different endpoint from /users (for example). + +**Field: `enabled` (`boolean`)** +Enabled activates preserving the trailing slash when routing requests. + +### **TLSTransport** + +TLSTransport contains the configuration for TLS transport settings. +This struct allows you to specify a custom proxy and set the minimum TLS versions and any SSL ciphers. + +Example: + + ``` + { + "proxy_url": "http(s)://proxy.url:1234", + "minVersion": "1.0", + "maxVersion": "1.0", + "ciphers": [ + "TLS_RSA_WITH_AES_128_GCM_SHA256", + "TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA" + ], + "insecureSkipVerify": true, + "forceCommonNameCheck": false + } ``` + +Tyk classic API definition: `proxy.transport` + +**Field: `insecureSkipVerify` (`boolean`)** +InsecureSkipVerify controls whether a client verifies the server's certificate chain and host name. +If InsecureSkipVerify is true, crypto/tls accepts any certificate presented by the server and any host name in that certificate. +In this mode, TLS is susceptible to machine-in-the-middle attacks unless custom verification is used. +This should be used only for testing or in combination with VerifyConnection or VerifyPeerCertificate. + + +Tyk classic API definition: `proxy.transport.ssl_insecure_skip_verify`. + +**Field: `ciphers` (`[]string`)** +Ciphers is a list of SSL ciphers to be used. If unset, the default ciphers will be used. + + +Tyk classic API definition: `proxy.transport.ssl_ciphers`. + +**Field: `minVersion` (`string`)** +MinVersion is the minimum SSL/TLS version that is acceptable. + +Tyk classic API definition: `proxy.transport.ssl_min_version`. + +**Field: `maxVersion` (`string`)** +MaxVersion is the maximum SSL/TLS version that is acceptable. + +**Field: `forceCommonNameCheck` (`boolean`)** +ForceCommonNameCheck forces the validation of the hostname against the certificate Common Name. + + +Tyk classic API definition: `proxy.transport.ssl_force_common_name_check`. + +### **Proxy** + +Proxy contains the configuration for an internal proxy. + +Tyk classic API definition: `proxy.proxy_url` + +**Field: `enabled` (`boolean`)** +Enabled determines if the proxy is active. + +**Field: `url` (`string`)** +URL specifies the URL of the internal proxy. + +### **ListenPath** + +ListenPath is the base path on Tyk to which requests for this API +should be sent. Tyk listens out for any requests coming into the host at +this path, on the port that Tyk is configured to run on and processes +these accordingly. + +**Field: `value` (`string`)** +Value is the value of the listen path e.g. `/api/` or `/` or `/httpbin/`. + +Tyk classic API definition: `proxy.listen_path`. + +**Field: `strip` (`boolean`)** +Strip removes the inbound listen path (as accessed by the client) when generating the outbound request for the upstream service. + +For example, consider the scenario where the Tyk base address is `http://acme.com/', the listen path is `example/` and the upstream URL is `http://httpbin.org/`: + +- If the client application sends a request to `http://acme.com/example/get` then the request will be proxied to `http://httpbin.org/example/get` +- If stripListenPath is set to `true`, the `example` listen path is removed and the request would be proxied to `http://httpbin.org/get`. + + +Tyk classic API definition: `proxy.strip_listen_path`. + +### **Authentication** + +Authentication contains configuration about the authentication methods and security policies applied to requests. + +**Field: `enabled` (`boolean`)** +Enabled makes the API protected when one of the authentication modes is enabled. + + +Tyk classic API definition: `!use_keyless`. + +**Field: `stripAuthorizationData` (`boolean`)** +StripAuthorizationData ensures that any security tokens used for accessing APIs are stripped and not passed to the upstream. + + +Tyk classic API definition: `strip_auth_data`. + +**Field: `baseIdentityProvider` (`string`)** +BaseIdentityProvider enables the use of multiple authentication mechanisms. +It provides the session object that determines access control, rate limits and usage quotas. + +It should be set to one of the following: + +- `auth_token` +- `hmac_key` +- `basic_auth_user` +- `jwt_claim` +- `oidc_user` +- `oauth_key` +- `custom_auth` + + +Tyk classic API definition: `base_identity_provided_by`. + +**Field: `hmac` ([HMAC](#hmac))** +HMAC contains the configurations related to HMAC authentication mode. + + +Tyk classic API definition: `auth_configs["hmac"]`. + +**Field: `oidc` ([OIDC](#oidc))** +OIDC contains the configurations related to OIDC authentication mode. + + +Tyk classic API definition: `auth_configs["oidc"]`. + +**Field: `custom` ([CustomPluginAuthentication](#custompluginauthentication))** +Custom contains the configurations related to Custom authentication mode. + + +Tyk classic API definition: `auth_configs["coprocess"]`. + +**Field: `securitySchemes` ([SecuritySchemes](#securityschemes))** +SecuritySchemes contains security schemes definitions. + +**Field: `customKeyLifetime` ([CustomKeyLifetime](#customkeylifetime))** +CustomKeyLifetime contains configuration for the maximum retention period for access tokens. + +### **ClientCertificates** + +ClientCertificates contains the configurations related to establishing static mutual TLS between the client and Tyk. + +**Field: `enabled` (`boolean`)** +Enabled activates static mTLS for the API. + + +Tyk classic API definition: `use_mutual_tls_auth`. + +**Field: `allowlist` (`[]string`)** +Allowlist is the list of client certificates which are allowed. + + +Tyk classic API definition: `client_certificates`. + +### **GatewayTags** + +GatewayTags holds a list of segment tags that should apply for a gateway. + +**Field: `enabled` (`boolean`)** +Enabled activates use of segment tags. + + +Tyk classic API definition: `tags_disabled` (negated). + +**Field: `tags` (`[]string`)** +Tags contains a list of segment tags. + + +Tyk classic API definition: `tags`. + +### **Domain** + +Domain holds the configuration of the domain name the server should listen on. + +**Field: `enabled` (`boolean`)** +Enabled allow/disallow the usage of the domain. + + +Tyk classic API definition: `domain_disabled` (negated). + +**Field: `name` (`string`)** +Name is the name of the domain. + + +Tyk classic API definition: `domain`. + +**Field: `certificates` (`[]string`)** +Certificates defines a field for specifying certificate IDs or file paths +that the Gateway can utilise to dynamically load certificates for your custom domain. + + +Tyk classic API definition: `certificates`. + +### **DetailedActivityLogs** + +DetailedActivityLogs holds the configuration related to recording detailed analytics. + +**Field: `enabled` (`boolean`)** +Enabled activates detailed activity logs. + + +Tyk classic API definition: `enable_detailed_recording`. + +### **DetailedTracing** + +DetailedTracing holds the configuration of the detailed tracing. + +**Field: `enabled` (`boolean`)** +Enabled activates detailed tracing. + + +Tyk classic API definition: `detailed_tracing`. + +### **EventHandlers** + +EventHandlers holds the list of events to be processed for the API. + +Type defined as array of `EventHandler` values, see [EventHandler](#eventhandler) definition. + +### **IPAccessControl** + +IPAccessControl represents IP access control configuration. + +**Field: `enabled` (`boolean`)** +Enabled indicates whether IP access control is enabled. + + +Tyk classic API definition: `ip_access_control_disabled` (negated). + +**Field: `allow` (`[]string`)** +Allow is a list of allowed IP addresses or CIDR blocks (e.g. "192.168.1.0/24"). +Note that if an IP address is present in both Allow and Block, the Block rule will take precedence. + + +Tyk classic API definition: `allowed_ips`. + +**Field: `block` (`[]string`)** +Block is a list of blocked IP addresses or CIDR blocks (e.g. "192.168.1.100/32"). +If an IP address is present in both Allow and Block, the Block rule will take precedence. + + +Tyk classic API definition: `blacklisted_ips`. + +### **BatchProcessing** + +BatchProcessing represents the configuration for enabling or disabling batch request support for an API. + +**Field: `enabled` (`boolean`)** +Enabled determines whether batch request support is enabled or disabled for the API. + + +Tyk classic API definition: `enable_batch_request_support`. + +### **Global** + +Global contains configuration that affects the whole API (all endpoints). + +**Field: `pluginConfig` ([PluginConfig](#pluginconfig))** +PluginConfig contains the common configuration for custom plugins. + +**Field: `cors` ([CORS](#cors))** +CORS contains the configuration related to Cross Origin Resource Sharing. + +Tyk classic API definition: `CORS`. + +**Field: `prePlugin` ([PrePlugin](#preplugin))** +PrePlugin contains configuration related to the custom plugin that is run before authentication. +Deprecated: Use PrePlugins instead. + +**Field: `prePlugins` ([CustomPlugins](#customplugins))** +PrePlugins contains configuration related to the custom plugin that is run before authentication. + +Tyk classic API definition: `custom_middleware.pre`. + +**Field: `postAuthenticationPlugin` ([PostAuthenticationPlugin](#postauthenticationplugin))** +PostAuthenticationPlugin contains configuration related to the custom plugin that is run immediately after authentication. +Deprecated: Use PostAuthenticationPlugins instead. + +**Field: `postAuthenticationPlugins` ([CustomPlugins](#customplugins))** +PostAuthenticationPlugins contains configuration related to the custom plugin that is run immediately after authentication. + +Tyk classic API definition: `custom_middleware.post_key_auth`. + +**Field: `postPlugin` ([PostPlugin](#postplugin))** +PostPlugin contains configuration related to the custom plugin that is run immediately prior to proxying the request to the upstream. +Deprecated: Use PostPlugins instead. + +**Field: `postPlugins` ([CustomPlugins](#customplugins))** +PostPlugins contains configuration related to the custom plugin that is run immediately prior to proxying the request to the upstream. + +Tyk classic API definition: `custom_middleware.post`. + +**Field: `responsePlugin` ([ResponsePlugin](#responseplugin))** +ResponsePlugin contains configuration related to the custom plugin that is run during processing of the response from the upstream service. +Deprecated: Use ResponsePlugins instead. + +**Field: `responsePlugins` ([CustomPlugins](#customplugins))** +ResponsePlugins contains configuration related to the custom plugin that is run during processing of the response from the upstream service. + + +Tyk classic API definition: `custom_middleware.response`. + +**Field: `cache` ([Cache](#cache))** +Cache contains the configurations related to caching. + +Tyk classic API definition: `cache_options`. + +**Field: `transformRequestHeaders` ([TransformHeaders](#transformheaders))** +TransformRequestHeaders contains the configurations related to API level request header transformation. + +Tyk classic API definition: `global_headers`/`global_headers_remove`. + +**Field: `transformResponseHeaders` ([TransformHeaders](#transformheaders))** +TransformResponseHeaders contains the configurations related to API level response header transformation. + +Tyk classic API definition: `global_response_headers`/`global_response_headers_remove`. + +**Field: `contextVariables` ([ContextVariables](#contextvariables))** +ContextVariables contains the configuration related to Tyk context variables. + +**Field: `trafficLogs` ([TrafficLogs](#trafficlogs))** +TrafficLogs contains the configurations related to API level log analytics. + +**Field: `requestSizeLimit` ([GlobalRequestSizeLimit](#globalrequestsizelimit))** +RequestSizeLimit contains the configuration related to limiting the global request size. + +**Field: `ignoreCase` ([IgnoreCase](#ignorecase))** +IgnoreCase contains the configuration to treat routes as case-insensitive. + +**Field: `skipRateLimit` (`boolean`)** +SkipRateLimit determines whether the rate-limiting middleware logic should be skipped. + +Tyk classic API definition: `disable_rate_limit`. + +**Field: `skipQuota` (`boolean`)** +SkipQuota determines whether quota enforcement should be bypassed. + +Tyk classic API definition: `disable_quota`. + +**Field: `skipQuotaReset` (`boolean`)** +SkipQuotaReset indicates if quota limits should not be reset when creating or updating quotas for the API. + +Tyk classic API definition: `dont_set_quota_on_create`. + +### **Operations** + +Operations holds Operation definitions. + +Type defined as object of `Operation` values, see [Operation](#operation) definition. + +### **VersionToID** + +VersionToID contains a single mapping from a version name into an API ID. +Tyk classic API definition: Entry in `version_definition.versions` map. + +**Field: `name` (`string`)** +Name contains the user chosen version name, e.g. `v1` or similar. + +**Field: `id` (`string`)** +ID is the API ID for the version set in Name. + +### **ServiceDiscoveryCache** + +ServiceDiscoveryCache holds configuration for caching ServiceDiscovery data. + +**Field: `enabled` (`boolean`)** +Enabled turns service discovery cache on or off. + + +Tyk classic API definition: `service_discovery.cache_disabled`. + +**Field: `timeout` (`int64`)** +Timeout is the TTL for a cached object in seconds. + + +Tyk classic API definition: `service_discovery.cache_timeout`. + +### **UptimeTest** + +UptimeTest configures an uptime test check. + +**Field: `url` (`string`)** +CheckURL is the URL for a request. If service discovery is in use, +the hostname will be resolved to a service host. + +Examples: + +- `http://database1.company.local` +- `https://webcluster.service/health` +- `tcp://127.0.0.1:6379` (for TCP checks). + +**Field: `timeout` (`time.ReadableDuration`)** +Timeout declares a timeout for the request. If the test exceeds +this timeout, the check fails. + +**Field: `method` (`string`)** +Method allows you to customize the HTTP method for the test (`GET`, `POST`,...). + +**Field: `headers` (`map[string]string`)** +Headers contain any custom headers for the back end service. + +**Field: `body` (`string`)** +Body is the body of the test request. + +**Field: `commands` ([[]UptimeTestCommand](#uptimetestcommand))** +Commands are used for TCP checks. + +**Field: `enableProxyProtocol` (`boolean`)** +EnableProxyProtocol enables proxy protocol support when making request. +The back end service needs to support this. + +### **DomainToCertificate** + +DomainToCertificate holds a single mapping of domain name into a certificate. + +**Field: `domain` (`string`)** +Domain contains the domain name. + +**Field: `certificate` (`string`)** +Certificate contains the certificate mapped to the domain. + +### **PinnedPublicKeys** + +PinnedPublicKeys is a list of domains and pinned public keys for them. + +Type defined as array of `PinnedPublicKey` values, see [PinnedPublicKey](#pinnedpublickey) definition. + +### **ReadableDuration** + +ReadableDuration is an alias maintained to be used in imports. + +### **UpstreamBasicAuth** + +UpstreamBasicAuth holds upstream basic authentication configuration. + +**Field: `enabled` (`boolean`)** +Enabled enables upstream basic authentication. + +**Field: `header` ([AuthSource](#authsource))** +Header contains configurations for the header value. + +**Field: `username` (`string`)** +Username is the username to be used for upstream basic authentication. + +**Field: `password` (`string`)** +Password is the password to be used for upstream basic authentication. + +### **UpstreamOAuth** + +UpstreamOAuth holds the configuration for OAuth2 Client Credentials flow. + +**Field: `enabled` (`boolean`)** +Enabled activates upstream OAuth2 authentication. + +**Field: `allowedAuthorizeTypes` (`[]string`)** +AllowedAuthorizeTypes specifies the allowed authorization types for upstream OAuth2 authentication. + +**Field: `clientCredentials` ([ClientCredentials](#clientcredentials))** +ClientCredentials holds the configuration for OAuth2 Client Credentials flow. + +**Field: `password` ([PasswordAuthentication](#passwordauthentication))** +PasswordAuthentication holds the configuration for upstream OAauth password authentication flow. + +### **UpstreamRequestSigning** + +UpstreamRequestSigning represents configuration for generating signed requests to an upstream API. +Tyk classic API definition: `request_signing`. + +**Field: `enabled` (`boolean`)** +Enabled determines if request signing is enabled or disabled. + +**Field: `signatureHeader` (`string`)** +SignatureHeader specifies the HTTP header name for the signature. + +**Field: `algorithm` (`string`)** +Algorithm represents the signing algorithm used (e.g., HMAC-SHA256). + +**Field: `keyId` (`string`)** +KeyID identifies the key used for signing purposes. + +**Field: `headers` (`[]string`)** +Headers contains a list of headers included in the signature calculation. + +**Field: `secret` (`string`)** +Secret holds the secret used for signing when applicable. + +**Field: `certificateId` (`string`)** +CertificateID specifies the certificate ID used in signing operations. + +### **LoadBalancingTarget** + +LoadBalancingTarget represents a single upstream target for load balancing with a URL and an associated weight. + +**Field: `url` (`string`)** +URL specifies the upstream target URL for load balancing, represented as a string. + +**Field: `weight` (`int`)** +Weight specifies the relative distribution factor for load balancing, determining the importance of this target. + +### **HMAC** + +HMAC holds the configuration for the HMAC authentication mode. + +**Field: `enabled` (`boolean`)** +Enabled activates the HMAC authentication mode. + + +Tyk classic API definition: `enable_signature_checking`. + +**Field: `allowedAlgorithms` (`[]string`)** +AllowedAlgorithms is the array of HMAC algorithms which are allowed. + +Tyk supports the following HMAC algorithms: + +- `hmac-sha1` +- `hmac-sha256` +- `hmac-sha384` +- `hmac-sha512` + +and reads the value from the algorithm header. + + +Tyk classic API definition: `hmac_allowed_algorithms`. + +**Field: `allowedClockSkew` (`float64`)** +AllowedClockSkew is the amount of milliseconds that will be tolerated for clock skew. It is used against replay attacks. +The default value is `0`, which deactivates clock skew checks. + + +Tyk classic API definition: `hmac_allowed_clock_skew`. + +### **OIDC** + +OIDC contains configuration for the OIDC authentication mode. +OIDC support will be deprecated starting from 5.7.0. +To avoid any disruptions, we recommend that you use JSON Web Token (JWT) instead, +as explained in https://tyk.io/docs/basic-config-and-security/security/authentication-authorization/openid-connect/. + +**Field: `enabled` (`boolean`)** +Enabled activates the OIDC authentication mode. + + +Tyk classic API definition: `use_openid`. + +**Field: `segregateByClientId` (`boolean`)** +SegregateByClientId is a boolean flag. If set to `true, the policies will be applied to a combination of Client ID and User ID. + + +Tyk classic API definition: `openid_options.segregate_by_client`. + +**Field: `providers` ([[]Provider](#provider))** +Providers contains a list of authorized providers, their Client IDs and matched policies. + + +Tyk classic API definition: `openid_options.providers`. + +**Field: `scopes` ([Scopes](#scopes))** +Scopes contains the defined scope claims. + +### **CustomPluginAuthentication** + +CustomPluginAuthentication holds configuration for custom plugins. + +**Field: `enabled` (`boolean`)** +Enabled activates the CustomPluginAuthentication authentication mode. + + +Tyk classic API definition: `enable_coprocess_auth`/`use_go_plugin_auth`. + +**Field: `config` ([AuthenticationPlugin](#authenticationplugin))** +Config contains configuration related to custom authentication plugin. + + +Tyk classic API definition: `custom_middleware.auth_check`. + +### **SecuritySchemes** + +SecuritySchemes holds security scheme values, filled with Import(). + +### **CustomKeyLifetime** + +CustomKeyLifetime contains configuration for custom key retention. + +**Field: `enabled` (`boolean`)** +Enabled enables custom maximum retention for keys for the API. + +**Field: `value` ([ReadableDuration](#readableduration))** +Value configures the expiry interval for a Key. +The value is a string that specifies the interval in a compact form, +where hours, minutes and seconds are denoted by 'h', 'm' and 's' respectively. +Multiple units can be combined to represent the duration. + +Examples of valid shorthand notations: +- "1h" : one hour +- "20m" : twenty minutes +- "30s" : thirty seconds +- "1m29s": one minute and twenty-nine seconds +- "1h30m" : one hour and thirty minutes + +An empty value is interpreted as "0s" + + +Tyk classic API definition: `session_lifetime`. + +**Field: `respectValidity` (`boolean`)** +RespectValidity ensures that Tyk respects the expiry configured in the key when the API level configuration grants a shorter lifetime. +That is, Redis waits until the key has expired before deleting it. + + +Tyk classic API definition: `session_lifetime_respects_key_expiration`. + +### **EventHandler** + +EventHandler holds information about individual event to be configured on the API. + +**Field: `enabled` (`boolean`)** +Enabled enables the event handler. + + +Tyk classic API definition: `event_handlers.events[].handler_meta.disabled` (negated). + +**Field: `trigger` (`event.Event`)** +Trigger specifies the TykEvent that should trigger the event handler. + + +Tyk classic API definition: `event_handlers.events` key. + +**Field: `type` ([Kind](#kind))** +Kind specifies the action to be taken on the event trigger. + + +Tyk classic API definition: `event_handlers.events[].handler`. + +**Field: `id` (`string`)** +ID is the ID of event handler in storage. + + +Tyk classic API definition: `event_handlers.events[].handler_meta.id`. + +**Field: `name` (`string`)** +Name is the name of event handler. + + +Tyk classic API definition: `event_handlers.events[].handler_meta.name`. + +**Field: `` ([WebhookEvent](#webhookevent))** +Webhook contains WebhookEvent configs. Encoding and decoding is handled by the custom marshaller. + +**Field: `` ([JSVMEvent](#jsvmevent))** +JSVMEvent holds information about JavaScript VM events. + +**Field: `` ([LogEvent](#logevent))** +LogEvent represents the configuration for logging events tied to an event handler. + +### **PluginConfig** + +PluginConfig holds configuration for custom plugins. + +**Field: `driver` (`string`)** +Driver configures which custom plugin driver to use. +The value should be set to one of the following: + +- `otto`, +- `python`, +- `lua`, +- `grpc`, +- `goplugin`. + + +Tyk classic API definition: `custom_middleware.driver`. + +**Field: `bundle` ([PluginBundle](#pluginbundle))** +Bundle configures custom plugin bundles. + +**Field: `data` ([PluginConfigData](#pluginconfigdata))** +Data configures custom plugin data. + +### **CORS** + +CORS holds configuration for cross-origin resource sharing. + +**Field: `enabled` (`boolean`)** +Enabled is a boolean flag, if set to `true`, this option enables CORS processing. + + +Tyk classic API definition: `CORS.enable`. + +**Field: `maxAge` (`int`)** +MaxAge indicates how long (in seconds) the results of a preflight request can be cached. The default is 0 which stands for no max age. + + +Tyk classic API definition: `CORS.max_age`. + +**Field: `allowCredentials` (`boolean`)** +AllowCredentials indicates if the request can include user credentials like cookies, +HTTP authentication or client side SSL certificates. + + +Tyk classic API definition: `CORS.allow_credentials`. + +**Field: `exposedHeaders` (`[]string`)** +ExposedHeaders indicates which headers are safe to expose to the API of a CORS API specification. + + +Tyk classic API definition: `CORS.exposed_headers`. + +**Field: `allowedHeaders` (`[]string`)** +AllowedHeaders holds a list of non simple headers the client is allowed to use with cross-domain requests. + + +Tyk classic API definition: `CORS.allowed_headers`. + +**Field: `optionsPassthrough` (`boolean`)** +OptionsPassthrough is a boolean flag. If set to `true`, it will proxy the CORS OPTIONS pre-flight +request directly to upstream, without authentication and any CORS checks. This means that pre-flight +requests generated by web-clients such as SwaggerUI or the Tyk Portal documentation system +will be able to test the API using trial keys. + +If your service handles CORS natively, then enable this option. + + +Tyk classic API definition: `CORS.options_passthrough`. + +**Field: `debug` (`boolean`)** +Debug is a boolean flag, If set to `true`, this option produces log files for the CORS middleware. + + +Tyk classic API definition: `CORS.debug`. + +**Field: `allowedOrigins` (`[]string`)** +AllowedOrigins holds a list of origin domains to allow access from. Wildcards are also supported, e.g. `http://*.foo.com` + + +Tyk classic API definition: `CORS.allowed_origins`. + +**Field: `allowedMethods` (`[]string`)** +AllowedMethods holds a list of methods to allow access via. + + +Tyk classic API definition: `CORS.allowed_methods`. + +### **PrePlugin** + +PrePlugin configures pre-request plugins. + +Pre-request plugins are executed before the request is sent to the +upstream target and before any authentication information is extracted +from the header or parameter list of the request. + +**Field: `plugins` ([CustomPlugins](#customplugins))** +Plugins configures custom plugins to be run on pre authentication stage. +The plugins would be executed in the order of configuration in the list. + +### **CustomPlugins** + +CustomPlugins is a list of CustomPlugin objects. + +Type defined as array of `CustomPlugin` values, see [CustomPlugin](#customplugin) definition. + +### **PostAuthenticationPlugin** + +PostAuthenticationPlugin configures post authentication plugins. + +**Field: `plugins` ([CustomPlugins](#customplugins))** +Plugins configures custom plugins to be run on pre authentication stage. +The plugins would be executed in the order of configuration in the list. + +### **CustomPlugins** + +CustomPlugins is a list of CustomPlugin objects. + +Type defined as array of `CustomPlugin` values, see [CustomPlugin](#customplugin) definition. + +### **PostPlugin** + +PostPlugin configures post plugins. + +**Field: `plugins` ([CustomPlugins](#customplugins))** +Plugins configures custom plugins to be run on post stage. +The plugins would be executed in the order of configuration in the list. + +### **CustomPlugins** + +CustomPlugins is a list of CustomPlugin objects. + +Type defined as array of `CustomPlugin` values, see [CustomPlugin](#customplugin) definition. + +### **ResponsePlugin** + +ResponsePlugin configures response plugins. + +**Field: `plugins` ([CustomPlugins](#customplugins))** +Plugins configures custom plugins to be run on post stage. +The plugins would be executed in the order of configuration in the list. + +### **CustomPlugins** + +CustomPlugins is a list of CustomPlugin objects. + +Type defined as array of `CustomPlugin` values, see [CustomPlugin](#customplugin) definition. + +### **Cache** + +Cache holds configuration for caching the requests. + +**Field: `enabled` (`boolean`)** +Enabled turns global cache middleware on or off. It is still possible to enable caching on a per-path basis +by explicitly setting the endpoint cache middleware. + + +Tyk classic API definition: `cache_options.enable_cache`. + +**Field: `timeout` (`int64`)** +Timeout is the TTL for a cached object in seconds. + + +Tyk classic API definition: `cache_options.cache_timeout`. + +**Field: `cacheAllSafeRequests` (`boolean`)** +CacheAllSafeRequests caches responses to (`GET`, `HEAD`, `OPTIONS`) requests overrides per-path cache settings in versions, +applies across versions. + + +Tyk classic API definition: `cache_options.cache_all_safe_requests`. + +**Field: `cacheResponseCodes` (`[]int`)** +CacheResponseCodes is an array of response codes which are safe to cache e.g. `404`. + + +Tyk classic API definition: `cache_options.cache_response_codes`. + +**Field: `cacheByHeaders` (`[]string`)** +CacheByHeaders allows header values to be used as part of the cache key. + + +Tyk classic API definition: `cache_options.cache_by_headers`. + +**Field: `enableUpstreamCacheControl` (`boolean`)** +EnableUpstreamCacheControl instructs Tyk Cache to respect upstream cache control headers. + + +Tyk classic API definition: `cache_options.enable_upstream_cache_control`. + +**Field: `controlTTLHeaderName` (`string`)** +ControlTTLHeaderName is the response header which tells Tyk how long it is safe to cache the response for. + + +Tyk classic API definition: `cache_options.cache_control_ttl_header`. + +### **TransformHeaders** + +TransformHeaders holds configuration about request/response header transformations. + +**Field: `enabled` (`boolean`)** +Enabled activates Header Transform for the given path and method. + + +Tyk classic API definition: `version_data.versions..extended_paths.transform_headers[].disabled` (negated). + +**Field: `remove` (`[]string`)** +Remove specifies header names to be removed from the request/response. + + +Tyk classic API definition: `version_data.versions..extended_paths.transform_headers[].delete_headers`. + +**Field: `add` ([Headers](#headers))** +Add specifies headers to be added to the request/response. + + +Tyk classic API definition: `version_data.versions..extended_paths.transform_headers[].add_headers`. + +### **TransformHeaders** + +TransformHeaders holds configuration about request/response header transformations. + +**Field: `enabled` (`boolean`)** +Enabled activates Header Transform for the given path and method. + + +Tyk classic API definition: `version_data.versions..extended_paths.transform_headers[].disabled` (negated). + +**Field: `remove` (`[]string`)** +Remove specifies header names to be removed from the request/response. + + +Tyk classic API definition: `version_data.versions..extended_paths.transform_headers[].delete_headers`. + +**Field: `add` ([Headers](#headers))** +Add specifies headers to be added to the request/response. + + +Tyk classic API definition: `version_data.versions..extended_paths.transform_headers[].add_headers`. + +### **ContextVariables** + +ContextVariables holds the configuration related to Tyk context variables. + +**Field: `enabled` (`boolean`)** +Enabled enables context variables to be passed to Tyk middleware. + + +Tyk classic API definition: `enable_context_vars`. + +### **TrafficLogs** + +TrafficLogs holds configuration about API log analytics. + +**Field: `enabled` (`boolean`)** +Enabled enables traffic log analytics for the API. + +Tyk classic API definition: `do_not_track`. + +**Field: `tagHeaders` (`[]string`)** +TagHeaders is a string array of HTTP headers that can be extracted +and transformed into analytics tags (statistics aggregated by tag, per hour). + +**Field: `customRetentionPeriod` ([ReadableDuration](#readableduration))** +CustomRetentionPeriod configures a custom value for how long the analytics is retained for, +defaults to 100 years. + +**Field: `plugins` ([CustomAnalyticsPlugins](#customanalyticsplugins))** +Plugins configures custom plugins to allow for extensive modifications to analytics records +The plugins would be executed in the order of configuration in the list. + +### **GlobalRequestSizeLimit** + +GlobalRequestSizeLimit holds configuration about the global limits for request sizes. + +**Field: `enabled` (`boolean`)** +Enabled activates the Request Size Limit. + + +Tyk classic API definition: `version_data.versions..global_size_limit_disabled` (negated). + +**Field: `value` (`int64`)** +Value contains the value of the request size limit. + + +Tyk classic API definition: `version_data.versions..global_size_limit`. + +### **IgnoreCase** + +IgnoreCase will make route matching be case insensitive. +This accepts request to `/AAA` or `/aaa` if set to true. + +**Field: `enabled` (`boolean`)** +Enabled activates case insensitive route matching. + + +Tyk classic API definition: `version_data.versions..ignore_endpoint_case`. + +### **UptimeTestCommand** + +UptimeTestCommand handles additional checks for tcp connections. + +**Field: `name` (`string`)** +Name can be either `send` or `expect`, designating if the +message should be sent, or read from the connection. + +**Field: `message` (`string`)** +Message contains the payload to send or expect. + +### **PinnedPublicKey** + +PinnedPublicKey contains a mapping from the domain name into a list of public keys. + +**Field: `domain` (`string`)** +Domain contains the domain name. + +**Field: `publicKeys` (`[]string`)** +PublicKeys contains a list of the public keys pinned to the domain name. + +### **AuthSource** + +AuthSource defines an authentication source. + +**Field: `enabled` (`boolean`)** +Enabled activates the auth source. + + +Tyk classic API definition: `auth_configs[X].use_param/use_cookie`. + +**Field: `name` (`string`)** +Name is the name of the auth source. + + +Tyk classic API definition: `auth_configs[X].param_name/cookie_name`. + +### **ClientCredentials** + +ClientCredentials holds the configuration for OAuth2 Client Credentials flow. + +**Field: `header` ([AuthSource](#authsource))** +Header holds the configuration for the custom header to be used for OAuth authentication. + +**Field: `tokenUrl` (`string`)** +TokenURL is the resource server's token endpoint +URL. This is a constant specific to each server. + +**Field: `scopes` (`[]string`)** +Scopes specifies optional requested permissions. + +**Field: `extraMetadata` (`[]string`)** +ExtraMetadata holds the keys that we want to extract from the token and pass to the upstream. + +### **PasswordAuthentication** + +PasswordAuthentication holds the configuration for upstream OAuth2 password authentication flow. + +**Field: `header` ([AuthSource](#authsource))** +Header holds the configuration for the custom header to be used for OAuth authentication. + +**Field: `username` (`string`)** +Username is the username to be used for upstream OAuth2 password authentication. + +**Field: `password` (`string`)** +Password is the password to be used for upstream OAuth2 password authentication. + +**Field: `tokenUrl` (`string`)** +TokenURL is the resource server's token endpoint +URL. This is a constant specific to each server. + +**Field: `scopes` (`[]string`)** +Scopes specifies optional requested permissions. + +**Field: `extraMetadata` (`[]string`)** +ExtraMetadata holds the keys that we want to extract from the token and pass to the upstream. + +### **Provider** + +Provider defines an issuer to validate and the Client ID to Policy ID mappings. + +**Field: `issuer` (`string`)** +Issuer contains a validation value for the issuer claim, usually a domain name e.g. `accounts.google.com` or similar. + + +Tyk classic API definition: `openid_options.providers[].issuer`. + +**Field: `clientToPolicyMapping` ([[]ClientToPolicy](#clienttopolicy))** +ClientToPolicyMapping contains mappings of Client IDs to Policy IDs. + + +Tyk classic API definition: `openid_options.providers[].client_ids`. + +### **Scopes** + +Scopes holds the scope to policy mappings for a claim name. +This struct is used for both JWT and OIDC authentication. + +**Field: `claimName` (`string`)** +ClaimName contains the claim name. + + +Tyk classic API definition:. +- For OIDC: `scopes.oidc.scope_claim_name` +- For JWT: `scopes.jwt.scope_claim_name` + +**Field: `scopeToPolicyMapping` ([[]ScopeToPolicy](#scopetopolicy))** +ScopeToPolicyMapping contains the mappings of scopes to policy IDs. + + +Tyk classic API definition:. +- For OIDC: `scopes.oidc.scope_to_policy` +- For JWT: `scopes.jwt.scope_to_policy` + +### **AuthenticationPlugin** + +AuthenticationPlugin holds the configuration for custom authentication plugin. + +**Field: `enabled` (`boolean`)** +Enabled activates custom authentication plugin. + + +Tyk classic API definition: `custom_middleware.auth_check.disabled` (negated). + +**Field: `functionName` (`string`)** +FunctionName is the name of authentication method. + + +Tyk classic API definition: `custom_middleware.auth_check.name`. + +**Field: `path` (`string`)** +Path is the path to shared object file in case of goplugin mode or path to JS code in case of otto auth plugin. + + +Tyk classic API definition: `custom_middleware.auth_check.path`. + +**Field: `rawBodyOnly` (`boolean`)** +RawBodyOnly if set to true, do not fill body in request or response object. + + +Tyk classic API definition: `custom_middleware.auth_check.raw_body_only`. + +**Field: `requireSession` (`boolean`)** +RequireSession passes down the session information for plugins after authentication. + + +Tyk classic API definition: `custom_middleware.auth_check.require_session`. + +**Field: `idExtractor` ([IDExtractor](#idextractor))** +IDExtractor configures ID extractor with coprocess custom authentication. + + +Tyk classic API definition: `custom_middleware.id_extractor`. + +### **Kind** + +Kind is an alias maintained to be used in imports. + +### **WebhookEvent** + +WebhookEvent stores the core information about a webhook event. + +**Field: `url` (`string`)** +URL is the target URL for the webhook. + + +Tyk classic API definition: `event_handlers.events[].handler_meta.target_path`. + +**Field: `method` (`string`)** +Method is the HTTP method for the webhook. + + +Tyk classic API definition: `event_handlers.events[].handler_meta.method`. + +**Field: `cooldownPeriod` ([ReadableDuration](#readableduration))** +CoolDownPeriod defines cool-down for the event, so it does not trigger again. +It uses shorthand notation. +The value of CoolDownPeriod is a string that specifies the interval in a compact form, +where hours, minutes and seconds are denoted by 'h', 'm' and 's' respectively. +Multiple units can be combined to represent the duration. + +Examples of valid shorthand notations: +- "1h" : one hour +- "20m" : twenty minutes +- "30s" : thirty seconds +- "1m29s": one minute and twenty-nine seconds +- "1h30m" : one hour and thirty minutes + +An empty value is interpreted as "0s", implying no cool-down. +It's important to format the string correctly, as invalid formats will +be considered as 0s/empty. + + +Tyk classic API definition: `event_handlers.events[].handler_meta.event_timeout`. + +**Field: `bodyTemplate` (`string`)** +BodyTemplate is the template to be used for request payload. + + +Tyk classic API definition: `event_handlers.events[].handler_meta.template_path`. + +**Field: `headers` ([Headers](#headers))** +Headers are the list of request headers to be used. + + +Tyk classic API definition: `event_handlers.events[].handler_meta.header_map`. + +### **JSVMEvent** + +JSVMEvent represents a JavaScript VM event configuration for event handlers. + +**Field: `functionName` (`string`)** +FunctionName specifies the JavaScript function name to be executed. + + +Tyk classic API definition: `event_handlers.events[].handler_meta.method_name`. + +**Field: `path` (`string`)** +Path specifies the path to the JavaScript file containing the function. + + +Tyk classic API definition: `event_handlers.events[].handler_meta.path`. + +### **LogEvent** + +LogEvent represents the configuration for logging events within an event handler. + +**Field: `logPrefix` (`string`)** +LogPrefix defines the prefix used for log messages in the logging event. + + +Tyk classic API definition: `event_handlers.events[].handler_meta.prefix`. + +### **PluginBundle** + +PluginBundle holds configuration for custom plugins. + +**Field: `enabled` (`boolean`)** +Enabled activates the custom plugin bundles. + + +Tyk classic API definition: `custom_middleware_bundle_disabled`. + +**Field: `path` (`string`)** +Path is the path suffix to construct the URL to fetch plugin bundle from. +Path will be suffixed to `bundle_base_url` in gateway config. + +### **PluginConfigData** + +PluginConfigData configures config data for custom plugins. + +**Field: `enabled` (`boolean`)** +Enabled activates custom plugin config data. + + +Tyk classic API definition: `config_data_disabled` (negated). + +**Field: `value` (`any`)** +Value is the value of custom plugin config data. + + +Tyk classic API definition: `config_data`. + +### **CustomPlugin** + +CustomPlugin configures custom plugin. + +**Field: `enabled` (`boolean`)** +Enabled activates the custom plugin. + + +Tyk classic API definition: `custom_middleware.pre[].disabled`, `custom_middleware.post_key_auth[].disabled`,. +`custom_middleware.post[].disabled`, `custom_middleware.response[].disabled` (negated). + +**Field: `functionName` (`string`)** +FunctionName is the name of authentication method. + + +Tyk classic API definition: `custom_middleware.pre[].name`, `custom_middleware.post_key_auth[].name`,. +`custom_middleware.post[].name`, `custom_middleware.response[].name`. + +**Field: `path` (`string`)** +Path is the path to shared object file in case of goplugin mode or path to JS code in case of otto auth plugin. + + +Tyk classic API definition: `custom_middleware.pre[].path`, `custom_middleware.post_key_auth[].path`,. +`custom_middleware.post[].path`, `custom_middleware.response[].path`. + +**Field: `rawBodyOnly` (`boolean`)** +RawBodyOnly if set to true, do not fill body in request or response object. + + +Tyk classic API definition: `custom_middleware.pre[].raw_body_only`, `custom_middleware.post_key_auth[].raw_body_only`,. +`custom_middleware.post[].raw_body_only`, `custom_middleware.response[].raw_body_only`. + +**Field: `requireSession` (`boolean`)** +RequireSession if set to true passes down the session information for plugins after authentication. +RequireSession is used only with JSVM custom middleware. + + +Tyk classic API definition: `custom_middleware.pre[].require_session`, `custom_middleware.post_key_auth[].require_session`,. +`custom_middleware.post[].require_session`, `custom_middleware.response[].require_session`. + +### **CustomPlugin** + +CustomPlugin configures custom plugin. + +**Field: `enabled` (`boolean`)** +Enabled activates the custom plugin. + + +Tyk classic API definition: `custom_middleware.pre[].disabled`, `custom_middleware.post_key_auth[].disabled`,. +`custom_middleware.post[].disabled`, `custom_middleware.response[].disabled` (negated). + +**Field: `functionName` (`string`)** +FunctionName is the name of authentication method. + + +Tyk classic API definition: `custom_middleware.pre[].name`, `custom_middleware.post_key_auth[].name`,. +`custom_middleware.post[].name`, `custom_middleware.response[].name`. + +**Field: `path` (`string`)** +Path is the path to shared object file in case of goplugin mode or path to JS code in case of otto auth plugin. + + +Tyk classic API definition: `custom_middleware.pre[].path`, `custom_middleware.post_key_auth[].path`,. +`custom_middleware.post[].path`, `custom_middleware.response[].path`. + +**Field: `rawBodyOnly` (`boolean`)** +RawBodyOnly if set to true, do not fill body in request or response object. + + +Tyk classic API definition: `custom_middleware.pre[].raw_body_only`, `custom_middleware.post_key_auth[].raw_body_only`,. +`custom_middleware.post[].raw_body_only`, `custom_middleware.response[].raw_body_only`. + +**Field: `requireSession` (`boolean`)** +RequireSession if set to true passes down the session information for plugins after authentication. +RequireSession is used only with JSVM custom middleware. + + +Tyk classic API definition: `custom_middleware.pre[].require_session`, `custom_middleware.post_key_auth[].require_session`,. +`custom_middleware.post[].require_session`, `custom_middleware.response[].require_session`. + +### **CustomPlugin** + +CustomPlugin configures custom plugin. + +**Field: `enabled` (`boolean`)** +Enabled activates the custom plugin. + + +Tyk classic API definition: `custom_middleware.pre[].disabled`, `custom_middleware.post_key_auth[].disabled`,. +`custom_middleware.post[].disabled`, `custom_middleware.response[].disabled` (negated). + +**Field: `functionName` (`string`)** +FunctionName is the name of authentication method. + + +Tyk classic API definition: `custom_middleware.pre[].name`, `custom_middleware.post_key_auth[].name`,. +`custom_middleware.post[].name`, `custom_middleware.response[].name`. + +**Field: `path` (`string`)** +Path is the path to shared object file in case of goplugin mode or path to JS code in case of otto auth plugin. + + +Tyk classic API definition: `custom_middleware.pre[].path`, `custom_middleware.post_key_auth[].path`,. +`custom_middleware.post[].path`, `custom_middleware.response[].path`. + +**Field: `rawBodyOnly` (`boolean`)** +RawBodyOnly if set to true, do not fill body in request or response object. + + +Tyk classic API definition: `custom_middleware.pre[].raw_body_only`, `custom_middleware.post_key_auth[].raw_body_only`,. +`custom_middleware.post[].raw_body_only`, `custom_middleware.response[].raw_body_only`. + +**Field: `requireSession` (`boolean`)** +RequireSession if set to true passes down the session information for plugins after authentication. +RequireSession is used only with JSVM custom middleware. + + +Tyk classic API definition: `custom_middleware.pre[].require_session`, `custom_middleware.post_key_auth[].require_session`,. +`custom_middleware.post[].require_session`, `custom_middleware.response[].require_session`. + +### **CustomPlugin** + +CustomPlugin configures custom plugin. + +**Field: `enabled` (`boolean`)** +Enabled activates the custom plugin. + + +Tyk classic API definition: `custom_middleware.pre[].disabled`, `custom_middleware.post_key_auth[].disabled`,. +`custom_middleware.post[].disabled`, `custom_middleware.response[].disabled` (negated). + +**Field: `functionName` (`string`)** +FunctionName is the name of authentication method. + + +Tyk classic API definition: `custom_middleware.pre[].name`, `custom_middleware.post_key_auth[].name`,. +`custom_middleware.post[].name`, `custom_middleware.response[].name`. + +**Field: `path` (`string`)** +Path is the path to shared object file in case of goplugin mode or path to JS code in case of otto auth plugin. + + +Tyk classic API definition: `custom_middleware.pre[].path`, `custom_middleware.post_key_auth[].path`,. +`custom_middleware.post[].path`, `custom_middleware.response[].path`. + +**Field: `rawBodyOnly` (`boolean`)** +RawBodyOnly if set to true, do not fill body in request or response object. + + +Tyk classic API definition: `custom_middleware.pre[].raw_body_only`, `custom_middleware.post_key_auth[].raw_body_only`,. +`custom_middleware.post[].raw_body_only`, `custom_middleware.response[].raw_body_only`. + +**Field: `requireSession` (`boolean`)** +RequireSession if set to true passes down the session information for plugins after authentication. +RequireSession is used only with JSVM custom middleware. + + +Tyk classic API definition: `custom_middleware.pre[].require_session`, `custom_middleware.post_key_auth[].require_session`,. +`custom_middleware.post[].require_session`, `custom_middleware.response[].require_session`. + +### **Headers** + +Headers is an array of Header. + +Type defined as array of `Header` values, see [Header](#header) definition. + +### **CustomAnalyticsPlugins** + +CustomAnalyticsPlugins is a list of CustomPlugin objects for analytics. + +Type defined as array of `CustomPlugin` values, see [CustomPlugin](#customplugin) definition. + +### **ClientToPolicy** + +ClientToPolicy contains a 1-1 mapping between Client ID and Policy ID. + +**Field: `clientId` (`string`)** +ClientID contains a Client ID. + + +Tyk classic API definition: Key in `openid_options.providers[].client_ids` map. + +**Field: `policyId` (`string`)** +PolicyID contains a Policy ID. + + +Tyk classic API definition: Value in `openid_options.providers[].client_ids` map. + +### **ScopeToPolicy** + +ScopeToPolicy contains a single scope to policy ID mapping. +This struct is used for both JWT and OIDC authentication. + +**Field: `scope` (`string`)** +Scope contains the scope name. + + +Tyk classic API definition:. +- For OIDC: Key in `scopes.oidc.scope_to_policy` map +- For JWT: Key in `scopes.jwt.scope_to_policy` map. + +**Field: `policyId` (`string`)** +PolicyID contains the Policy ID. + + +Tyk classic API definition:. +- For OIDC: Value in `scopes.oidc.scope_to_policy` map +- For JWT: Value in `scopes.jwt.scope_to_policy` map. + +### **IDExtractor** + +IDExtractor configures ID Extractor. + +**Field: `enabled` (`boolean`)** +Enabled activates ID extractor with coprocess authentication. + + +Tyk classic API definition: `custom_middleware.id_extractor.disabled` (negated). + +**Field: `source` (`string`)** +Source is the source from which ID to be extracted from. +Valid values are: +- `header` - Extract ID from a header +- `form` - Extract ID from a form parameter +- `body` - Extract ID from the request body + + +Tyk classic API definition: `custom_middleware.id_extractor.extract_from`. + +**Field: `with` (`string`)** +With is the type of ID extractor to be used. +Valid values are: +- `value` - Extract ID from a value +- `xpath` - Extract ID using an XPath expression +- `regex` - Extract ID using a regular expression + + +Tyk classic API definition: `custom_middleware.id_extractor.extract_with`. + +**Field: `config` ([IDExtractorConfig](#idextractorconfig))** +Config holds the configuration specific to ID extractor type mentioned via With. + + +Tyk classic API definition: `custom_middleware.id_extractor.extractor_config`. + +### **Header** + +Header holds a header name and value pair. + +**Field: `name` (`string`)** +Name is the name of the header. + +**Field: `value` (`string`)** +Value is the value of the header. + +### **CustomPlugin** + +CustomPlugin configures custom plugin. + +**Field: `enabled` (`boolean`)** +Enabled activates the custom plugin. + + +Tyk classic API definition: `custom_middleware.pre[].disabled`, `custom_middleware.post_key_auth[].disabled`,. +`custom_middleware.post[].disabled`, `custom_middleware.response[].disabled` (negated). + +**Field: `functionName` (`string`)** +FunctionName is the name of authentication method. + + +Tyk classic API definition: `custom_middleware.pre[].name`, `custom_middleware.post_key_auth[].name`,. +`custom_middleware.post[].name`, `custom_middleware.response[].name`. + +**Field: `path` (`string`)** +Path is the path to shared object file in case of goplugin mode or path to JS code in case of otto auth plugin. + + +Tyk classic API definition: `custom_middleware.pre[].path`, `custom_middleware.post_key_auth[].path`,. +`custom_middleware.post[].path`, `custom_middleware.response[].path`. + +**Field: `rawBodyOnly` (`boolean`)** +RawBodyOnly if set to true, do not fill body in request or response object. + + +Tyk classic API definition: `custom_middleware.pre[].raw_body_only`, `custom_middleware.post_key_auth[].raw_body_only`,. +`custom_middleware.post[].raw_body_only`, `custom_middleware.response[].raw_body_only`. + +**Field: `requireSession` (`boolean`)** +RequireSession if set to true passes down the session information for plugins after authentication. +RequireSession is used only with JSVM custom middleware. + + +Tyk classic API definition: `custom_middleware.pre[].require_session`, `custom_middleware.post_key_auth[].require_session`,. +`custom_middleware.post[].require_session`, `custom_middleware.response[].require_session`. + +### **IDExtractorConfig** + +IDExtractorConfig specifies the configuration for ID extractor. + +**Field: `headerName` (`string`)** +HeaderName is the header name to extract ID from. +Used when Source is set to "header" and With is set to "value". + + +Tyk classic API definition: `custom_middleware.id_extractor.extractor_config.header_name`. + +**Field: `formParamName` (`string`)** +FormParamName is the form parameter name to extract ID from. +Used when Source is set to "form" and With is set to "value". + + +Tyk classic API definition: `custom_middleware.id_extractor.extractor_config.form_param_name`. + +**Field: `regexp` (`string`)** +Regexp is the regular expression to match ID. +Used when With is set to "regex". + + +Tyk classic API definition: `custom_middleware.id_extractor.extractor_config.regex_expression`. + +**Field: `regexpMatchIndex` (`int`)** +RegexpMatchIndex is the index from which ID to be extracted after a match. +Default value is 0, ie if regexpMatchIndex is not provided ID is matched from index 0. +Used when With is set to "regex". + + +Tyk classic API definition: `custom_middleware.id_extractor.extractor_config.regex_match_index`. + +**Field: `xPathExp` (`string`)** +XPathExp is the xpath expression to match ID. +Used when With is set to "xpath". + + +Tyk classic API definition: `custom_middleware.id_extractor.extractor_config.xpath_expression`. + +### **Allowance** + +Allowance describes allowance actions and behaviour. + +**Field: `enabled` (`boolean`)** +Enabled is a boolean flag, if set to `true`, then individual allowances (allow, block, ignore) will be enforced. + +**Field: `ignoreCase` (`boolean`)** +IgnoreCase is a boolean flag, If set to `true`, checks for requests allowance will be case insensitive. + +### **AllowanceType** + +AllowanceType holds the valid allowance types values. + +### **AuthSources** + +AuthSources defines authentication source configuration: headers, cookies and query parameters. + +Tyk classic API definition: `auth_configs{}`. + +**Field: `header` ([AuthSource](#authsource))** +Header contains configurations for the header value auth source, it is enabled by default. + + +Tyk classic API definition: `auth_configs[x].header`. + +**Field: `cookie` ([AuthSource](#authsource))** +Cookie contains configurations for the cookie value auth source. + + +Tyk classic API definition: `auth_configs[x].cookie`. + +**Field: `query` ([AuthSource](#authsource))** +Query contains configurations for the query parameters auth source. + + +Tyk classic API definition: `auth_configs[x].query`. + +### **Basic** + +Basic type holds configuration values related to http basic authentication. + +**Field: `enabled` (`boolean`)** +Enabled activates the basic authentication mode. + +Tyk classic API definition: `use_basic_auth`. + +**Field: `disableCaching` (`boolean`)** +DisableCaching disables the caching of basic authentication key. + +Tyk classic API definition: `basic_auth.disable_caching`. + +**Field: `cacheTTL` (`int`)** +CacheTTL is the TTL for a cached basic authentication key in seconds. + +Tyk classic API definition: `basic_auth.cache_ttl`. + +**Field: `extractCredentialsFromBody` ([ExtractCredentialsFromBody](#extractcredentialsfrombody))** +ExtractCredentialsFromBody helps to extract username and password from body. In some cases, like dealing with SOAP, +user credentials can be passed via request body. + +### **CachePlugin** + +CachePlugin holds the configuration for the cache plugins. + +**Field: `enabled` (`boolean`)** +Enabled is a boolean flag. If set to `true`, the advanced caching plugin will be enabled. + + +Tyk classic API definition: `version_data.versions..extended_paths.advance_cache_config[].disabled` (negated). + +**Field: `cacheByRegex` (`string`)** +CacheByRegex defines a regular expression used against the request body to produce a cache key. + +Example value: `\"id\":[^,]*` (quoted json value). + + +Tyk classic API definition: `version_data.versions..extended_paths.advance_cache_config[].cache_key_regex`. + +**Field: `cacheResponseCodes` (`[]int`)** +CacheResponseCodes contains a list of valid response codes for responses that are okay to add to the cache. + + +Tyk classic API definition: `version_data.versions..extended_paths.advance_cache_config[].cache_response_codes`. + +**Field: `timeout` (`int64`)** +Timeout is the TTL for the endpoint level caching in seconds. 0 means no caching. + + +Tyk classic API definition: `version_data.versions..extended_paths.advance_cache_config[].timeout`. + +### **CircuitBreaker** + +CircuitBreaker holds configuration for the circuit breaker middleware. +Tyk classic API definition: `version_data.versions..extended_paths.circuit_breakers[*]`. + +**Field: `enabled` (`boolean`)** +Enabled activates the Circuit Breaker functionality. + + +Tyk classic API definition: `version_data.versions..extended_paths.circuit_breakers[*].disabled` (negated). + +**Field: `threshold` (`float64`)** +Threshold is the proportion from each `sampleSize` requests that must fail for the breaker to be tripped. This must be a value between 0.0 and 1.0. If `sampleSize` is 100 then a threshold of 0.4 means that the breaker will be tripped if 40 out of every 100 requests fails. + + +Tyk classic API definition: `version_data.versions..extended_paths.circuit_breakers[*].threshold_percent`. + +**Field: `sampleSize` (`int`)** +SampleSize is the size of the circuit breaker sampling window. Combining this with `threshold` gives the failure rate required to trip the circuit breaker. + + +Tyk classic API definition: `version_data.versions..extended_paths.circuit_breakers[*].samples`. + +**Field: `coolDownPeriod` (`int`)** +CoolDownPeriod is the period of time (in seconds) for which the circuit breaker will remain open before returning to service. + + +Tyk classic API definition: `version_data.versions..extended_paths.circuit_breakers[*].return_to_service_after`. + +**Field: `halfOpenStateEnabled` (`boolean`)** +HalfOpenStateEnabled , if enabled, allows some requests to pass through the circuit breaker during the cool down period. If Tyk detects that the path is now working, the circuit breaker will be automatically reset and traffic will be resumed to the upstream. + + +Tyk classic API definition: `version_data.versions..extended_paths.circuit_breakers[*].disable_half_open_state` (negated). + +### **ClientAuthData** + +ClientAuthData holds the client ID and secret for OAuth2 authentication. + +**Field: `clientId` (`string`)** +ClientID is the application's ID. + +**Field: `clientSecret` (`string`)** +ClientSecret is the application's secret. + +### **EndpointPostPlugin** + +EndpointPostPlugin contains endpoint level post plugin configuration. + +**Field: `enabled` (`boolean`)** +Enabled activates post plugin. + + +Tyk classic API definition: `version_data.versions..extended_paths.go_plugin.disabled`(negated). + +**Field: `name` (`string`)** +Name is the name of plugin function to be executed. +Deprecated: Use FunctionName instead. + +**Field: `functionName` (`string`)** +FunctionName is the name of plugin function to be executed. + + +Tyk classic API definition: `version_data.versions..extended_paths.go_plugin.symbol_name`(negated). + +**Field: `path` (`string`)** +Path is the path to plugin. + + +Tyk classic API definition: `version_data.versions..extended_paths.go_plugin.plugin_path`(negated). + +### **EndpointPostPlugins** + +EndpointPostPlugins is a list of EndpointPostPlugins. It's used where multiple plugins can be run. + +Type defined as array of `EndpointPostPlugin` values, see [EndpointPostPlugin](#endpointpostplugin) definition. + +### **EnforceTimeout** + +EnforceTimeout holds the configuration for enforcing request timeouts. + +**Field: `enabled` (`boolean`)** +Enabled is a boolean flag. If set to `true`, requests will enforce a configured timeout. + + +Tyk classic API definition: `version_data.versions..extended_paths.hard_timeouts[].disabled` (negated). + +**Field: `value` (`int`)** +Value is the configured timeout in seconds. + + +Tyk classic API definition: `version_data.versions..extended_paths.hard_timeouts[].timeout`. + +### **ExternalOAuth** + +ExternalOAuth holds configuration for an external OAuth provider. +ExternalOAuth support will be deprecated starting from 5.7.0. +To avoid any disruptions, we recommend that you use JSON Web Token (JWT) instead, +as explained in https://tyk.io/docs/basic-config-and-security/security/authentication-authorization/ext-oauth-middleware/. + +**Field: `enabled` (`boolean`)** +Enabled activates external oauth functionality. + + +Tyk classic API definition: `external_oauth.enabled`. + +**Field: `providers` ([[]OAuthProvider](#oauthprovider))** +Providers is used to configure OAuth providers. + + +Tyk classic API definition: `external_oauth.providers`. + +### **ExtractCredentialsFromBody** + +ExtractCredentialsFromBody configures extracting credentials from the request body. + +**Field: `enabled` (`boolean`)** +Enabled activates extracting credentials from body. + +Tyk classic API definition: `basic_auth.extract_from_body`. + +**Field: `userRegexp` (`string`)** +UserRegexp is the regex for username e.g. `<User>(.*)</User>`. + +Tyk classic API definition: `basic_auth.userRegexp`. + +**Field: `passwordRegexp` (`string`)** +PasswordRegexp is the regex for password e.g. `<Password>(.*)</Password>`. + +Tyk classic API definition: `basic_auth.passwordRegexp`. + +### **FromOASExamples** + +FromOASExamples configures mock responses that should be returned from OAS example responses. + +**Field: `enabled` (`boolean`)** +Enabled activates getting a mock response from OAS examples or schemas documented in OAS. + +**Field: `code` (`int`)** +Code is the default HTTP response code that the gateway reads from the path responses documented in OAS. + +**Field: `contentType` (`string`)** +ContentType is the default HTTP response body type that the gateway reads from the path responses documented in OAS. + +**Field: `exampleName` (`string`)** +ExampleName is the default example name among multiple path response examples documented in OAS. + +### **Internal** + +Internal holds the endpoint configuration, configuring the endpoint for internal requests. +Tyk classic API definition: `version_data.versions...extended_paths.internal[*]`. + +**Field: `enabled` (`boolean`)** +Enabled if set to true makes the endpoint available only for internal requests. + +### **Introspection** + +Introspection holds configuration for OAuth token introspection. + +**Field: `enabled` (`boolean`)** +Enabled activates OAuth access token validation by introspection to a third party. + + +Tyk classic API definition: `external_oauth.providers[].introspection.enabled`. + +**Field: `url` (`string`)** +URL is the URL of the third party provider's introspection endpoint. + + +Tyk classic API definition: `external_oauth.providers[].introspection.url`. + +**Field: `clientId` (`string`)** +ClientID is the public identifier for the client, acquired from the third party. + + +Tyk classic API definition: `external_oauth.providers[].introspection.client_id`. + +**Field: `clientSecret` (`string`)** +ClientSecret is a secret known only to the client and the authorisation server, acquired from the third party. + + +Tyk classic API definition: `external_oauth.providers[].introspection.client_secret`. + +**Field: `identityBaseField` (`string`)** +IdentityBaseField is the key showing where to find the user id in the claims. If it is empty, the `sub` key is looked at. + + +Tyk classic API definition: `external_oauth.providers[].introspection.identity_base_field`. + +**Field: `cache` ([IntrospectionCache](#introspectioncache))** +Cache is the caching mechanism for introspection responses. + + +Tyk classic API definition: `external_oauth.providers[].introspection.cache`. + +### **IntrospectionCache** + +IntrospectionCache holds configuration for caching introspection requests. + +**Field: `enabled` (`boolean`)** +Enabled activates the caching mechanism for introspection responses. + + +Tyk classic API definition: `external_oauth.providers[].introspection.cache.enabled`. + +**Field: `timeout` (`int64`)** +Timeout is the duration in seconds of how long the cached value stays. +For introspection caching, it is suggested to use a short interval. + + +Tyk classic API definition: `external_oauth.providers[].introspection.cache.timeout`. + +### **JWT** + +JWT holds the configuration for the JWT middleware. + +**Field: `enabled` (`boolean`)** +Enabled activates the basic authentication mode. + + +Tyk classic API definition: `enable_jwt`. + +**Field: `source` (`string`)** +Source contains the source for the JWT. + + +Tyk classic API definition: `jwt_source`. + +**Field: `signingMethod` (`string`)** +SigningMethod contains the signing method to use for the JWT. + + +Tyk classic API definition: `jwt_signing_method`. + +**Field: `identityBaseField` (`string`)** +IdentityBaseField specifies the claim name uniquely identifying the subject of the JWT. +The identity fields that are checked in order are: `kid`, IdentityBaseField, `sub`. + + +Tyk classic API definition: `jwt_identity_base_field`. + +**Field: `skipKid` (`boolean`)** +SkipKid controls skipping using the `kid` claim from a JWT (default behaviour). +When this is true, the field configured in IdentityBaseField is checked first. + + +Tyk classic API definition: `jwt_skip_kid`. + +**Field: `policyFieldName` (`string`)** +PolicyFieldName is a configurable claim name from which a policy ID is extracted. +The policy is applied to the session as a base policy. + + +Tyk classic API definition: `jwt_policy_field_name`. + +**Field: `clientBaseField` (`string`)** +ClientBaseField is used when PolicyFieldName is not provided. It will get +a session key and use the policies from that. The field ensures that requests +use the same session. + + +Tyk classic API definition: `jwt_client_base_field`. + +**Field: `scopes` ([Scopes](#scopes))** +Scopes holds the scope to policy mappings for a claim name. + +**Field: `defaultPolicies` (`[]string`)** +DefaultPolicies is a list of policy IDs that apply to the session. + + +Tyk classic API definition: `jwt_default_policies`. + +**Field: `issuedAtValidationSkew` (`uint64`)** +IssuedAtValidationSkew contains the duration in seconds for which token issuance can predate the current time during the request. + + +Tyk classic API definition: `jwt_issued_at_validation_skew`. + +**Field: `notBeforeValidationSkew` (`uint64`)** +NotBeforeValidationSkew contains the duration in seconds for which token validity can predate the current time during the request. + + +Tyk classic API definition: `jwt_not_before_validation_skew`. + +**Field: `expiresAtValidationSkew` (`uint64`)** +ExpiresAtValidationSkew contains the duration in seconds for which the token can be expired before we consider it expired. + + +Tyk classic API definition: `jwt_expires_at_validation_skew`. + +**Field: `idpClientIdMappingDisabled` (`boolean`)** +IDPClientIDMappingDisabled prevents Tyk from automatically detecting the use of certain IDPs based on standard claims +that they include in the JWT: `client_id`, `cid`, `clientId`. Setting this flag to `true` disables the mapping and avoids +accidentally misidentifying the use of one of these IDPs if one of their standard values is configured in your JWT. + + +Tyk classic API definition: `idp_client_id_mapping_disabled`. + +### **JWTValidation** + +JWTValidation holds configuration for validating access tokens by inspecing them +against a third party API, usually one provided by the IDP. + +**Field: `enabled` (`boolean`)** +Enabled activates OAuth access token validation. + + +Tyk classic API definition: `external_oauth.providers[].jwt.enabled`. + +**Field: `signingMethod` (`string`)** +SigningMethod to verify signing method used in jwt - allowed values HMAC/RSA/ECDSA. + + +Tyk classic API definition: `external_oauth.providers[].jwt.signing_method`. + +**Field: `source` (`string`)** +Source is the secret to verify signature. Valid values are: + +- a base64 encoded static secret, +- a valid JWK URL in plain text, +- a valid JWK URL in base64 encoded format. + + +Tyk classic API definition: `external_oauth.providers[].jwt.source`. + +**Field: `identityBaseField` (`string`)** +IdentityBaseField is the identity claim name. + + +Tyk classic API definition: `external_oauth.providers[].jwt.identity_base_field`. + +**Field: `issuedAtValidationSkew` (`uint64`)** +IssuedAtValidationSkew is the clock skew to be considered while validating the iat claim. + + +Tyk classic API definition: `external_oauth.providers[].jwt.issued_at_validation_skew`. + +**Field: `notBeforeValidationSkew` (`uint64`)** +NotBeforeValidationSkew is the clock skew to be considered while validating the nbf claim. + + +Tyk classic API definition: `external_oauth.providers[].jwt.not_before_validation_skew`. + +**Field: `expiresAtValidationSkew` (`uint64`)** +ExpiresAtValidationSkew is the clock skew to be considered while validating the exp claim. + + +Tyk classic API definition: `external_oauth.providers[].jwt.expires_at_validation_skew`. + +### **MockResponse** + +MockResponse configures the mock responses. + +**Field: `enabled` (`boolean`)** +Enabled activates the mock response middleware. + +**Field: `code` (`int`)** +Code is the HTTP response code that will be returned. + +**Field: `body` (`string`)** +Body is the HTTP response body that will be returned. + +**Field: `headers` ([Headers](#headers))** +Headers are the HTTP response headers that will be returned. + +**Field: `fromOASExamples` ([FromOASExamples](#fromoasexamples))** +FromOASExamples is the configuration to extract a mock response from OAS documentation. + +### **Notifications** + +Notifications holds configuration for updates to keys. + +**Field: `sharedSecret` (`string`)** +SharedSecret is the shared secret used in the notification request. + + +Tyk classic API definition: `notifications.shared_secret`. + +**Field: `onKeyChangeUrl` (`string`)** +OnKeyChangeURL is the URL a request will be triggered against. + + +Tyk classic API definition: `notifications.oauth_on_keychange_url`. + +### **OAuth** + +OAuth configures the OAuth middleware. + +**Field: `enabled` (`boolean`)** +Enabled activates the OAuth middleware. + + +Tyk classic API definition: `use_oauth2`. + +**Field: `allowedAuthorizeTypes` (`[]string`)** +AllowedAuthorizeTypes is an array of OAuth authorization types. + + +Tyk classic API definition: `oauth_meta.allowed_authorize_types`. + +**Field: `refreshToken` (`boolean`)** +RefreshToken enables clients using a refresh token to get a new bearer access token. + + +Tyk classic API definition: `oauth_meta.allowed_access_types` (contains REFRESH_TOKEN). + +**Field: `authLoginRedirect` (`string`)** +AuthLoginRedirect configures a URL to redirect to after a successful login. + + +Tyk classic API definition: `oauth_meta.auth_login_redirect`. + +**Field: `notifications` ([Notifications](#notifications))** +Notifications configures a URL trigger on key changes. + + +Tyk classic API definition: `notifications`. + +### **OAuthProvider** + +OAuthProvider holds the configuration for validation and introspection of OAuth tokens. + +**Field: `jwt` ([JWTValidation](#jwtvalidation))** +JWT configures JWT validation. + + +Tyk classic API definition: `external_oauth.providers[].jwt`. + +**Field: `introspection` ([Introspection](#introspection))** +Introspection configures token introspection. + + +Tyk classic API definition: `external_oauth.providers[].introspection`. + +### **Operation** + +Operation holds a request operation configuration, allowances, tranformations, caching, timeouts and validation. + +**Field: `allow` ([Allowance](#allowance))** +Allow request by allowance. + +**Field: `block` ([Allowance](#allowance))** +Block request by allowance. + +**Field: `ignoreAuthentication` ([Allowance](#allowance))** +IgnoreAuthentication ignores authentication on request by allowance. + +**Field: `internal` ([Internal](#internal))** +Internal makes the endpoint only respond to internal requests. + +**Field: `transformRequestMethod` ([TransformRequestMethod](#transformrequestmethod))** +TransformRequestMethod allows you to transform the method of a request. + +**Field: `transformRequestBody` ([TransformBody](#transformbody))** +TransformRequestBody allows you to transform request body. +When both `path` and `body` are provided, body would take precedence. + +**Field: `transformResponseBody` ([TransformBody](#transformbody))** +TransformResponseBody allows you to transform response body. +When both `path` and `body` are provided, body would take precedence. + +**Field: `transformRequestHeaders` ([TransformHeaders](#transformheaders))** +TransformRequestHeaders allows you to transform request headers. + +**Field: `transformResponseHeaders` ([TransformHeaders](#transformheaders))** +TransformResponseHeaders allows you to transform response headers. + +**Field: `urlRewrite` ([URLRewrite](#urlrewrite))** +URLRewrite contains the URL rewriting configuration. + +**Field: `cache` ([CachePlugin](#cacheplugin))** +Cache contains the caching plugin configuration. + +**Field: `enforceTimeout` ([EnforceTimeout](#enforcetimeout))** +EnforceTimeout contains the request timeout configuration. + +**Field: `validateRequest` ([ValidateRequest](#validaterequest))** +ValidateRequest contains the request validation configuration. + +**Field: `mockResponse` ([MockResponse](#mockresponse))** +MockResponse contains the mock response configuration. + +**Field: `virtualEndpoint` ([VirtualEndpoint](#virtualendpoint))** +VirtualEndpoint contains virtual endpoint configuration. + +**Field: `postPlugins` ([EndpointPostPlugins](#endpointpostplugins))** +PostPlugins contains endpoint level post plugins configuration. + +**Field: `circuitBreaker` ([CircuitBreaker](#circuitbreaker))** +CircuitBreaker contains the configuration for the circuit breaker functionality. + +**Field: `trackEndpoint` ([TrackEndpoint](#trackendpoint))** +TrackEndpoint contains the configuration for enabling analytics and logs. + +**Field: `doNotTrackEndpoint` ([TrackEndpoint](#trackendpoint))** +DoNotTrackEndpoint contains the configuration for disabling analytics and logs. + +**Field: `requestSizeLimit` ([RequestSizeLimit](#requestsizelimit))** +RequestSizeLimit limits the maximum allowed size of the request body in bytes. + +**Field: `rateLimit` ([RateLimitEndpoint](#ratelimitendpoint))** +RateLimit contains endpoint level rate limit configuration. + +### **Path** + +Path holds plugin configurations for HTTP method verbs. + +**Field: `DELETE` ([Plugins](#plugins))** +Delete holds plugin configuration for DELETE requests. + +**Field: `GET` ([Plugins](#plugins))** +Get holds plugin configuration for GET requests. + +**Field: `HEAD` ([Plugins](#plugins))** +Head holds plugin configuration for HEAD requests. + +**Field: `OPTIONS` ([Plugins](#plugins))** +Options holds plugin configuration for OPTIONS requests. + +**Field: `PATCH` ([Plugins](#plugins))** +Patch holds plugin configuration for PATCH requests. + +**Field: `POST` ([Plugins](#plugins))** +Post holds plugin configuration for POST requests. + +**Field: `PUT` ([Plugins](#plugins))** +Put holds plugin configuration for PUT requests. + +**Field: `TRACE` ([Plugins](#plugins))** +Trace holds plugin configuration for TRACE requests. + +**Field: `CONNECT` ([Plugins](#plugins))** +Connect holds plugin configuration for CONNECT requests. + +### **Paths** + +Paths is a mapping of API endpoints to Path plugin configurations. + +Type defined as object of `Path` values, see [Path](#path) definition. + +### **Plugins** + +Plugins configures common settings for each plugin, allowances, transforms, caching and timeouts. + +**Field: `allow` ([Allowance](#allowance))** +Allow request by allowance. + +**Field: `block` ([Allowance](#allowance))** +Block request by allowance. + +**Field: `ignoreAuthentication` ([Allowance](#allowance))** +IgnoreAuthentication ignores authentication on request by allowance. + +**Field: `transformRequestMethod` ([TransformRequestMethod](#transformrequestmethod))** +TransformRequestMethod allows you to transform the method of a request. + +**Field: `cache` ([CachePlugin](#cacheplugin))** +Cache allows you to cache the server side response. + +**Field: `enforcedTimeout` ([EnforceTimeout](#enforcetimeout))** +EnforceTimeout allows you to configure a request timeout. + +### **RateLimitEndpoint** + +RateLimitEndpoint carries same settings as RateLimit but for endpoints. + +Type defined as `RateLimit`, see [RateLimit](#ratelimit) definition. + +### **RequestSizeLimit** + +RequestSizeLimit limits the maximum allowed size of the request body in bytes. + +**Field: `enabled` (`boolean`)** +Enabled activates the Request Size Limit functionality. + + +Tyk classic API definition: `version_data.versions..extended_paths.size_limits[].disabled` (negated). + +**Field: `value` (`int64`)** +Value is the maximum allowed size of the request body in bytes. + + +Tyk classic API definition: `version_data.versions..extended_paths.size_limits[].size_limit`. + +### **SecurityScheme** + +SecurityScheme defines an Importer interface for security schemes. + +### **Signature** + +Signature holds the configuration for signature validation. + +**Field: `enabled` (`boolean`)** +Enabled activates signature validation. + + +Tyk classic API definition: `auth_configs[X].validate_signature`. + +**Field: `algorithm` (`string`)** +Algorithm is the signature method to use. + + +Tyk classic API definition: `auth_configs[X].signature.algorithm`. + +**Field: `header` (`string`)** +Header is the name of the header to consume. + + +Tyk classic API definition: `auth_configs[X].signature.header`. + +**Field: `query` ([AuthSource](#authsource))** +Query is the name of the query parameter to consume. + + +Tyk classic API definition: `auth_configs[X].signature.use_param/param_name`. + +**Field: `secret` (`string`)** +Secret is the signing secret used for signature validation. + + +Tyk classic API definition: `auth_configs[X].signature.secret`. + +**Field: `allowedClockSkew` (`int64`)** +AllowedClockSkew configures a grace period in seconds during which an expired token is still valid. + + +Tyk classic API definition: `auth_configs[X].signature.allowed_clock_skew`. + +**Field: `errorCode` (`int`)** +ErrorCode configures the HTTP response code for a validation failure. +If unconfigured, a HTTP 401 Unauthorized status code will be emitted. + + +Tyk classic API definition: `auth_configs[X].signature.error_code`. + +**Field: `errorMessage` (`string`)** +ErrorMessage configures the error message that is emitted on validation failure. +A default error message is emitted if unset. + + +Tyk classic API definition: `auth_configs[X].signature.error_message`. + +### **Token** + +Token holds the values related to authentication tokens. + +**Field: `enabled` (`boolean`)** +Enabled activates the token based authentication mode. + + +Tyk classic API definition: `auth_configs["authToken"].use_standard_auth`. + +**Field: `enableClientCertificate` (`boolean`)** +EnableClientCertificate allows to create dynamic keys based on certificates. + + +Tyk classic API definition: `auth_configs["authToken"].use_certificate`. + +**Field: `signatureValidation` ([Signature](#signature))** +Signature holds the configuration for verifying the signature of the token. + + +Tyk classic API definition: `auth_configs["authToken"].use_certificate`. + +### **TrackEndpoint** + +TrackEndpoint configures Track or DoNotTrack behaviour for an endpoint. +Tyk classic API definition: `version_data.versions..extended_paths.track_endpoints`, `version_data.versions..extended_paths.do_not_track_endpoints`. + +**Field: `enabled` (`boolean`)** +Enabled if set to true enables or disables tracking for an endpoint depending +if it's used in `trackEndpoint` or `doNotTrackEndpoint`. + +### **TransformBody** + +TransformBody holds configuration about request/response body transformations. + +**Field: `enabled` (`boolean`)** +Enabled activates transform request/request body middleware. + + +Tyk classic API definition: `version_data.versions..extended_paths.transform[].disabled` (negated). + +**Field: `format` (`string`)** +Format of the request/response body, xml or json. + + +Tyk classic API definition: `version_data.versions..extended_paths.transform[].template_data.input_type`. + +**Field: `path` (`string`)** +Path file path for the template. + + +Tyk classic API definition: `version_data.versions..extended_paths.transform[].template_data.template_source` when `template_data.template_mode` is `file`. + +**Field: `body` (`string`)** +Body base64 encoded representation of the template. + + +Tyk classic API definition: `version_data.versions..extended_paths.transform[].template_data.template_source` when `template_data.template_mode` is `blob`. + +### **TransformRequestMethod** + +TransformRequestMethod holds configuration for rewriting request methods. + +**Field: `enabled` (`boolean`)** +Enabled activates Method Transform for the given path and method. + +**Field: `toMethod` (`string`)** +ToMethod is the http method value to which the method of an incoming request will be transformed. + +### **URLRewrite** + +URLRewrite configures URL rewriting. +Tyk classic API definition: `version_data.versions..extended_paths.url_rewrite`. + +**Field: `enabled` (`boolean`)** +Enabled activates URL rewriting if set to true. + +**Field: `pattern` (`string`)** +Pattern is the regular expression against which the request URL is compared for the primary rewrite check. +If this matches the defined pattern, the primary URL rewrite is triggered. + +**Field: `rewriteTo` (`string`)** +RewriteTo specifies the URL to which the request shall be rewritten if the primary URL rewrite is triggered. + +**Field: `triggers` ([[]*URLRewriteTrigger](#urlrewritetrigger))** +Triggers contain advanced additional triggers for the URL rewrite. +The triggers are processed only if the requested URL matches the pattern above. + +### **URLRewriteCondition** + +URLRewriteCondition defines the matching mode for an URL rewrite rules. +Tyk classic API definition: Matching condition in `version_data.versions..extended_paths.url_rewrite[].triggers[].on`. + +- Value `any` means any of the defined trigger rules may match. +- Value `all` means all the defined trigger rules must match. + +### **URLRewriteInput** + +URLRewriteInput defines the input for an URL rewrite rule. +Tyk classic API definition: Input source for URL rewrite rules in `version_data.versions..extended_paths.url_rewrite[].triggers[].options`. + +The following values are valid: + +- `url`, match pattern against URL +- `query`, match pattern against named query parameter value +- `path`, match pattern against named path parameter value +- `header`, match pattern against named header value +- `sessionMetadata`, match pattern against session metadata +- `requestBody`, match pattern against request body +- `requestContext`, match pattern against request context + +The default `url` is used as the input source. + +### **URLRewriteRule** + +URLRewriteRule represents a rewrite matching rules. +Tyk classic API definition: `version_data.versions..extended_paths.url_rewrite[].triggers[].options`. + +**Field: `in` ([URLRewriteInput](#urlrewriteinput))** +In specifies one of the valid inputs for URL rewriting. + +**Field: `name` (`string`)** +Name is the index in the value declared inside `in`. + +Example: for `in=query`, `name=q`, the parameter `q` would +be read from the request query parameters. + +The value of name is unused when `in` is set to `requestBody`, +as the request body is a single value and not a set of values. + +**Field: `pattern` (`string`)** +Pattern is the regular expression against which the `in` values are compared for this rule check. +If the value matches the defined `pattern`, the URL rewrite is triggered for this rule. + +**Field: `negate` (`boolean`)** +Negate is a boolean negation operator. Setting it to true inverts the matching behaviour +such that the rewrite will be triggered if the value does not match the `pattern` for this rule. + +### **URLRewriteTrigger** + +URLRewriteTrigger represents a set of matching rules for a rewrite. +Tyk classic API definition: `version_data.versions..extended_paths.url_rewrite[].triggers`. + +**Field: `condition` ([URLRewriteCondition](#urlrewritecondition))** +Condition indicates the logical combination that will be applied to the rules for an advanced trigger. + +**Field: `rules` ([[]*URLRewriteRule](#urlrewriterule))** +Rules contain individual checks that are combined according to the +`condition` to determine if the URL rewrite will be triggered. +If empty, the trigger is ignored. + +**Field: `rewriteTo` (`string`)** +RewriteTo specifies the URL to which the request shall be rewritten +if indicated by the combination of `condition` and `rules`. + +### **ValidateRequest** + +ValidateRequest holds configuration required for validating requests. + +**Field: `enabled` (`boolean`)** +Enabled is a boolean flag, if set to `true`, it enables request validation. + +**Field: `errorResponseCode` (`int`)** +ErrorResponseCode is the error code emitted when the request fails validation. +If unset or zero, the response will returned with http status 422 Unprocessable Entity. + +### **VirtualEndpoint** + +VirtualEndpoint contains virtual endpoint configuration. + +**Field: `enabled` (`boolean`)** +Enabled activates virtual endpoint. + + +Tyk classic API definition: `virtual.disabled` (negated). + +**Field: `name` (`string`)** +Name is the name of plugin function to be executed. +Deprecated: Use FunctionName instead. + +**Field: `functionName` (`string`)** +FunctionName is the name of plugin function to be executed. + + +Tyk classic API definition: `virtual.response_function_name`. + +**Field: `path` (`string`)** +Path is the path to JS file. + + +Tyk classic API definition: `virtual.function_source_uri` when `virtual.function_source_type` is `file`. + +**Field: `body` (`string`)** +Body is the JS function to execute encoded in base64 format. + + +Tyk classic API definition: `virtual.function_source_uri` when `virtual.function_source_type` is `blob`. + +**Field: `proxyOnError` (`boolean`)** +ProxyOnError proxies if virtual endpoint errors out. + + +Tyk classic API definition: `virtual.proxy_on_error`. + +**Field: `requireSession` (`boolean`)** +RequireSession if enabled passes session to virtual endpoint. + + +Tyk classic API definition: `virtual.use_session`. + +### **XTykStreaming** + +XTykStreaming represents the structure for Tyk streaming configurations. + +**Field: `streams` (`any`)** +Streams contains the configurations related to Tyk Streams. + diff --git a/style.css b/style.css new file mode 100644 index 00000000..fad6b4cc --- /dev/null +++ b/style.css @@ -0,0 +1,727 @@ +li[id="topbar-cta-button"] > a > div > span { + color: #000 !important; +} +li[id="topbar-cta-button"] > a > div > svg > path { + stroke: #000 !important; +} + +.dark li[id="topbar-cta-button"] > a > div > span { + color: #000 !important; +} + +/* Target the background span directly */ +#topbar-cta-button span[class*="bg-primary-dark"] { + background-color: #20EDBA !important; /* Green */ + border: 1.2px solid black !important; /* Black border */ +} + +#topbar-cta-button:hover span[class*="bg-primary-dark"] { + background-color: #20EDBA !important; /* Darker green on hover */ + border: 1.2px solid black !important; /* Black border on hover */ +} + + +/* ---------- typography helpers with dark mode support ---------- */ +.home-section-heading { + color: #03031C; /* $brand-black */ + font-family: "Inter", sans-serif; + font-size: 20px; + font-weight: 700; + line-height: 28px; +} + +.dark .home-section-heading { + color: #ffffff; /* White text in dark mode */ +} + +.fs-18 { font-size: 18px; } + +.fs-16 { font-size: 16px; } +.fs-xs { font-size: 14px; } +.font-400 { font-weight: 400; } +.text-sec { color: var(--text-subdued, #515071); } + +.dark .text-sec { + color: #FFF; /* Light gray text in dark mode */ +} + +/* ---------- layout helpers ---------- */ +.home-section { margin-top: 40px; } + +/* keeps the β€œ1 % inline padding” behaviour β‰₯ 992 px */ +@media (min-width: 992px) { + .section-padding { padding-inline: 1%; } +} + +/* centres imgs exactly like .image-container in Bootstrap */ +.image-container { + display: flex; + justify-content: center; +} + +.parent-container { + display: flex; + flex-direction: column; + justify-content: space-between; + height: 100%; +} + +/* ---------- CTA button style with dark mode support ---------- */ +.sec-btn { + display: inline-block; + border-radius: 20px; + border: 1.2px solid #03031C; + background: #ffffff; + padding: 8px 24px; + color: #03031C; + font-size: 14px; + font-weight: 500; + line-height: 20px; + text-decoration: none; + transition: all 0.2s ease; +} +.sec-btn:hover { background: #E5E5E8; } + +.dark .sec-btn { + background: transparent; + color: #ffffff; + border: 1.2px solid #ffffff; +} +.dark .sec-btn:hover { + background: rgba(107, 114, 128, 0.1); + border-color: #ffffff; +} + +.stack-heading { + color: var(--action-primary-default, var(--primary-primary-50, #8438FA)); + font-size: 14px; + font-style: normal; + font-weight: 600; + line-height: 20px; /* 142.857% */ +} + +.fs-xs { + font-size: 14px; +} + +.lh-20 { + line-height: 20px ; +} + +.banner { + background-size: auto; + background-repeat: no-repeat; + background-position: right; + margin-left: 36px; + margin-right: 36px; + +} + +.home-container{ + padding-top: 10px; +} + +.home-bottom{ + margin-bottom: 70px; +} + +.feature-grid { + display: grid; + grid-template-columns: 1fr; + gap: 16px; + margin-bottom: 2rem; +} +@media (min-width: 576px) { + .feature-grid { + grid-template-columns: repeat(2, 1fr); + } +} +@media (min-width: 992px) { + .feature-grid { + grid-template-columns: repeat(4, 1fr); + } +} +.feature-item { + background-color: #ffffff; + border-radius: 8px; + padding: 16px; + height: 100%; + display: flex; + flex-direction: column; + align-items: center; + text-align: center; + box-shadow: 0 2px 12px rgba(0, 0, 0, 0.1); +} +.feature-icon { + font-size: 3.5rem; + margin-bottom: 16px; +} +.feature-title { + font-size: 1.4rem; + margin-bottom: 12px; + color: #333; +} +.feature-description { + color: #666; + flex-grow: 1; +} +.feature-button { + margin-top: 16px; +} +.feature-button a { + display: inline-block; + background-color: #00AEF0; + color: white; + padding: 8px 16px; + border-radius: 4px; + text-decoration: none; +} + +/* Button styles from Hugo theme */ +.button { + display: inline-block; + padding: 12px 24px; + border: none; + border-radius: 6px; + text-decoration: none; + font-weight: 500; + font-size: 16px; + transition: background-color 0.3s ease; + cursor: pointer; +} + +.button.center { + display: inline-block; +} + +.button.button-green { + background-color: #20EDBA; + color: white; +} + +.button.button-green:hover { + background-color: #20EDBA; +} + +.button.button-red { + background-color: #dc3545; + color: white; +} + +.button.button-red:hover { + background-color: #c82333; +} + +.button.button-black { + background-color: #343a40; + color: white; +} + +.button.button-black:hover { + background-color: #23272b; +} + +/* Grid styles for Hugo grid shortcode */ +.grid-container { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(300px, 1fr)); + gap: 1.5rem; + margin: 2rem 0; +} + +@media (min-width: 768px) { + .grid-container { + grid-template-columns: repeat(3, 1fr); + } +} + +@media (max-width: 767px) { + .grid-container { + grid-template-columns: 1fr; + } +} + +/* Badge styles - Card style badges for your BadgeCard component */ +.badge { + display: block; + background-color: #ffffff; + border-radius: 8px; + padding: 1rem; + text-decoration: none; + color: inherit; + box-shadow: 0 2px 12px rgba(0, 0, 0, 0.1); + transition: transform 0.2s ease, box-shadow 0.2s ease; + margin-bottom: 1rem; +} + +.badge:hover { + transform: translateY(-2px); + box-shadow: 0 4px 20px rgba(0, 0, 0, 0.15); + text-decoration: none; + color: inherit; +} + +.badge.with_image { + text-align: center; +} + +.badge .nav { + font-size: 0.875rem; + margin-bottom: 0.5rem; +} + +.badge .nav.read { + color: #666; + text-transform: uppercase; + font-weight: 500; + letter-spacing: 0.5px; +} + +.badge .nav.title { + font-size: 1.25rem; + font-weight: 600; + color: #333; + margin-bottom: 1rem; +} + +.badge p { + margin-bottom: 1rem; + color: #666; +} + +.badge p img { + max-width: 100%; + height: auto; + border-radius: 4px; + margin-bottom: 0.5rem; +} + +.badge .bottom { + display: flex; + justify-content: space-between; + align-items: center; + margin-top: 1rem; + padding-top: 1rem; + border-top: 1px solid #e5e7eb; +} + +.badge .bottom .read { + font-size: 0.875rem; + color: #666; + text-transform: uppercase; + font-weight: 500; + letter-spacing: 0.5px; +} + +/* Simple badge styles for inline use */ +.simple-badge { + display: inline-flex; + align-items: center; + gap: 0.25rem; + font-weight: 500; + text-decoration: none; + border-radius: 6px; + transition: all 0.2s ease; + white-space: nowrap; +} + +.simple-badge-small { + padding: 0.25rem 0.5rem; + font-size: 0.75rem; +} + +.simple-badge-medium { + padding: 0.375rem 0.75rem; + font-size: 0.875rem; +} + +.simple-badge-large { + padding: 0.5rem 1rem; + font-size: 1rem; +} + +.simple-badge-default { + background-color: #e5e7eb; + color: #374151; +} + +.simple-badge-default:hover { + background-color: #d1d5db; +} + +.simple-badge-primary { + background-color: #00AEF0; + color: white; +} + +.simple-badge-primary:hover { + background-color: #0090c4; +} + +.simple-badge-success { + background-color: #20EDBA; + color: white; +} + +.simple-badge-success:hover { + background-color: #1bc7a1; +} + +.badge-next { + background-color: #00AEF0; + color: white; + padding: 0.5rem 1rem; + margin-top: 1rem; + display: inline-flex; + align-items: center; + gap: 0.5rem; +} + +.badge-next:hover { + background-color: #0090c4; + color: white; +} + +.badge-icon { + display: inline-flex; + align-items: center; +} + +.badge-icon img, +.badge-icon object { + width: 16px; + height: 16px; + vertical-align: middle; +} + +/* GitHub Star Button styles */ +.github-button { + display: inline-block; + padding: 2px 5px 2px 4px; + color: #333; + text-decoration: none; + white-space: nowrap; + cursor: pointer; + border-radius: 3px; + border: 1px solid #d5d5d5; + background: linear-gradient(#fcfcfc, #eee); + font-size: 11px; + font-weight: bold; + line-height: 14px; + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif; +} + +.github-button:hover { + color: #333; + text-decoration: none; + background: linear-gradient(#ddd, #ccc); + border-color: #ccc; +} + +.github-button:focus { + outline: none; + border-color: #4183c4; +} + +.github-button:active { + background: #dcdcdc; + border-color: #b5b5b5; + box-shadow: inset 0 2px 4px rgba(0,0,0,0.15); +} + +.github-button .octicon { + display: inline-block; + vertical-align: text-top; + fill: currentColor; + margin-right: 2px; +} + + +/* ResponsiveGrid component styles */ +.responsive-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); + gap: 1.5rem; + margin: 1rem 0; + max-width: 1200px; + column-gap: 1.5rem; /* Keep horizontal spacing */ + row-gap: 0.5rem; +} + +/* Responsive breakpoints for grid */ +@media (max-width: 768px) { + .responsive-grid { + grid-template-columns: 1fr; + gap: 0.5rem; + } +} + +/* ONLY constrain cards INSIDE ResponsiveGrid - not global cards */ +.responsive-grid .card { + max-width: 300px !important; + min-width: 200px !important; + min-height: 160px !important; + width: 100% !important; + margin: 5px auto !important; +} + +/* ONLY constrain images INSIDE ResponsiveGrid cards */ +.responsive-grid .card img { + max-height: 100px !important; + max-width: 100px !important; + object-fit: scale-down !important; + margin: 0 auto !important; + display: block !important; +} + +/* Hover effects ONLY for ResponsiveGrid cards */ +.responsive-grid .card:hover { + transform: translateY(-2px); + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); + transition: all 0.2s ease; +} + +/* ---------- Dark mode improvements for homepage cards ---------- */ +/* Override Tailwind's border-gray-300 in dark mode for better contrast */ +.dark .border-gray-300 { + border-color: #4c5a6b !important; /* Strong blue-gray border */ +} + +/* Add subtle background to cards in dark mode for better definition */ +.dark .grid .border-gray-300 { + background-color: rgba(44, 52, 71, 0.4) !important; /* Deep navy background */ + border-color: #4c5a6b !important; /* Strong blue-gray border for cards */ + border-width: 1.5px !important; /* Slightly thicker border */ +} + +/* Improve hover effects for cards in dark mode */ +.dark .grid .border-gray-300:hover { + background-color: rgba(54, 63, 85, 0.5) !important; + border-color: #5d6b7d !important; + transform: translateY(-2px); + transition: all 0.2s ease; +} + +/* Ensure banner border looks good in dark mode */ +.dark .banner.border-gray-300 { + border-color: #4c5a6b !important; + background-color: rgba(44, 52, 71, 0.3) !important; +} + +/* ---------- Dark mode improvements for stack boxes ---------- */ +/* Style the stack boxes in the "Explore by Tyk Stack" section */ +.dark .stack-box { + background-color: rgba(44, 52, 71, 0.4) !important; + border-color: #4c5a6b !important; + border-width: 1.5px !important; +} + +.dark .stack-box:hover { + background-color: rgba(54, 63, 85, 0.5) !important; + border-color: #5d6b7d !important; +} + +/* Ensure stack headings are properly styled in dark mode */ +.dark .stack-heading { + color: #ffffff !important; /* White text instead of purple */ +} + +/* Style any other border-gray-300 elements that might be stack boxes */ +.dark .border.border-gray-300.rounded-md { + background-color: rgba(44, 52, 71, 0.4) !important; + border-color: #4c5a6b !important; + border-width: 1.5px !important; +} + +.dark .border.border-gray-300.rounded-md:hover { + background-color: rgba(54, 63, 85, 0.5) !important; + border-color: #5d6b7d !important; +} + +/* ---------- Dark mode link colors for "Most viewed" section only ---------- */ +/* Target only the text-primary links in the "Most viewed" section */ +.dark .text-primary.fs-xs { + color: #20EDBA !important; /* Tyk green color */ +} + +.dark .text-primary.fs-xs:hover { + color: #1bc7a1 !important; /* Slightly darker green on hover */ + text-decoration: underline; +} + +/* More specific targeting for all text-primary links in grid layouts */ +.dark .grid a.text-primary { + color: #20EDBA !important; +} + +.dark .grid a.text-primary:hover { + color: #1bc7a1 !important; +} + +/* ---------- Refined Professional Dark Mode for Tyk Stack Section ---------- */ +/* Enhanced styling to match the sophisticated reference design */ + +.dark .grid-cols-2 > div, +.dark .grid > div[class*="border"], +.dark .grid > div[class*="rounded"] { + background: #1e293b !important; /* Deeper, more sophisticated dark background */ + border: 1px solid #334155 !important; /* Refined subtle border */ + border-radius: 16px !important; /* More generous rounded corners */ + padding: 24px !important; /* Increased padding for better spacing */ + transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1) !important; /* Smoother easing */ + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15), 0 2px 4px rgba(0, 0, 0, 0.1) !important; /* Layered shadows */ + position: relative !important; + overflow: hidden !important; +} + +.dark .grid-cols-2 > div:hover, +.dark .grid > div[class*="border"]:hover, +.dark .grid > div[class*="rounded"]:hover { + background: #334155 !important; /* Refined hover background */ + border-color: #475569 !important; /* Subtle border color change */ + transform: translateY(-4px) !important; /* More pronounced lift effect */ + box-shadow: 0 12px 24px rgba(0, 0, 0, 0.25), 0 4px 8px rgba(0, 0, 0, 0.15) !important; /* Enhanced shadow */ +} + +/* Enhanced typography for better hierarchy and readability - matching reference design */ +.dark .grid-cols-2 h3, +.dark .grid > div h3, +.dark .grid-cols-2 .stack-heading, +.dark .grid > div .stack-heading, +.dark .grid-cols-2 > div h3, +.dark .grid > div[class*="border"] h3, +.dark .grid > div[class*="rounded"] h3 { + color: #ffffff !important; /* Pure white titles like reference */ + font-weight: 600 !important; /* Medium weight for clean look */ + font-size: 16px !important; /* Consistent with reference */ + line-height: 1.3 !important; /* Tight line height */ + margin-bottom: 6px !important; /* Compact spacing */ + letter-spacing: 0 !important; /* No letter spacing */ +} + +.dark .grid-cols-2 p, +.dark .grid > div p, +.dark .grid-cols-2 .text-sec, +.dark .grid > div .text-sec, +.dark .grid-cols-2 > div p, +.dark .grid > div[class*="border"] p, +.dark .grid > div[class*="rounded"] p, +.dark .grid-cols-2 > div .text-sec, +.dark .grid > div[class*="border"] .text-sec, +.dark .grid > div[class*="rounded"] .text-sec { + color: #9ca3af !important; /* Light gray descriptions matching reference */ + font-size: 13px !important; /* Slightly smaller for hierarchy */ + line-height: 1.4 !important; /* Readable line height */ + margin-top: 2px !important; /* Tight spacing */ + font-weight: 400 !important; /* Normal weight */ +} + +/* Additional specific targeting for all text elements in stack cards */ +.dark .grid-cols-2 > div *, +.dark .grid > div[class*="border"] *, +.dark .grid > div[class*="rounded"] * { + color: inherit !important; /* Inherit from parent */ +} + +/* Ensure titles are always white */ +.dark .grid-cols-2 > div h1, +.dark .grid-cols-2 > div h2, +.dark .grid-cols-2 > div h3, +.dark .grid-cols-2 > div h4, +.dark .grid-cols-2 > div h5, +.dark .grid-cols-2 > div h6, +.dark .grid > div[class*="border"] h1, +.dark .grid > div[class*="border"] h2, +.dark .grid > div[class*="border"] h3, +.dark .grid > div[class*="border"] h4, +.dark .grid > div[class*="border"] h5, +.dark .grid > div[class*="border"] h6, +.dark .grid > div[class*="rounded"] h1, +.dark .grid > div[class*="rounded"] h2, +.dark .grid > div[class*="rounded"] h3, +.dark .grid > div[class*="rounded"] h4, +.dark .grid > div[class*="rounded"] h5, +.dark .grid > div[class*="rounded"] h6 { + color: #ffffff !important; /* White titles */ + font-weight: 600 !important; +} + +/* Ensure descriptions are light gray */ +.dark .grid-cols-2 > div span, +.dark .grid-cols-2 > div div:not([class*="flex"]):not([class*="grid"]), +.dark .grid > div[class*="border"] span, +.dark .grid > div[class*="border"] div:not([class*="flex"]):not([class*="grid"]), +.dark .grid > div[class*="rounded"] span, +.dark .grid > div[class*="rounded"] div:not([class*="flex"]):not([class*="grid"]) { + color: #9ca3af !important; /* Light gray for descriptions */ +} + +/* Enhanced icon styling for better integration */ +.dark .grid-cols-2 img, +.dark .grid > div img { + filter: brightness(1.2) contrast(1.1) !important; /* Enhanced icon visibility */ + width: 48px !important; /* Consistent icon size */ + height: 48px !important; /* Consistent icon size */ + margin-bottom: 16px !important; /* Better spacing below icons */ + border-radius: 8px !important; /* Subtle rounding for icons */ + transition: transform 0.2s ease !important; /* Smooth icon transitions */ +} + +/* Add proper spacing between icon and text content in stack cards */ +.stack-text-container { + margin-left: 16px !important; /* Add spacing between icon and text for both light and dark mode */ +} + +/* ---------- Dark mode styling for card headings in "Try it out" section ---------- */ +.dark .grid-cols-2 > div:hover img, +.dark .grid > div[class*="border"]:hover img, +.dark .grid > div[class*="rounded"]:hover img { + transform: scale(1.05) !important; /* Subtle icon scale on hover */ +} + +/* Improved grid layout and spacing */ +.dark .grid-cols-2, +.dark .grid { + gap: 20px !important; /* Refined gap between cards */ + margin-top: 24px !important; /* Better section spacing */ +} + +/* Enhanced responsive behavior */ +@media (max-width: 768px) { + .dark .grid-cols-2 > div, + .dark .grid > div[class*="border"], + .dark .grid > div[class*="rounded"] { + padding: 20px !important; /* Adjusted padding for mobile */ + margin-bottom: 16px !important; /* Better mobile spacing */ + } + + .dark .grid-cols-2 h3, + .dark .grid > div h3, + .dark .grid-cols-2 .stack-heading, + .dark .grid > div .stack-heading { + font-size: 16px !important; /* Responsive font sizing */ + } +} + +/* Subtle background pattern for added sophistication */ +.dark .grid-cols-2 > div::before, +.dark .grid > div[class*="border"]::before, +.dark .grid > div[class*="rounded"]::before { + content: '' !important; + position: absolute !important; + top: 0 !important; + left: 0 !important; + right: 0 !important; + bottom: 0 !important; + background: linear-gradient(135deg, rgba(255, 255, 255, 0.02) 0%, rgba(255, 255, 255, 0) 50%) !important; + pointer-events: none !important; + border-radius: inherit !important; +} + +/* Enhanced focus states for accessibility */ +.dark .grid-cols-2 > div:focus, +.dark .grid > div[class*="border"]:focus, +.dark .grid > div[class*="rounded"]:focus { + outline: 2px solid #3b82f6 !important; + outline-offset: 2px !important; +} diff --git a/transform-traffic/url-rewriting.mdx b/transform-traffic/url-rewriting.mdx new file mode 100644 index 00000000..7ee17735 --- /dev/null +++ b/transform-traffic/url-rewriting.mdx @@ -0,0 +1,733 @@ +--- +title: "URL Rewriting" +'og:description': "How to rewrite URL in Tyk" +tags: ['Traffic Transformation', 'URL Rewrite'] +sidebarTitle: "URL Rewrite" +--- + +<h2 id="url-rewriting-overview">Overview</h2> +URL rewriting in Tyk is a powerful feature that enables the modification of incoming API request paths to match the expected endpoint format of your backend services. This process is accomplished by using regular expressions (regexes) to identify and capture specific segments of the request URL, which can then be rearranged or augmented to construct the desired endpoint URL. + +The flexibility of Tyk's URL rewriting mechanism allows for conditional rewrites based on the presence of certain parameters within the request, ensuring that the rewrite logic is applied only when appropriate. This allows for granular redirection of requests, for example to direct certain users to a beta service while others are sent to the production version. + +By employing URL rewriting, Tyk facilitates seamless communication between client applications and backend services, ensuring that API requests are efficiently routed and processed. This feature is instrumental in maintaining a clean and organized API architecture, while also providing the adaptability required to handle evolving backend systems. + +### Use Cases + +#### Internal Looping + +API requests can be redirected to other endpoints or APIs deployed on Tyk using the URL rewrite functionality. This allows requests to be redirected to internal APIs that are not directly exposed on the Gateway (for example to reduce complexity of the external interface or to perform additional processing or security checks before reaching sensitive upstream APIs). We refer to this practice as [looping](/advanced-configuration/transform-traffic/looping). By performing the looping internally using the URL rewrite middleware, latency is reduced because the redirection is handled entirely within Tyk with no unnecessary external network calls. + +#### Improved Performance Optimization + +You can use URL rewriting to route traffic intelligently to particular API endpoints, distributing the processing burden evenly across your system and minimizing load on your backend resources. This reduces the chances of overwhelming individual nodes and ensures consistent performance levels throughout the entire infrastructure. + +#### Enhanced Scalability + +As your API portfolio scales, you may find yourself dealing with an ever-increasing array of unique URLs. Instead of creating separate endpoints for every permutation, URL rewriting allows you to consolidate those disparate routes into a centralised location. This simplification makes it easier to monitor and manage the overall system, ultimately enhancing its resilience and stability. + +#### Better User Experiences + +With URL rewriting, you can design cleaner, more straightforward navigation structures for your APIs, making it simpler for consumers to locate and interact with the information they require. + +### Working + +Tyk's URL rewrite middleware uses the concepts of [triggers](/transform-traffic/url-rewriting#url-rewrite-triggers) and [rules](/transform-traffic/url-rewriting#url-rewrite-rules). These can be combined in flexible ways to create sophisticated logic to direct requests made to a single endpoint to various upstream services (or other APIs internally exposed within Tyk). + +A rule is a simple pattern match - you identify the location of a `key` and define a regex (called the `pattern`). Tyk will examine the API request and compare the `key` with the `pattern`. If there is a match, the rule will pass. + +A trigger combines one or more rules with a target (or `rewriteTo`) URL. If the logical combination of the rules results in a pass outcome, then the trigger is considered to have been fired and the target URL for the request will be rewritten. + +More detail on URL Rewrite triggers and rules can be found [here](/transform-traffic/url-rewriting#url-rewriting-overview). + +If you're using Tyk OAS APIs, then you can find details and examples of how to configure the URL rewrite middleware [here](/transform-traffic/url-rewriting#url-rewriting-using-tyk-oas). + +If you're using Tyk Classic APIs, then you can find details and examples of how to configure the URL rewrite middleware [here](/transform-traffic/url-rewriting#url-rewriting-using-classic). + +{/* proposed "summary box" to be shown graphically on each middleware page + +## URL Rewrite middleware summary + - The URL Rewrite middleware is an optional stage in Tyk's API Request processing chain, sitting between the [Request Header Transform](/api-management/traffic-transformation/request-headers) and [Response Caching](/api-management/response-caching) middleware. + - URL Rewrite is configured at the per-endpoint level within the API Definition and is supported by the API Designer within the Tyk Dashboard. + - URL Rewrite can access both [session metadata](/api-management/policies#what-is-a-session-metadata) and [request context variables](/api-management/traffic-transformation/request-context-variables). */} + + +## URL Rewrite middleware + +Tyk's [URL rewrite](/transform-traffic/url-rewriting#url-rewrite-middleware) middleware uses the concepts of [triggers](#url-rewrite-triggers) and [rules](#url-rewrite-rules) to determine if the request (target) URL should be modified. These can be combined in flexible ways to create sophisticated logic to direct requests made to a single endpoint to various upstream services (or other APIs internally exposed within Tyk Gateway through [looping](/advanced-configuration/transform-traffic/looping)). + +### URL rewrite rules + +The URL rewrite middleware compares a [key](#key) with a [pattern](#pattern) to determine if there is a match; the rules define the location of the key and the structure of the pattern. + +#### Key + +The key value is the content of a field in some element of the request; as such each key has a location (which element of the request) and a name (the name of the field within that element). For example, to obtain the key value `book` from the request `GET /asset?type=book` the key location would be `query parameter` and the key name would be `type`. + +Keys can be located in the following elements of the request: +- request path / path parameter (i.e. components of the path itself) +- request header +- query parameter +- request body (payload) +- session metadata +- request context (for example to match by IP address or JWT scope) + + + + <Note> + Note that there is no key name when using the request body location, as the entire body (payload) of the request is used as the key value. + </Note> + + +When using the request header location, the key name is the normalized form of the header name: with capitalization and use of `-` as separator. For example, the header name`customer_identifier` would be identified in a rule via the key name `Customer-Identifier`. + +When using the request path location, you can use wildcards in the key name (which is the URL path) - for example `/asset/{type}/author/`. The URL rewrite middleware will treat the wildcard as a `(.*)` regex so that any value matches. The wildcard value itself will be ignored, is not extracted from the key, and is not available for use in constructing the [rewrite path](#creating-the-rewrite-path). + +#### Pattern + +The pattern takes the form of a regular expression (regex) against which the key value will be compared. + +This pattern can be a static regex or can contain dynamic variables: +- [context variables](/api-management/traffic-transformation/request-context-variables), extracted from the request at the start of the middleware chain, can be injected into the pattern regex using the `$tyk_context.` namespace +- [session metadata](/api-management/policies#what-is-a-session-metadata), from the Tyk Session Object linked to the request, can be injected into the pattern regex using the `$tyk_meta.METADATA_KEY` namespace + +Percent-encoded (URL-encoded) characters can be used in the pattern regex when the key is the request path or path parameter +- if the middleware is called with percent-encoded characters in the key, matching will first be attempted using the raw URL as provided +- if there is no match, the percent-encoded characters will be replaced with their non-encoded form (e.g. `%2D` -> `-`) and checked again + + + + <Note> + Tyk does not check all combinations of encoded and decoded characters in the same string (so `my-test%2Durl` will only be checked as `my-test%2Durl` and `my-test-url`, it will not be checked against `my%2Dtest%2Durl` or `my%2Dtest-url`). + </Note> + + +### URL rewrite triggers + +There are two types of trigger in the URL rewriter: basic and advanced. + +#### Basic trigger + +The basic trigger has a single rule for which the key location is always the request path. For the simplest case of URL rewriting, you can simply configure the `pattern` regex and `rewriteTo` target URL for this basic trigger. + +#### Advanced triggers + +One or more advanced triggers can be configured alongside the basic trigger. These are processed in the order that they are defined in the API Definition. When using the API Designer in Tyk Dashboard, advanced triggers are automatically numbered in the order they are created and will be processed in numberical order. + +Advanced triggers can have multiple rules that can be combined using a logical AND or OR operation, such that either `all` the rules must pass or `any` rule must pass for the trigger to fire. + +Within advanced triggers, but not the basic trigger, each rule can be negated such that the rule is considered to have passed if the pattern does not match the key value. + +Once an advanced trigger has fired, the middleware will apply its `rewriteTo` target and stop processing any more triggers. + + +<Note> +The basic trigger acts as a control switch for any advanced triggers that are defined for the middleware: if the basic trigger is not fired then no rewriting will take place even if an advanced trigger would have fired based on the request. +</Note> + + +### Creating the rewrite path + +Each trigger (basic or advanced) defines a `rewriteTo` target. +- if just the basic trigger is fired, then the request path (target URL) will be rewritten with the `rewriteTo` value defined in that trigger. +- if both an advanced trigger and the basic trigger are fired, then the request path will be written with the `rewriteTo` value defined for the advanced trigger. +- if the basic trigger does not fire then no rewriting will take place. + +#### Dynamic data in the rewrite path + +The `rewriteTo` values can be simple URLs or they can be dynamically created using data available to the middleware: +- context variables, which can be referenced using the `$tyk_context.` namespace +- values from the successful [pattern match](#using-data-from-the-key-in-the-rewrite-path) +- values from [key-value (KV) storage](#using-data-from-kv-storage-in-the-rewrite-path) + + + + <Note> + You can redirect to a new hostname but to do so you must provide the full URL, for example: + ``` expandable + https://my-new-target-host.com/library/service?value1=books&value2=author + ``` + </Note> + + +#### Using data from the key in the rewrite path + +For the basic trigger, each capture group you specify in the pattern regex is designated with an index (`n`), and can then be referenced in the `rewriteTo` target using the format `$n`. +- for example, if the `pattern` to be matched is `"(\w+)/(\w+)"` then the regex will attempt to capture two word groups. The first of these will be given index 1 and the second will be index 2. You can reference these in the `rewriteTo` target using `$1` and `$2` such as: `"my/service?value1=$1&value2=$2"` + +With advanced triggers, the key values used in the pattern matches for each rule are stored as context variables which can then be referenced in the `rewriteTo` target as for other context variables. + +The format for these advanced trigger context variables is: `$tyk_context.trigger-{n}-{name}-{i}` where `n` is the trigger index, `name` is the key name and `i` is the index of that match (since query strings and headers can be arrays of values). +- for example, if the first trigger fires based on a rule where the key is the query parameter "type", a context variable with the name `trigger-0-type-0` will be created and can be referenced in the `rewriteTo` target + +#### Using data from KV storage in the rewrite path + +You can retrieve a value from KV storage by including a reference in the [appropriate notation](/tyk-configuration-reference/kv-store#transformation-middleware) for the KV location where the key-value pair is stored. + +If you use a value retrieved from [Consul](/tyk-configuration-reference/kv-store#using-consul-as-a-kv-store) or [Vault](/tyk-configuration-reference/kv-store#using-vault-as-a-kv-store), this must be the <b>last</b> part in the `rewriteTo` URL. + +For example, say you have a key named `userName` with value `jo` in my Consul KV store: +- if you configure `rewriteTo` as `/my-api/users/$secret_consul.userName` this will redirect calls to `/my-api/users/jo` +- if, however, you configure my `rewriteTo` as `/my-api/users/$secret_consul.userName/contract` this will not redirect to `my-api/jo/contract` but instead the KV lookup will fail, as Tyk will check for a key named `userName/contract` in Consul (returning null), so the URL rewrite middleware will redirect to `/my-api/users` + + +This limitation does not apply to KV accessed from the other [supported KV stores](/tyk-configuration-reference/kv-store) (environment variable or Gateway `secrets`). + + +<h2 id="url-rewriting-using-tyk-oas">Using Tyk OAS</h2> +Tyk's [URL rewriter](/transform-traffic/url-rewriting#url-rewrite-middleware) uses the concepts of triggers and rules to determine if the request (target) URL should be modified. These can be combined in flexible ways to create sophisticated logic to direct requests made to a single endpoint to various upstream services (or other APIs internally exposed within Tyk). + +URL rewrite triggers and rules are explained in detail [here](/transform-traffic/url-rewriting#url-rewriting-overview). + +When working with Tyk OAS APIs the rules and triggers are configured in the [Tyk OAS API Definition](/api-management/gateway-config-tyk-oas#operation); this can be done manually within the `.json` file or from the API Designer in the Tyk Dashboard. + +If you're using the legacy Tyk Classic APIs, then check out [this](/transform-traffic/url-rewriting#url-rewriting-using-classic) page. + +### API Definition + +The design of the Tyk OAS API Definition takes advantage of the `operationId` defined in the OpenAPI Document that declares both the path and method for which the middleware should be added. Endpoint `paths` entries (and the associated `operationId`) can contain wildcards in the form of any string bracketed by curly braces, for example `/status/{code}`. These wildcards are so they are human readable and do not translate to variable names. Under the hood, a wildcard translates to the β€œmatch everything” regex of: `(.*)`. + +The URl rewrite middleware can be added to the `operations` section of the Tyk OAS Extension (`x-tyk-api-gateway`) in your Tyk OAS API Definition for the appropriate `operationId` (as configured in the `paths` section of your OpenAPI Document). + +#### Using the basic trigger + +For the **basic trigger**, you only need to enable the middleware (set `enabled:true`) and then configure the `pattern` and the `rewriteTo` (target) URL. The design of the Tyk OAS API Definition takes advantage of the `operationID` defined in the OpenAPI Document that declares both the path and method required by the basic trigger. + +```json {hl_lines=["39-43"],linenos=true, linenostart=1} +{ + "info": { + "title": "example-url-rewrite", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "servers": [ + { + "url": "http://localhost:8181/example-url-rewrite/" + } + ], + "security": [], + "paths": { + "/json": { + "get": { + "operationId": "jsonget", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "components": { + "securitySchemes": {} + }, + "x-tyk-api-gateway": { + "info": { + "name": "example-url-rewrite", + "state": { + "active": true, + "internal": false + } + }, + "middleware": { + "operations": { + "jsonget": { + "urlRewrite": { + "enabled": true, + "pattern": "/(\\w+)/(\\w+)", + "rewriteTo": "anything?value1=$1&value2=$2" + } + } + } + }, + "server": { + "listenPath": { + "strip": true, + "value": "/example-url-rewrite/" + } + }, + "upstream": { + "url": "http://httpbin.org/" + } + } +} +``` + +In this example the basic trigger has been configured to match the path for a request to the `GET /json` endpoint against the regex `/(\w+)/(\w+)`. This is looking for two word groups in the path (after the API listen path) which, if found, will store the first string in context variable `$1` and the second in `$2`. The request (target) URL will then be rewritten to `anything?value1=$1&value2=$2`. + +If you send a request to `GET http://localhost:8181/example-url-rewrite/json/hello` + +```bash {hl_lines=["1"],linenos=true, linenostart=1} +GET /example-url-rewrite/json/hello HTTP/1.1 +User-Agent: PostmanRuntime/7.36.3 +Accept: */* +Postman-Token: 1a84a792-f0c4-4c40-932a-795485cdd252 +Host: localhost:8181 +Accept-Encoding: gzip, deflate, br +Connection: keep-alive +``` + +The URL rewrite middleware will match the pattern: +`/json/hello` -> `/(\w+)/(\w+)` -> `$1` will take the value `json` and `$2` will take the value `hello` + +It will then rewrite the target URL to `/anything?value1=json&value2=hello` and `httpbin.org` will respond with: + +```bash {hl_lines=["13-16", "31"],linenos=true, linenostart=1} +HTTP/1.1 200 OK +Access-Control-Allow-Credentials: true +Access-Control-Allow-Origin: * +Content-Length: 536 +Content-Type: application/json +Date: Mon, 18 Mar 2024 17:37:40 GMT +Server: gunicorn/19.9.0 +X-Ratelimit-Limit: 0 +X-Ratelimit-Remaining: 0 +X-Ratelimit-Reset: 0 + +{ +"args": { +"value1": "json", +"value2": "hello" +}, +"data": "", +"files": {}, +"form": {}, +"headers": { +"Accept": "*/*", +"Accept-Encoding": "gzip, deflate, br", +"Host": "httpbin.org", +"Postman-Token": "1a84a792-f0c4-4c40-932a-795485cdd252", +"User-Agent": "PostmanRuntime/7.36.3", +"X-Amzn-Trace-Id": "Root=1-65f87be4-18c50d554886f46f6b73d42b" +}, +"json": null, +"method": "GET", +"origin": "::1, 85.9.213.196", +"url": "http://httpbin.org/anything?value1=json&value2=hello" +} +``` expandable +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the URL rewrite middleware. + +#### Using advanced triggers + +You can add **advanced triggers** to your URL rewriter configuration by adding the `triggers` element within the `urlRewrite` middleware configuration for the operation. + +The `triggers` element is an array, with one entry per advanced trigger. For each of those triggers you configure: +- `condition` to set the logical condition to be applied to the rules (`any` or `all`) +- `rules` a list of rules for the trigger +- `rewriteTo` the address to which the (target) URL should be rewritten if the trigger fires + +The rules are defined using this format: +```yaml +{ + "in": key_location, + "name": key_name, + "pattern": pattern, + "negate": true/false //set to true to trigger if pattern does not match +} +``` expandable + +Key locations are encoded as follows: +- `header` - request header parameter +- `query` - query parameter +- `path` - path parameter (i.e. components of the path itself) +- `sessionMetadata` - session metadata +- `requestBody`- request body +- `requestContext`- request context + +For example: + +```json {hl_lines=["31-67"],linenos=true, linenostart=1} +{ + "info": { + "title": "example-url-rewrite2", + "version": "1.0.0" + }, + "openapi": "3.0.3", + "paths": { + "/json": { + "get": { + "operationId": "jsonget", + "responses": { + "200": { + "description": "" + } + } + } + } + }, + "components": {}, + "x-tyk-api-gateway": { + "info": { + "name": "example-url-rewrite2", + "state": { + "active": true, + "internal": false + } + }, + "middleware": { + "operations": { + "jsonget": { + "urlRewrite": { + "enabled": true, + "pattern": "/(\\w+)/(\\w+)", + "rewriteTo": "anything?value1=$1&value2=$2", + "triggers": [ + { + "condition": "all", + "rewriteTo": "anything?value1=$1&query=$tyk_context.trigger-0-numBytes-0", + "rules": [ + { + "in": "query", + "pattern": "[0-9]+", + "negate": false, + "name": "numBytes" + }, + { + "in": "header", + "pattern": "true", + "negate": true, + "name": "x-bytes" + } + ] + }, + { + "condition": "any", + "rewriteTo": "bytes/$tyk_context.trigger-1-numBytes-0", + "rules": [ + { + "in": "query", + "pattern": "[0-9]+", + "negate": false, + "name": "numBytes" + } + ] + } + ] + } + } + } + }, + "server": { + "listenPath": { + "strip": true, + "value": "/example-url-rewrite2/" + } + }, + "upstream": { + "url": "http://httpbin.org/" + } + } +} +``` expandable +In this example, the basic trigger is configured as before, but two advanced triggers have been added. + +The first advanced trigger will fire if the request has this configuration: +- query parameter `numBytes` is provided with a numeric value, AND +- header parameter `x-bytes` is *not* set to `true` (note that `negate` is set to `true` in this rule) + +Such a request will be redirected to `/anything` passing two query parameters +- `value1` with the first string matched in the basic trigger (i.e. `json`) +- `query` with the value provided in the `numBytes` query parameter + +The second advanced trigger will fire if the first doesn't and if this condition is met: +- query parameter `numBytes` is provided with a numeric value + +Such a request will be redirected to `/bytes/{numBytes}`, which will return `numBytes` random bytes from `httpbin.org`. + +If neither advanced trigger fires, then the basic trigger will redirect the request to `/anything?value1=json&value2=hello` as before. + +The configuration above is a complete and valid Tyk OAS API Definition that you can import into Tyk to try out the URL rewrite middleware. + +### API Designer + +Adding and configuring the URL rewrite feature to your API endpoints is easy when using the API Designer in the Tyk Dashboard, simply follow these steps: + +1. **Add an endpoint** + + From the **API Designer** add an endpoint that matches the path and method to which you want to apply the middleware. + + <img src="/img/dashboard/api-designer/tyk-oas-no-endpoints.png" alt="Tyk OAS API Designer showing no endpoints created" /> + + <img src="/img/dashboard/api-designer/tyk-oas-add-endpoint.png" alt="Adding an endpoint to an API using the Tyk OAS API Designer" /> + + <img src="/img/dashboard/api-designer/tyk-oas-no-middleware.png" alt="Tyk OAS API Designer showing no middleware enabled on endpoint" /> + +2. **Select the URL Rewrite middleware** + + Select **ADD MIDDLEWARE** and choose the **URL Rewrite** middleware from the *Add Middleware* screen. + + <img src="/img/dashboard/api-designer/tyk-oas-add-url-rewrite.png" alt="Adding the URL Rewrite middleware" /> + +3. **Configure the basic trigger** + + Add the match pattern and the new URL to configure the basic trigger rule. + + <img src="/img/dashboard/api-designer/tyk-oas-url-rewrite-basic.png" alt="Configuring the rewrite rule for the Basic Trigger" /> + +4. **Optionally configure advanced triggers** + + You can optionally apply advanced triggers by selecting **ADD TRIGGER** for each trigger you wish to configure. For each advanced trigger you can add one or more rules, selecting **ADD RULE** to add the second, third, etc. + + <img src="/img/dashboard/api-designer/tyk-oas-url-rewrite-advanced.png" alt="Configuring the rewrite rules for Advanced Triggers" /> + +5. **Save the API** + + Select **ADD MIDDLEWARE** to save the middleware configuration. Remember to select **SAVE API** to apply the changes. + +<h2 id="url-rewriting-using-classic">Using Classic</h2> +Tyk's [URL rewriter](/transform-traffic/url-rewriting#url-rewrite-middleware) uses the concepts of triggers and rules to determine if the request (target) URL should be modified. These can be combined in flexible ways to create sophisticated logic to direct requests made to a single endpoint to various upstream services (or other APIs internally exposed within Tyk). + +URL rewrite triggers and rules are explained in detail [here](/transform-traffic/url-rewriting#url-rewriting-overview). + +When working with Tyk Classic APIs the rules and triggers are configured in the Tyk Classic API Definition; this can be done manually within the `.json` file or from the API Designer in the Tyk Dashboard. + +If you want to use dynamic data from context variables, you must [enable](/api-management/traffic-transformation/request-context-variables#enabling-context-variables-for-use-with-tyk-classic-apis) context variables for the API to be able to access them from the request header transform middleware. + +If you're using the newer Tyk OAS APIs, then check out [this](/transform-traffic/url-rewriting#url-rewriting-using-tyk-oas) page. + +If you're using Tyk Operator then check out the [configuring the URL rewriter in Tyk Operator](#tyk-operator) section below. + +### API Definition + +To configure the URL rewriter you must add a new `url_rewrites` object to the `extended_paths` section of your API definition, for example: + +```json +{ + "url_rewrites": [ + { + "path": "books/author", + "method": "GET", + "match_pattern": "(\w+)/(\w+)", + "rewrite_to": "library/service?value1=$1&value2=$2" + } + ] +} +``` expandable + +In this example the basic trigger has been configured to match the path for a request to the `GET /books/author` endpoint against the pure regex `(\w+)/(\w+)`. This is looking for two word groups in the path which, if found, will store the first string (`books`) in context variable `$1` and the second (`author`) in `$2`. The request (target) URL will then be rewritten to `library/service?value1=books&value2=author` ready for processing by the next middleware in the chain. + +You can add advanced triggers to your URL rewriter configuration by adding the `triggers` element within the `url_rewrites` object. + +The `triggers` element is an array, with one entry per advanced trigger. For each of those triggers you configure: +- `on` to set the logical condition to be applied to the rules (`any` or `all`) +- `options` a list of rules for the trigger +- `rewrite_to` the address to which the (target) URL should be rewritten if the trigger fires + +The rules are defined using this format: +```yaml +{ + key_location: { + key_name: { + "match_rx": pattern + "reverse": true/false (set to true to trigger if pattern does not match) + } + } +} + +Key locations are encoded as follows: +- `header_matches` - request header parameter +- `query_val_matches` - query parameter +- `path_part_matches` - path parameter (i.e. components of the path itself) +- `session_meta_matches` - session metadata +- `payload_matches`- request body +- `request_context_matches`- request context + +For example: + +```json expandable +{ + "url_rewrites": [ + { + "path": "books/author", + "method": "GET", + "match_pattern": "(\w+)/(\w+)", + "rewrite_to": "library/service?value1=$1&value2=$2", + "triggers": [ + { + "on": "any", + "options": { + "query_val_matches": { + "genre": { + "match_rx": "fiction", + "reverse": false + } + } + }, + "rewrite_to": "library/service/author?genre=$tyk_context.trigger-0-genre-0" + }, + { + "on": "all", + "options": { + "header_matches": { + "X-Enable-Beta": { + "match_rx": "true", + "reverse": false + } + }, + "session_meta_matches": { + "beta_enabled": { + "match_rx": "true", + "reverse": false + } + } + }, + "rewrite_to": "https://beta.library.com/books/author" + } + ] + } + ] +} +``` + +In this example, the basic trigger is configured as before, but two advanced triggers have been added. + +The first advanced trigger has this configuration: +- key location is query parameter +- key name is genre +- pattern is fiction + +So if a `GET` request is made to `/books/author?genre=fiction` the advanced trigger will fire and the URL will be rewritten to `library/service/author?genre=fiction`. + +The second advanced trigger has this configuration: +- rule condition: ALL +- rule 1 + - key location is header parameter + - key name is `X-Enable-Beta` + - pattern is `true`` +- rule 2 + - key location is session metadata + - key name is `beta_enabled` + - pattern is `true` + +So if a request is made to `GET /books/author` with a header `"X-Enable-Beta":"true"` and, within the session metadata, `"beta_enabled":"true"` the second advanced trigger will fire and the URL will be written to `https://beta.library.com/books/author` taking the request to a different upstream host entirely. + +### API Designer + +You can use the API Designer in the Tyk Designer to configure the URL rewrite middleware for your Tyk Classic API by following these steps. + +1. **Add an endpoint for the path and select the URL rewrite plugin** + + From the **Endpoint Designer** add an endpoint that matches the path you want to rewrite. Select the **URL Rewrite** plugin. + + <img src="/img/2.10/url_rewrite.png" alt="Endpoint designer" /> + +2. **Configure the basic trigger** + + Add the regex capture groups and the new URL to the relevant sections. + + <img src="/img/2.10/url_rewrite_settings.png" alt="URL rewrite configuration" /> + +3. **Configure an advanced trigger** + + You can optionally configure advanced triggers by using the **Create Advanced Trigger** option from the **URL Rewriter** plugin. + + You will see a screen like this: + + <img src="/img/2.10/url_re-write_advanced.png" alt="URL rewrite add trigger" /> + + When triggers are added, you can edit or remove them inside the **Advanced URL rewrite** section: + + <img src="/img/2.10/url_rewrite-advanced-edit.png" alt="URL rewrite list trigger" /> + +4. **Save the API** + + Use the *save* or *create* buttons to save the changes and activate the middleware. + +### Tyk Operator + +The process for configuring the URL rewriter in Tyk Operator is similar to that explained in configuring the URL rewriter in the Tyk Classic API Definition. To configure the URL rewriter you must add a new `url_rewrites` object to the `extended_paths` section of your API definition. + +The example API Definition provides the corresponding custom resource configuration for the Tyk Classic API Definition example, configuring an API to listen on path `/url-rewrite` and forward requests upstream to http://httpbin.org. The URL rewrites middleware would match the path for a request to the `GET /anything/books/author` endpoint against the pure regex `/anything/(\w+)/(\w+)`. The request (target) URL will then be rewritten to `/anything/library/service?value1=$1&value2=$2`. + +```yaml {linenos=true, linenostart=1, hl_lines=["26-31"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: url-rewrite +spec: + name: URL Rewrite + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /url-rewrite + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + url_rewrites: + - path: /anything/books/author + match_pattern: /anything/(\w+)/(\w+) + method: GET + rewrite_to: /anything/library/service?value1=$1&value2=$2 + triggers: [] +``` + +URL Rewrite Triggers can be specified in a similar way. The Tyk Operator example below is the equivalent for the advanced triggers example included in the configuring the URL rewriter in the Tyk Classic API Definition section above. + +```yaml {linenos=true, linenostart=1, hl_lines=["26-49"]} expandable +apiVersion: tyk.tyk.io/v1alpha1 +kind: ApiDefinition +metadata: + name: url-rewrite-advanced +spec: + name: URL Rewrite Advanced + use_keyless: true + protocol: http + active: true + proxy: + target_url: http://httpbin.org + listen_path: /url-rewrite + strip_listen_path: true + version_data: + default_version: Default + not_versioned: true + versions: + Default: + name: Default + use_extended_paths: true + paths: + black_list: [] + ignored: [] + white_list: [] + extended_paths: + url_rewrites: + - path: /anything/books/author + match_pattern: /anything/(\w+)/(\w+) + method: GET + rewrite_to: /anything/library/service?value1=$1&value2=$2 + triggers: + - "on": "any" + "rewrite_to": "library/service/author?genre=$tyk_context.trigger-0-genre-0" + "options": + "query_val_matches": + "genre": + "match_rx": "fiction" + "reverse": false + - "on": "all" + "options": + "header_matches": + "X-Enable-Beta": + "match_rx": "true" + "reverse": false + "session_meta_matches": + "beta_enabled": + "match_rx": "true" + "reverse": false + "rewrite_to": "https://beta.library.com/books/author" +``` + +For further examples check out the [internal looping](/api-management/automations/operator#internal-looping-with-tyk-operator) page. + + diff --git a/troubleshooting/tyk-dashboard/api-catalogue-not-found-error-developer-portal.mdx b/troubleshooting/tyk-dashboard/api-catalogue-not-found-error-developer-portal.mdx new file mode 100644 index 00000000..8493cf4f --- /dev/null +++ b/troubleshooting/tyk-dashboard/api-catalogue-not-found-error-developer-portal.mdx @@ -0,0 +1,22 @@ +--- +title: "β€œAPI catalog not foundβ€œ error in the Developer Portal" +order: 5 +noindex: True +sidebarTitle: "β€œAPI catalogue not foundβ€œ error in the Developer Portal" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +### Description + +Users receive an "API catalog not found" error message when they visit the Developer Portal. + +### Cause + +The catalog hasn't been initialised yet. + +### Solution + +We have [tutorials](/getting-started/tutorials/publish-api) on how to publish to a developer portal using API catalogs. It may also be worth checking that the key generated by the policy hasn't expired. \ No newline at end of file diff --git a/troubleshooting/tyk-dashboard/cors-issues-on-developer-portal.mdx b/troubleshooting/tyk-dashboard/cors-issues-on-developer-portal.mdx new file mode 100644 index 00000000..8bb27b7a --- /dev/null +++ b/troubleshooting/tyk-dashboard/cors-issues-on-developer-portal.mdx @@ -0,0 +1,37 @@ +--- +title: "CORS issues on developer portal" +order: 6 +noindex: True +sidebarTitle: "CORS issues on developer portal" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +### Description + +You get following errors in the browser console: +``` +Access to fetch at 'https://<Your Gateway URL>/<Your API>/' from origin 'https://<Your Dashboard URL>' has been blocked by CORS policy: Response to preflight request doesn't pass access control check: No 'Access-Control-Allow-Origin' header is present on the requested resource. If an opaque response serves your needs, set the request's mode to 'no-cors' to fetch the resource with CORS disabled. +``` +``` +POST https://<Your Gateway URL>/<Your API>/ net::ERR_FAILED +``` + +### Cause + +The CORS middleware in the Gateway is blocking this request. This can happen when the CORS settings of the API are not enabled or misconfigured for the developer portal. + +### Solution + +Make sure that your CORS in the `Advanced Options` of the API is enabled and the settings are correct. This means: + - `Allowed Origins` should allow the developer portal domain + - `Allowed Methods` should allow all methods needed for API documentation (at least `GET` and `POST`) + - `Allowed Headers` should allow at least `Origin`, `Content-Type` and for authenticated requests the authorization header (e.g. `Authorization`) + + > **Note:** When creating a new API with Dashboard v3.1 and higher the CORS settings will be prefilled with some default values (but **disabled** by default). + + You can learn more about CORS on this pages: + - [CORS in API Definition](/api-management/gateway-config-tyk-classic#cross-origin-resource-sharing-cors) + - [How to setup CORS](/api-management/troubleshooting-debugging#how-to-setup-cors) diff --git a/troubleshooting/tyk-dashboard/not-found-error-developer-portal.mdx b/troubleshooting/tyk-dashboard/not-found-error-developer-portal.mdx new file mode 100644 index 00000000..80769510 --- /dev/null +++ b/troubleshooting/tyk-dashboard/not-found-error-developer-portal.mdx @@ -0,0 +1,24 @@ +--- +title: "β€œNot Found” error in the Developer Portal" +order: 5 +noindex: True +sidebarTitle: "β€œNot Foundβ€œ error in the Developer Portal" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +### Description + +When you attempt to access the Developer Portal (`https://xxxxxx:3000/portal`), you receive a `Not Found` error message + +### Cause + +The portal may not have been configured or may have been set up with the wrong domain name. + +### Solution + +You should make sure that your portal has been configured to use the correct domain name in your `tyk_analytics.conf`. Once this change has been made you may need to restart the Dashboard process so as to avoid having to reconfigure the Gateway as well. + +You should also look at the [portal tutorial](/getting-started/tutorials/publish-api) for creating an API and publishing it to your Portal. diff --git a/troubleshooting/tyk-dashboard/receive-csrf-error-developer-portal.mdx b/troubleshooting/tyk-dashboard/receive-csrf-error-developer-portal.mdx new file mode 100644 index 00000000..9d8109f1 --- /dev/null +++ b/troubleshooting/tyk-dashboard/receive-csrf-error-developer-portal.mdx @@ -0,0 +1,27 @@ +--- +title: "Receive a CSRF error in the Developer Portal" +order: 5 +noindex: True +sidebarTitle: "Receive a CSRF error in the Developer Portal" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +### Description + +When the user attempts to log into the Developer Portal a CSRF error (or some variant of this error such as `Forbidden - CSRF token invalid`) is displayed on the page. + +### Cause + +Most probably the portal has yet to be activated. Common reasons for this are: + +1. The `CNAME` was not set in the dashboard. Without a `CNAME`, the system won't react to a new domain name. +2. There were no active APIs set up under the account which means that the account was not active for a portal either and essentially incapable of proxying traffic. + +The use of an incorrect signup form can also cause this issue. + +### Solution + +Users must make sure that they add a `CNAME` and an active API to the Dashboard. If the form will require TLS, the user will need to set this up for their custom load balancer. To add this to a cloud instance, a copy of the TLS certificate and the private key file will need to be sent to Tyk Support. \ No newline at end of file diff --git a/tyk-apis.mdx b/tyk-apis.mdx new file mode 100644 index 00000000..b89f0c8f --- /dev/null +++ b/tyk-apis.mdx @@ -0,0 +1,80 @@ +--- +title: "Tyk APIs Hub" +'og:description': "The Tyk API Hub. In this section you will find OpenAPI docs for the APIs for Tyk Gateway, Tyk Dashboard and other Tyk products" +sidebarTitle: "Overview" +tags: ['swagger', 'REST', 'Tyk APIs', 'Tyk OpenAPI Specs APIs', 'OPenAPI', 'OAS'] +--- + +import { ResponsiveGrid } from '/snippets/ResponsiveGrid.mdx'; + +All Tyk components are designed with an API that allows you to configure and control the product. On this page you will find documentation for all of these APIs, designed to help you to integrate Tyk into your software stack and then manage your API ecosystem efficiently. + +## Tyk Postman Collections + +Explore our official Tyk Postman collections for a hands-on, practical approach to using our APIs. +These provide an interactive way to explore and test Tyk components' APIs. They are designed to help you quickly get +started and understand the capabilities of our products. + + +<ResponsiveGrid> + + +<Card title="Postman" href="https://www.postman.com/tyk-technologies/workspace/379673ec-4cc5-4b8e-bef5-8a6a988071cb/overview" img="/img/logos/Postman-logo-vertical-orange-2021.svg"> +Tyk Public Workspace +</Card> + + +</ResponsiveGrid> + +## Tyk Enterprise + +Here you can access detailed documentation enabling you to unlock the full capabilities of Tyk's enterprise products. This API documentation provides all the +information you need to integrate Tyk's products seamlessly into your projects. + +<ResponsiveGrid> + + +<Card title="OpenAPI" href="/tyk-dashboard-api" img="/img/logos/tyk-logo-selfmanaged.svg"> +Manage the Tyk platform +</Card> + + +</ResponsiveGrid> + +<ResponsiveGrid> + + +<Card title="OpenAPI" href="/dashboard-admin-api"> +Dashboard Admin API +<br /><br /> +*Provision & administrate Tyk Dashboard* +</Card> + +<Card title="OpenAPI" href="/tyk-mdcb-api"> +MDCB API +<br /><br /> +*Monitor & diagnose Tyk data planes* +</Card> + +<Card title="OpenAPI" href="/product-stack/tyk-enterprise-developer-portal/api-documentation/tyk-edp-api"> +Enterprise Portal API +<br /> +<br /> +*Configure & control Tyk Enterprise Developer Portal* +</Card> + + +</ResponsiveGrid> + + +## Tyk Open Source + +<ResponsiveGrid> + + +<Card title="OpenAPI" href="/tyk-gateway-api" img="/img/logos/tyk-logo-opensource.svg"> +Configure & control Tyk GW +</Card> + + +</ResponsiveGrid> \ No newline at end of file diff --git a/tyk-apis/tyk-dashboard-api/manage-key-requests.mdx b/tyk-apis/tyk-dashboard-api/manage-key-requests.mdx new file mode 100644 index 00000000..7af609a5 --- /dev/null +++ b/tyk-apis/tyk-dashboard-api/manage-key-requests.mdx @@ -0,0 +1,246 @@ +--- +title: "Manage Key Requests" +order: 9 +noindex: True +sidebarTitle: "Portal Key Requests" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +### List key requests + +| **Property** | **Description** | +| :------------ | :---------------------------------------------------- | +| Resource URL | `/api/portal/requests` | +| Method | GET | +| Type | None | +| Body | None | +| Param | `p={page-num}` (optional, set to `-1` for no paging) | +| Param | `approved={true or false}` (optional, returns `false` for Pending Key Requests, or `true` for Approved Key Requests) | + + + +### Sample Request + +```{.copyWrapper} +GET /api/portal/requests?p=0 HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` expandable +{ + "Data": [{ + "id": "5ad097c87af3f40001b27f40", + "org_id": "xxxxxxxxxxxxxxxxxxxxx", + "for_plan": "5a9847db3602190001f44427", // deprecated + "apply_policies": ["5a9847db3602190001f44427"], + "by_user": "5a9ea3a019efc400011107ae", + "fields": {}, + "approved": false, + "date_created": "2018-04-13T11:43:04.698Z", + "version": "v2", + "jwt_secret": "", + "portal_developer": { + "id": "5a9ea3a019efc400011107ae", + "email": "xxx@zzz.io", + "date_created": "2018-03-06T14:20:16.876Z", + "inactive": false, + "org_id": "57ecd735f467ac0001000003", + "keys": { + "<key-id>": ["<policy-id>"], + }, + "subscriptions": { + "<policy-id>": "<key-id>", + }, + "fields": {}, + "nonce": "", + "sso_key": "" + }, + "catalogue_entry": { + "name": "New catalogue", + "short_description": "", + "long_description": "", + "show": true, + "api_id": "", + "policy_id": "5a9847db3602190001f44427", + "documentation": "5acd2f1b4242d10001ab2cbc", + "version": "v2", + "config": { + "id": "", + "org_id": "", + "signup_fields": [], + "key_request_fields": [], + "require_key_approval": false, + "redirect_on_key_request": false, + "redirect_to": "", + "disable_login": false, + "disable_signup": false, + "disable_auto_login": false, + "catalogue_login_only": false, + "mail_options": { + "mail_from_name": "", + "mail_from_email": "", + "email_copy": { + "welcome_email": { + "enabled": false, + "subject": "", + "body": "", + "sign_off": "", + "hide_token_data": false + }, + "key_email": { + "enabled": false, + "subject": "", + "body": "", + "sign_off": "", + "hide_token_data": false + } + } + }, + "override": false + } + } + + }], + "Pages": 0 +} +``` + +### Get a specific key request + +| **Property** | **Description** | +| :------------ | :--------------------------- | +| Resource URL | `/api/portal/requests/{id}` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +GET /api/portal/requests/554c789030c55e4ca0000002 HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` expandable +{ + "id": "554c789030c55e4ca0000002", + "org_id": "53ac07777cbb8c2d53000002", + "by_user": "554c733a30c55e4b16000002", + "fields": {}, + "approved": true, + "date_created": "2015-05-08T04:49:20.992-04:00", + "version": "v2", + "for_plan": "554c789030c55e4ca0101002", // deprecated + "apply_policies": ["554c789030c55e4ca0101002"] +} +``` + +### Approve a key request + +| **Property** | **Description** | +| :------------ | :----------------------------------- | +| Resource URL | `/api/portal/requests/approve/{id}` | +| Method | PUT | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +PUT /api/portal/requests/approve/554c789030c55e4ca0000002 HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response: + +``` +{ + "RawKey":"53ac07777cbb8c2d5300000215811f02c21540dd5257eb68d3d73f35" +} +``` + +### Create a key request + +Key requests are an easy way to associate developer accounts with new policies, they do not need to be linked to API Catalog entries, they represent an instruction to Tyk to combine a generate a token with a specific policy ID, and to associate the token and policy with a specific developer account. + +**It is now required to pass a version parameter of `v2` and `for_plan` if you wish to associate a key request with a policy ID** legacy key requests will continue to work, but could cause issues as this version is deprecated in future releases. + +By default, all key requests created for new catalog entries will be version 2. + +| **Property** | **Description** | +| :------------ | :---------------------- | +| Resource URL | `/api/portal/requests` | +| Method | POST | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} expandable +POST /api/portal/requests HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "by_user": "554c733a30c55e4b16000002", + "date_created": "2015-05-08T04:49:20.992-04:00", + "fields": { + "custom1": "sdf", + "custom2": "sdf" + }, + "for_plan": "554c789030c55e4ca0101002", // deprecated + "apply_policies": ["554c789030c55e4ca0101002"], + "version": "v2" +} +``` + +#### Sample Response + +``` +{ + "Status":"OK", + "Message":"554c789030c55e4ca0000002", + "Meta":"" +} +``` + +### Delete a specific key request + +| **Property** | **Description** | +| :------------ | :--------------------------- | +| Resource URL | `/api/portal/requests/{id}` | +| Method | DELETE | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +DELETE /api/portal/requests/554c789030c55e4ca0000002 HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` +{ + "Status":"OK", + "Message":"Data deleted", + "Meta":"" +} +``` diff --git a/tyk-apis/tyk-dashboard-api/portal-policies.mdx b/tyk-apis/tyk-dashboard-api/portal-policies.mdx new file mode 100644 index 00000000..1b90a2c4 --- /dev/null +++ b/tyk-apis/tyk-dashboard-api/portal-policies.mdx @@ -0,0 +1,315 @@ +--- +title: "Portal Policies" +order: 2 +noindex: True +sidebarTitle: "Portal Policies" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +### Get List of Policies + +| **Property** | **Description** | +| :------------ | :----------------------- | +| Resource URL | `/api/portal/policies/` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +GET /api/portal/policies/ HTTP/1.1 +Host: localhost:3000 +authorization: 7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` expandable +{ + "Data": [ + { + "_id": "56b9fed54e86e40001000002", + "access_rights": { + "35447b1469df4e846894b1e87372f6d7": { + "allowed_urls": [ + { + "methods": [ + "GET" + ], + "url": "/some_resources" + }, + { + "methods": [ + "POST" + ], + "url": "/some_resource/(.*)" + } + ], + "apiid": "35447b1269df4e846894b7e87312f6d7", + "apiname": "My API", + "versions": [ + "Default" + ] + } + }, + "active": true, + "date_created": "0001-01-01T00:00:00Z", + "hmac_enabled": false, + "is_inactive": false, + "key_expires_in": 0, + "name": "My Policy", + "org_id": "5629ca78eebd180001000001", + "per": 1, + "quota_max": -1, + "quota_renewal_rate": 60, + "rate": 1000, + "tags": [] + } + ], + "Pages": 0 +} +``` + +Notice that the `apiid` field is different than the rest of the policy definitions! (See [GitHub issue 192][1]) + +### Search list of Policies + +| **Property** | **Description** | +| :------------ | :----------------------- | +| Resource URL | `/api/portal/policies/search` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +GET /api/portal/policies/search?q=Policy+name HTTP/1.1 +Host: localhost:3000 +authorization: 7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +Similar to Policies list endpoint + +### Retrieve a single policy by ID + +| **Property** | **Description** | +| :------------ | :--------------------------- | +| Resource URL | `/api/portal/policies/{id}` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +#### Sample request + +```{.copyWrapper} +GET /api/portal/policies/56b9fed54e86e40001000002 HTTP/1.1 +Host: localhost +authorization: 7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` expandable +{ + "_id": "56b9fed54e86e40001000002", + "access_rights": { + "35447b1469df4e846894b1e87372f6d7": { + "allowed_urls": [ + { + "methods": [ + "GET", + ], + "url": "/some_resources" + }, + { + "methods": [ + "POST" + ], + "url": "/some_resource/(.*)" + }, + ], + "apiid": "35447b1269df4e846894b7e87312f6d7", + "apiname": "My API", + "versions": [ + "Default" + ] + } + }, + "active": true, + "date_created": "0001-01-01T00:00:00Z", + "hmac_enabled": false, + "is_inactive": false, + "key_expires_in": 0, + "name": "My Policy", + "org_id": "5629ca78eebd180001000001", + "per": 1, + "quota_max": -1, + "quota_renewal_rate": 60, + "rate": 1000, + "tags": [] +} +``` + +### Create Policy Definition + +Creating policy definitions is slightly different to the core API, API definitions are wrapped inside an `api_definition` field and event handlers, such as webhooks are not embedded in the main `api_defintion` object (though they can be), webhooks are instead appended as references into the `hook_references` field, the API will embed the correct webhook data into the event handler interface. + +| **Property** | **Description** | +| :------------ | :-------------------------- | +| Resource URL | `/api/portal/policies/` | +| Method | POST | +| Type | None | +| Body | Advanced Policy Definition | +| Param | None | + +#### Sample Request + +```{.copyWrapper} expandable +POST /api/portal/policies/ HTTP/1.1 +Host: localhost:3000 +Connection: keep-alive +Content-Type: application/json +Content-Length: <body length> +authorization: 7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "last_updated": "0001-01-01T00:00:00Z", + "rate": 1000, + "per": 60, + "quota_max": -1, + "quota_renews": 1481546970, + "quota_remaining": 0, + "quota_renewal_rate": 60, + "access_rights": { + "35447b1469df4e846894b1e87372f6d7": { + "apiid": "35447b1469df4e846894b1e87372f6d7", + "api_name": "My API", + "versions": ["Default"] + } + }, + "name": "My Policy", + "active": true +} +``` + +#### Sample Response + +``` +{ + "Status": "OK", + "Message": "56b9fed54e86e40001000002", + "Meta": "null" +} +``` + +### Update Policy Definition + +| **Property** | **Description** | +| :------------ | :--------------------------- | +| Resource URL | `/api/portal/policies/{id}` | +| Method | PUT | +| Type | None | +| Body | Advanced Policy Definition | +| Param | None | + +#### Sample Request + +```{.copyWrapper} expandable +PUT /api/portal/policies/56b9fed54e86e40001000002 HTTP/1.1 +Host: localhost:3000 +Connection: keep-alive +Content-Type: application/json +Content-Length: <body length> +authorization: 7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "_id": "56b9fed54e86e40001000002", + "id": "", + "org_id": "589b4be9dbd34702ee2ed8c5", + "rate": 1000, + "per": 60, + "quota_max": -1, + "quota_renewal_rate": 60, + "access_rights": { + "35447b1469df4e846894b1e87372f6d7": { + "apiid": "35447b1469df4e846894b1e87372f6d7", + "api_name": "My API", + "versions": ["Default"], + "allowed_urls": [] + } + }, + "hmac_enabled": false, + "active": true, + "name": "My Updated Policy", + "is_inactive": false, + "date_created": "0001-01-01T00:00:00Z", + "tags": [], + "key_expires_in": 0, + "partitions": { + "quota": false, + "rate_limit": false, + "acl": false + }, + "last_updated": "0001-01-01T00:00:00Z" +} +``` + +#### Sample Response + +``` +{ + "Status": "OK", + "Message": "Data updated", + "Meta": "" +} +``` + +### Delete Policy Definition + +| **Property** | **Description** | +| :------------ | :--------------------------- | +| Resource URL | `/api/portal/policies/{id}` | +| Method | DELETE | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +DELETE /api/portal/policies/56b9fed54e86e40001000002 HTTP/1.1 +Host: localhost:3000 +authorization: 7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` +{ + "Status": "OK", + "Message": "Data deleted", + "Meta": null +} +``` + +### Graphql API + +Presently, the Tyk Dashboard uses the GraphQL API for policies. + +| **Method** | **URL** | **Description** | +| :---------- | :------------- | :--------------------------- | +| POST | `/graphql` | GraphQL query endpoint | +| GET | `/playground` | Dashboard Graphql Playground - where you could see docs and run queries | + + + [1]: https://github.com/TykTechnologies/tyk/issues/192 \ No newline at end of file diff --git a/tyk-apis/tyk-portal-api/portal-configuration.mdx b/tyk-apis/tyk-portal-api/portal-configuration.mdx new file mode 100644 index 00000000..f67f76f5 --- /dev/null +++ b/tyk-apis/tyk-portal-api/portal-configuration.mdx @@ -0,0 +1,595 @@ +--- +title: "Portal Configuration" +noindex: True +sidebarTitle: "Configuration" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +This section will cover the following endpoints: +* [Configuration](#configuration) +* [Menus](#menus) +* [Pages](#pages) + + +## Configuration + +### List Portal Configuration + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/configuration` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +GET /api/portal/configuration HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` expandable +{ + "id":"5cc03284d07e7f00019404b6", + "org_id":"5cc03283d07e7f00019404b3", + "signup_fields":[ + + ], + "key_request_fields":[ + + ], + "require_key_approval":false, + "redirect_on_key_request":false, + "redirect_to":"", + "enable_multi_selection":false, + "disable_login":false, + "disable_signup":false, + "disable_auto_login":false, + "catalogue_login_only":false, + "oauth_usage_limit":0, + "email":"", + "mail_options":{ + "mail_from_name":"", + "mail_from_email":"", + "email_copy":{ + "welcome_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "key_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "reset_password_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + } + } + }, + "override":false, + "HashKeys":false +} +``` + +### Create Portal Configuration + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/configuration` | +| Method | POST | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +POST /api/portal/configuration HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` +{ + Response here +} +``` + +### Update Portal Configuration + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/configuration` | +| Method | PUT | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +PUT/api/portal/configuration HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` +{ + Response here +} +``` + +## Menus + +### List Portal Menus + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/menus` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +GET /api/portal/menus HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` expandable +{ + "id": "5d12262e0313b300010eb5bb", + "menus": {}, + "is_active": true, + "org_id": "5cc03283d07e7f00019404b3" +} +``` + +### Create Portal Menus + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/menus` | +| Method | POST | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +POST /api/portal/menus HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` +{ + Response here +} +``` + +### Update Portal Menus + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/menus` | +| Method | PUT | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +PUT /api/portal/menus HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` +{ + Response here +} +``` + + + +## Pages + +### List Portal Pages + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/pages` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +GET /api/portal/pages HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` expandable +{ + "Data":[ + { + "id":"5cc03284d07e7f00019404b5", + "title":"Tyk Developer Portal", + "slug":"home", + "template_name":"", + "fields":{ + "JumboCTALink":"#cta", + "JumboCTALinkTitle":"Your awesome APIs, hosted with Tyk!", + "JumboCTATitle":"Tyk Developer Portal", + "PanelOneContent":"Panel 1 content.", + "PanelOneLink":"#panel1", + "PanelOneLinkTitle":"Panel 1 Button", + "PanelOneTitle":"Panel 1 Title", + "PanelThereeContent":"", + "PanelThreeContent":"Panel 3 content.", + "PanelThreeLink":"#panel3", + "PanelThreeLinkTitle":"Panel 3 Button", + "PanelThreeTitle":"Panel 3 Title", + "PanelTwoContent":"Panel 2 content.", + "PanelTwoLink":"#panel2", + "PanelTwoLinkTitle":"Panel 2 Button", + "PanelTwoTitle":"Panel 2 Title", + "SubHeading":"Sub Header" + }, + "org_id":"5cc03283d07e7f00019404b3", + "is_homepage":true, + "page_settings":{ + + } + } + ], + "Pages":1 +} +``` + +### Create Portal Pages + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/pages` | +| Method | POST | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +POST /api/portal/pages HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` +{ + Response here +} +``` + +### Get a Portal Page + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/pages/{id}` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +GET /api/portal/pages/{id} HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` expandable +{ + "id":"5cc03284d07e7f00019404b5", + "title":"Tyk Developer Portal", + "slug":"home", + "template_name":"", + "fields":{ + "JumboCTALink":"#cta", + "JumboCTALinkTitle":"Your awesome APIs, hosted with Tyk!", + "JumboCTATitle":"Tyk Developer Portal", + "PanelOneContent":"Panel 1 content.", + "PanelOneLink":"#panel1", + "PanelOneLinkTitle":"Panel 1 Button", + "PanelOneTitle":"Panel 1 Title", + "PanelThereeContent":"", + "PanelThreeContent":"Panel 3 content.", + "PanelThreeLink":"#panel3", + "PanelThreeLinkTitle":"Panel 3 Button", + "PanelThreeTitle":"Panel 3 Title", + "PanelTwoContent":"Panel 2 content.", + "PanelTwoLink":"#panel2", + "PanelTwoLinkTitle":"Panel 2 Button", + "PanelTwoTitle":"Panel 2 Title", + "SubHeading":"Sub Header" + }, + "org_id":"5cc03283d07e7f00019404b3", + "is_homepage":true, + "page_settings":{ + + } +} +``` + +### Update a Portal Page + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/pages/{id}` | +| Method | PUT | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +PUT /api/portal/pages/{id} HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` +{ + Response here +} +``` + +### Delete a Portal Page + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/pages/{id}` | +| Method | DELETE | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +DELETE /api/portal/pages/{id} HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` +{ + Response here +} +``` +## CSS + +### List Portal CSS + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/css` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +GET /api/portal/css HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` expandable +{ + "id": "5ce3b8ffe84526000117899a", + "org_id": "5cc03283d07e7f00019404b3", + "page_css": "", + "email_css": "" +} +``` + +### Create Portal CSS + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/css` | +| Method | POST | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +POST /api/portal/css HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` +{ + Response here +} +``` + +### Update Portal CSS + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/css` | +| Method | PUT | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +PUT /api/portal/css HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` +{ + Response here +} +``` + +## JavaScript + +### List Portal JavaScript + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/js` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +GET /api/portal/js HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` +{ + "id": "5ce3b8ffe84526000117899a", + "org_id": "5cc03283d07e7f00019404b3", + "page_js": "" +} +``` + +### Create Portal JavaScript + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/js` | +| Method | POST | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} expandable +POST /api/portal/js HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "page_js": "console.log(1)" +} +``` + +#### Sample Response + +``` +{ + "Status": "OK", + "Message": "5ce3b8ffe84526000117899a", + "Meta": null +} +``` + +### Update Portal CSS + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/js` | +| Method | PUT | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} expandable +PUT /api/portal/js HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "id": "5ce3b8ffe84526000117899a", + "page_js": "console.log(1)" +} +``` + +#### Sample Response + +``` +{ + "Status": "OK", + "Message": "Data updated", + "Meta": null +} +``` + diff --git a/tyk-apis/tyk-portal-api/portal-developers.mdx b/tyk-apis/tyk-portal-api/portal-developers.mdx new file mode 100644 index 00000000..9011b427 --- /dev/null +++ b/tyk-apis/tyk-portal-api/portal-developers.mdx @@ -0,0 +1,520 @@ +--- +title: "Portal Developers" +order: 3 +noindex: True +sidebarTitle: "Developers" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; +import PortalDeveloperAnalytics from '/snippets/portal-developer-analytics.mdx'; + +<LegacyClassicPortalApi/> + +### List Developers + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/developers` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +GET /api/portal/developers HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` expandable +{ + "Data": [ + { + "_id": "554c733a30c55e4b16000002", + "keys": { + "<key-id>": ["<policy-id>"] + }, + "date_created": "2015-05-08T04:26:34.287-04:00", + "email": "test@test.com", + "fields": { + "Name": "Mondgo Brtian", + "custom1": "Test", + "custom2": "Test" + }, + "inactive": false, + "org_id": "53ac07777cbb8c2d53000002" + }, + { + "_id": "5555ec63a8b6b60001000001", + "keys": {}, + "date_created": "2015-05-15T08:53:54.873-04:00", + "email": "foo@bar.com", + "fields": { + "Name": "Tes", + "custom1": "", + "custom2": "" + }, + "inactive": false, + "org_id": "53ac07777cbb8c2d53000002" + }, + ... + ], + "Pages": 1 +} +``` + +### Retrieve a developer object by ID + +| **Property** | **Description** | +| :------------ | :----------------------------- | +| Resource URL | `/api/portal/developers/{id}` | +| Method | GET | +| Type | None | +| Body | Developer Object | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +GET /api/portal/developers/5e397e714ea461ef1d474efa HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` expandable +{ + "id": "5e397e714ea461ef1d474efa", + "email": "blip@blop.com", + "date_created": "2020-02-04T16:23:44.936+02:00", + "inactive": false, + "org_id": "5a79019d4ea461cdd0c77480", + "keys": {}, + "subscriptions": {}, + "fields": {}, + "nonce": "", + "sso_key": "", + "oauth_clients": { + "5e397e804ea461ef1d474efb": [ + { + "client_id": "bf27562a806d4ee1976c11b69e154a88", + "secret": "NzNiY2M0NWYtNmIwMi00ZjE4LWJhODMtNTc3YzlhMmQyY2I3", + "redirect_uri": "http://my_redirect_url", + "app_description": "My App", + "use_case": "My use case", + "date_created": "2020-02-04T16:24:21.939+02:00" + } + ] + }, + "password_max_days": 0, + "password_updated": "2020-02-04T16:23:45.014+02:00", + "PWHistory": [], + "last_login_date": "2020-02-04T16:23:00Z" +} +``` + +### Add a Developer Record + +| **Property** | **Description** | +| :------------ | :----------------------------- | +| Resource URL | `/api/portal/developers` | +| Method | POST | +| Type | None | +| Body | Developer Object | +| Param | None | + +#### Sample Request + +```{.copyWrapper} expandable +POST /api/portal/developers HTTP/1.1 +Host: localhost +Authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "email": "blip@blop.com", + "password": "$2a$10$hlG1ujAHWUpnM37k.l1RhO6RrxkCpXki2yrGhufnDs1IBiUo4Kzqy", + "date_created": "2015-05-15T08:56:13.257-04:00", + "inactive": false, + "org_id": "53ac07777cbb8c2d53000002", + "keys": { + "<api-key>": ["<policy-id>"] + }, + "fields": { + "Name": "", + "custom1": "", + "custom2": "" + } +} +``` + +#### Sample Response + +``` +{ + "Status":"OK", + "Message":"<developer-id>", + "Meta":"" +} +``` + + + +### Update a Developer Record + +| **Property** | **Description** | +| :------------ | :----------------------------- | +| Resource URL | `/api/portal/developers/{id}` | +| Method | PUT | +| Type | None | +| Body | Developer Object | +| Param | None | + +#### Sample Request + +```{.copyWrapper} expandable +PUT /api/portal/developers/5555eceda8b6b60001000004 HTTP/1.1 +Host: localhost +Authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "id": "5555eceda8b6b60001000004", + "email": "blip@blop.com", + "password": "$2a$10$hlG1ujAHWUpnM37k.l1RhO6RrxkCpXki2yrGhufnDs1IBiUo4Kzqy", + "date_created": "2015-05-15T08:56:13.257-04:00", + "inactive": false, + "org_id": "53ac07777cbb8c2d53000002", + "keys": { + "<key-id>": ["<policy-id>"] + }, + "fields": { + "Name": "", + "custom1": "", + "custom2": "" + } +} +``` + +#### Sample Response + +``` +{ + "Status":"OK", + "Message":"Data updated", + "Meta":"" +} +``` + +### Delete a Developer + +| **Property** | **Description** | +| :------------ | :----------------------------- | +| Resource URL | `/api/portal/developers/{id}` | +| Method | DELETE | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +DELETE /api/portal/developers/554c733a30c55e4b16000002 HTTP/1.1 +Host: localhost +Authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` +{ + "Status":"OK", + "Message":"Data deleted", + "Meta":"" +} +``` + +### Verify Developer Credentials + +> **NOTE**: This functionality is available from v2.3.8 onwards + +| **Property** | **Description** | +| :------------ | :------------------------------------------- | +| Resource URL | `/api/portal/developers/verify_credentials` | +| Method | POST | +| Type | None | +| Body | None | +| Param | None | + +```{.copyWrapper} expandable +curl https://admin.cloud.tyk.io/api/portal/developers/verify_credentials \ + -X POST \ + -H "authorization: $TYK_API_KEY" \ + -d \ + '{ + "username": "<developer-email>", + "password": "<developer-password>" + }' +``` + +If the user credentials are verified, the HTTP response code will be 200 (OK), otherwise credentials do match and a 401 error (Unauthorized) will be returned. + +### Reset Developer Password + +> **NOTE**: This functionality is available from v2.3.8 onwards + +| **Property** | **Description** | +| :------------ | :------------------------------------------- | +| Resource URL | `/api/portal/developers/password/:Id` | +| Method | POST | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} expandable +curl https://admin.cloud.tyk.io/api/portal/developers/password/:Id \ +-X POST \ +-H "authorization: $TYK_API_KEY" \ +-d \ +'{ + "password": "" +}' +``` + +#### Sample Response - Password Changed + +``` +{ + "Message":"Password changed" +} +``` + +#### Sample Response - Incorrect Developer ID + +``` +{ + "Status": "Error", "Message":"Developer password validation failed." +} +``` + +<PortalDeveloperAnalytics/> + +### Add Key To Developer + +| **Property** | **Description** | +| :------------ | :----------------------------- | +| Resource URL | `/portal/developers/:Id/subscriptions` | +| Method | POST | +| Type | None | +| Body | Subscription Object | +| Param | None | + + +#### Sample Request + +```{.copyWrapper} expandable +curl https://admin.cloud.tyk.io/api/portal/developers/:Id/keys \ +-X POST \ +-H "authorization: $TYK_API_KEY" \ +-d \ +'{ + "apply_policies": ["<pol-id>"], + "fields": { + "foo": "bar" + }, + "oauth_info": { + "redirect_uri": "..." + } +}' +``` + +### Change Developer Key Policy + +| **Property** | **Description** | +| :------------ | :----------------------------- | +| Resource URL | `/portal/developers/:developerId/keys/:keyId` | +| Method | PUT | +| Type | None | +| Body | Policy change Object | +| Param | None | + + +#### Sample Request + +```{.copyWrapper} expandable +curl https://admin.cloud.tyk.io/api/portal/developers/:developerId/keys/:keyId \ +-X PUT \ +-H "authorization: $TYK_API_KEY" \ +-d \ +'{ + "apply_policies": ["<pol-id>"] +}' +``` + +### Revoke Developer Key + +| **Property** | **Description** | +| :------------ | :----------------------------- | +| Resource URL | `/portal/developers/:developerId/keys/:keyID` | +| Method | DELETE | +| Type | None | +| Body | None | +| Param | None | + +### Reset Developer Key Quota + +| **Property** | **Description** | +| :------------ | :----------------------------- | +| Resource URL | `/portal/developers/:developerId/keys/:keyID/reset_quota` | +| Method | POST | +| Type | None | +| Body | None | +| Param | None | + +### Delete OAuth app + +| **Property** | **Description** | +| :------------ | :----------------------------- | +| Resource URL | `/portal/developers/oauth/:appId` | +| Method | DELETE | +| Type | None | +| Body | None | +| Param | None | + + +### Revoke a Single OAuth Client Token + +| **Property** | **Description** | +| :------------ | :---------------------------------------------- | +| Resource URL | `/oauth-clients/{oauthClientId}/revoke` | +| Method | POST | +| Type | JSON | +| Body | Client Object | +| Param | None | + +#### Sample Request + +```{.json} expandable +POST /oauth-clients/411f0800957c4a3e81fe181141dbc22a/revoke +Host: tyk-portal:3000 +Authorization: CSRF token +Body: { + "token":"eyJvcmciOiI1ZTIwOTFjNGQ0YWVmY2U2MGMwNGZiOTIiLCJpZCI6ImZiMGFmNWQ4ZGY1MzQ3MjY4YmYxNTE5MmJjN2YzN2QyIiwiaCI6Im11cm11cjY0In0=", + "token_type_hint":"access_token" +} +``` +#### Sample Response + +```{.json} +{ + "Status": "OK", + "Message": "token revoked successfully", + "Meta": null +} +``` + +### Revoke all OAuth Client Tokens + +| **Property** | **Description** | +| :------------ | :---------------------------------------------- | +| Resource URL | `/oauth-clients/{oauthClientId}/revoke_all` | +| Method | POST | +| Type | JSON | +| Body | Client Object | +| Param | None | + +#### Sample Request + +```{.json} expandable +POST /oauth-clients/411f0800957c4a3e81fe181141dbc22a/revoke_all +Host: tyk-portal:3000 +Authorization: CSRF token +Body: { + "client_secret":"MzUyNDliNzItMDhlNy00MzM3LTk1NWUtMWQyODMyMjkwZTc0" + } +``` +#### Sample Response + +```{.json} +{ + "Status": "OK", + "Message": "tokens revoked successfully", + "Meta": null +} +``` + +## Deprecated APIS + +### Add Subscription To Developer + +| **Property** | **Description** | +| :------------ | :----------------------------- | +| Resource URL | `/portal/developers/:Id/subscriptions` | +| Method | POST | +| Type | None | +| Body | Subscription Object | +| Param | None | + + +#### Sample Request + +```{.copyWrapper} expandable +curl https://admin.cloud.tyk.io/api//portal/developers/:Id/subscriptions \ +-X POST \ +-H "authorization: $TYK_API_KEY" \ +-d \ +'{ + "policy_id": "<pol-id>", + "fields": { + "foo": "bar" + }, + "oauth_info": { + "redirect_uri": "..." + } +}' +``` + +### Change Developer Key Policy + +| **Property** | **Description** | +| :------------ | :----------------------------- | +| Resource URL | `/portal/developers/:developerId/:keyId/:policyId` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +### Revoke Developer Key + +| **Property** | **Description** | +| :------------ | :----------------------------- | +| Resource URL | `/portal/developers/key/:apiID/:keyID/:Id` | +| Method | DELETE | +| Type | None | +| Body | None | +| Param | None | + +### Reset Developer Key Quota + +| **Property** | **Description** | +| :------------ | :----------------------------- | +| Resource URL | `/portal/developers/key/:apiID/:keyID/:Id/reset_quota` | +| Method | POST | +| Type | None | +| Body | None | +| Param | None | + +### Delete OAuth app + +| **Property** | **Description** | +| :------------ | :----------------------------- | +| Resource URL | `/portal/developers/oauth/:apiId/:appId` | +| Method | DELETE | +| Type | None | +| Body | None | +| Param | None | diff --git a/tyk-apis/tyk-portal-api/portal-documentation.mdx b/tyk-apis/tyk-portal-api/portal-documentation.mdx new file mode 100644 index 00000000..f252ff99 --- /dev/null +++ b/tyk-apis/tyk-portal-api/portal-documentation.mdx @@ -0,0 +1,313 @@ +--- +title: "API Publishing Endpoints" +'og:description': "This page details the API endpoint used for publishing APIs to Tyk classic Dev Portal. API platform teams and API owners can use this endpoint to integrate their APIs, making them visible and accessible to developers." +tags: ['Tyk Classic Portal API Publishing Endpoints', 'Classic Portal API'] +noindex: True +sidebarTitle: "API Publishing" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> +This page describes the endpoints to create [catalog](#catalog) and [Swagger documentation](#documentation) for your API. + +<br /> + + +<Note> +**Important Note on Spelling:** + +While our documentation now uses American English, the product itself, including UI, configuration fields, environment +variables, and APIs endpoints, retain British English spellings. When interacting with the product, please continue +using the British English spellings as they appear in the interface and API. (This means that for existing users nothing +has changed). +<br /> +**Example:** The API endpoint `/api/portal/catalogue` as shown throughout this page uses British spelling. In all other +instances, such as when describing or referring to this object in the documentation, we will use the +American spelling "catalog". +</Note> + + +## Documentation + +### Create Documentation + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/documentation` | +| Method | POST | +| Type | None | +| Body | Documentation Object | +| Param | None | + +The Swagger or Blueprint should be base64 encoded and included in the `documentation` field of the Request Body, as per the example below. + + +<Note> +Support for API Blueprint is being deprecated. See [Importing APIs](/api-management/gateway-config-managing-classic#api-blueprint-is-being-deprecated) for more details. +</Note> + + +#### Sample Request + +```{.copyWrapper} expandable +POST /api/portal/documentation HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "api_id": "", + "doc_type": "swagger", + "documentation": "<base64-encoded-swagger>" +} +``` + +#### Sample Response + +``` +{ + "Status": "OK", + "Message": "5ea6b2bd971eed0001009ddc", + "Meta": null +} +``` + +### Delete Documentation + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/documentation/{id}` | +| Method | DELETE | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```{.copyWrapper} +DELETE/api/portal/documentation HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +``` +{ + "Status": "OK", + "Message": "Data deleted", + "Meta": null +} +``` + +## Catalog + +### List Catalog + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/catalogue` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```http +GET /api/portal/catalogue HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +```json expandable +{ + "id":"5cc03284d07e7f00019404b4", + "org_id":"5cc03283d07e7f00019404b3", + "apis":[ + { + "name":"Portal OAuth API", + "short_description":"", + "long_description":"", + "show":true, + "api_id":"", + "policy_id":"5ce4086ce845260001c1e1f5", + "documentation":"", + "version":"v2", + "is_keyless":false, + "config":{ + "id":"", + "org_id":"", + "signup_fields":[ + + ], + "key_request_fields":[ + + ], + "require_key_approval":false, + "redirect_on_key_request":false, + "redirect_to":"", + "enable_multi_selection":false, + "disable_login":false, + "disable_signup":false, + "disable_auto_login":false, + "catalogue_login_only":false, + "oauth_usage_limit":-1, + "email":"", + "mail_options":{ + "mail_from_name":"", + "mail_from_email":"", + "email_copy":{ + "welcome_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "key_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "reset_password_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + } + } + }, + "override":false, + "HashKeys":false + }, + "fields":{ + + }, + "auth_type":"oauth" + }, + { + "name":"Test API", + "short_description":"", + "long_description":"", + "show":true, + "api_id":"", + "policy_id":"5ce51721e845260001d0a550", + "documentation":"5cf0d65d0313b300010b89ab", + "version":"v2", + "is_keyless":false, + "config":{ + "id":"", + "org_id":"", + "signup_fields":[ + + ], + "key_request_fields":[ + + ], + "require_key_approval":false, + "redirect_on_key_request":false, + "redirect_to":"", + "enable_multi_selection":false, + "disable_login":false, + "disable_signup":false, + "disable_auto_login":false, + "catalogue_login_only":false, + "oauth_usage_limit":-1, + "email":"", + "mail_options":{ + "mail_from_name":"", + "mail_from_email":"", + "email_copy":{ + "welcome_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "key_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "reset_password_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + } + } + }, + "override":false, + "HashKeys":false + }, + "fields":{ + + }, + "auth_type":"authToken" + } + ], + "email":"" +} +``` + +### Create Catalog + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/catalogue` | +| Method | POST | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```http +POST /api/portal/catalogue HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +```yaml +{ + Response here +} +``` + +### Update Catalog + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/catalogue` | +| Method | PUT | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```http +PUT /api/portal/catalogue HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +```yaml +{ + Response here +} +``` diff --git a/tyk-apis/tyk-portal-api/portal-keys.mdx b/tyk-apis/tyk-portal-api/portal-keys.mdx new file mode 100644 index 00000000..5e6083ea --- /dev/null +++ b/tyk-apis/tyk-portal-api/portal-keys.mdx @@ -0,0 +1,1633 @@ +--- +title: "Portal Keys" +order: 1 +noindex: True +sidebarTitle: "Key Requests" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +### List All Key Requests + +| **Property** | **Description** | +| :------------ | :------------------------ | +| Resource URL | `/api/portal/requests` | +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```http +GET /api/portal/requests HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +```json expandable +{ + "Data":[ + { + "id":"5cf61bff0313b300010b89ac", + "org_id":"5cc03283d07e7f00019404b3", + "for_api":"", + "for_plan":"5ce4086ce845260001c1e1f5", + "apply_policies":[ + "5ce4086ce845260001c1e1f5" + ], + "by_user":"5ce4090ee845260001c1e1f6", + "fields":{ + "oauth_app_description":"Application details", + "oauth_use_case":"" + }, + "approved":true, + "date_created":"2019-06-04T07:21:35.97Z", + "version":"v2", + "jwt_secret":"", + "certificate":"", + "oauth_info":{ + "redirect_uri":"http://localhost" + }, + "portal_developer":{ + "id":"5ce4090ee845260001c1e1f6", + "email":"mark+5@tyk.io", + "date_created":"2019-05-21T14:19:57.99Z", + "inactive":false, + "org_id":"5cc03283d07e7f00019404b3", + "keys":{ + "55b1ba47":[ + "5ce4086ce845260001c1e1f5" + ] + }, + "subscriptions":{ + "5ce4086ce845260001c1e1f5":"55b1ba47" + }, + "fields":{ + + }, + "nonce":"", + "sso_key":"", + "oauth_clients":{ + "5ce4086ce845260001c1e1f5":[ + { + "client_id":"00a3d6916da6448381ea6c254937eda9", + "secret":"ZGU4OWRlZjUtMzA0NS00Njk0LTljYTYtNDJmODY0ZWI1NWUz", + "redirect_uri":"http://localhost", + "app_description":"Application details", + "use_case":"", + "date_created":"2019-06-04T07:21:35.97Z" + } + ] + }, + "password_max_days":0, + "password_updated":"2019-10-23T10:01:56.25Z", + "PWHistory":[ + + ], + "last_login_date":"2019-10-23T10:02:14.117Z" + }, + "catalogue_entry":{ + "name":"Portal OAuth API", + "short_description":"", + "long_description":"", + "show":true, + "api_id":"", + "policy_id":"5ce4086ce845260001c1e1f5", + "documentation":"", + "version":"v2", + "is_keyless":false, + "config":{ + "id":"", + "org_id":"", + "signup_fields":[ + + ], + "key_request_fields":[ + + ], + "require_key_approval":false, + "redirect_on_key_request":false, + "redirect_to":"", + "enable_multi_selection":false, + "disable_login":false, + "disable_signup":false, + "disable_auto_login":false, + "catalogue_login_only":false, + "oauth_usage_limit":-1, + "email":"", + "mail_options":{ + "mail_from_name":"", + "mail_from_email":"", + "email_copy":{ + "welcome_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "key_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "reset_password_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + } + } + }, + "override":false, + "HashKeys":false + }, + "fields":{ + + }, + "auth_type":"oauth" + }, + "catalogues":[ + { + "name":"Portal OAuth API", + "short_description":"", + "long_description":"", + "show":true, + "api_id":"", + "policy_id":"5ce4086ce845260001c1e1f5", + "documentation":"", + "version":"v2", + "is_keyless":false, + "config":{ + "id":"", + "org_id":"", + "signup_fields":[ + + ], + "key_request_fields":[ + + ], + "require_key_approval":false, + "redirect_on_key_request":false, + "redirect_to":"", + "enable_multi_selection":false, + "disable_login":false, + "disable_signup":false, + "disable_auto_login":false, + "catalogue_login_only":false, + "oauth_usage_limit":-1, + "email":"", + "mail_options":{ + "mail_from_name":"", + "mail_from_email":"", + "email_copy":{ + "welcome_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "key_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "reset_password_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + } + } + }, + "override":false, + "HashKeys":false + }, + "fields":{ + + }, + "auth_type":"oauth" + } + ] + }, + { + "id":"5ce51790e845260001d0a551", + "org_id":"5cc03283d07e7f00019404b3", + "for_api":"", + "for_plan":"5ce4086ce845260001c1e1f5", + "apply_policies":[ + "5ce4086ce845260001c1e1f5" + ], + "by_user":"5ce4090ee845260001c1e1f6", + "fields":{ + "oauth_app_description":"", + "oauth_use_case":"" + }, + "approved":true, + "date_created":"2019-05-22T09:34:08.211Z", + "version":"v2", + "jwt_secret":"", + "certificate":"", + "oauth_info":{ + "redirect_uri":"localhost" + }, + "portal_developer":{ + "id":"5ce4090ee845260001c1e1f6", + "email":"mark+5@tyk.io", + "date_created":"2019-05-21T14:19:57.99Z", + "inactive":false, + "org_id":"5cc03283d07e7f00019404b3", + "keys":{ + "55b1ba47":[ + "5ce4086ce845260001c1e1f5" + ] + }, + "subscriptions":{ + "5ce4086ce845260001c1e1f5":"55b1ba47" + }, + "fields":{ + + }, + "nonce":"", + "sso_key":"", + "oauth_clients":{ + "5ce4086ce845260001c1e1f5":[ + { + "client_id":"00a3d6916da6448381ea6c254937eda9", + "secret":"ZGU4OWRlZjUtMzA0NS00Njk0LTljYTYtNDJmODY0ZWI1NWUz", + "redirect_uri":"http://localhost", + "app_description":"Application details", + "use_case":"", + "date_created":"2019-06-04T07:21:35.97Z" + } + ] + }, + "password_max_days":0, + "password_updated":"2019-10-23T10:01:56.25Z", + "PWHistory":[ + + ], + "last_login_date":"2019-10-23T10:02:14.117Z" + }, + "catalogue_entry":{ + "name":"Portal OAuth API", + "short_description":"", + "long_description":"", + "show":true, + "api_id":"", + "policy_id":"5ce4086ce845260001c1e1f5", + "documentation":"", + "version":"v2", + "is_keyless":false, + "config":{ + "id":"", + "org_id":"", + "signup_fields":[ + + ], + "key_request_fields":[ + + ], + "require_key_approval":false, + "redirect_on_key_request":false, + "redirect_to":"", + "enable_multi_selection":false, + "disable_login":false, + "disable_signup":false, + "disable_auto_login":false, + "catalogue_login_only":false, + "oauth_usage_limit":-1, + "email":"", + "mail_options":{ + "mail_from_name":"", + "mail_from_email":"", + "email_copy":{ + "welcome_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "key_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "reset_password_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + } + } + }, + "override":false, + "HashKeys":false + }, + "fields":{ + + }, + "auth_type":"oauth" + }, + "catalogues":[ + { + "name":"Portal OAuth API", + "short_description":"", + "long_description":"", + "show":true, + "api_id":"", + "policy_id":"5ce4086ce845260001c1e1f5", + "documentation":"", + "version":"v2", + "is_keyless":false, + "config":{ + "id":"", + "org_id":"", + "signup_fields":[ + + ], + "key_request_fields":[ + + ], + "require_key_approval":false, + "redirect_on_key_request":false, + "redirect_to":"", + "enable_multi_selection":false, + "disable_login":false, + "disable_signup":false, + "disable_auto_login":false, + "catalogue_login_only":false, + "oauth_usage_limit":-1, + "email":"", + "mail_options":{ + "mail_from_name":"", + "mail_from_email":"", + "email_copy":{ + "welcome_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "key_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "reset_password_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + } + } + }, + "override":false, + "HashKeys":false + }, + "fields":{ + + }, + "auth_type":"oauth" + } + ] + }, + { + "id":"5ce40db1e845260001c1e1f9", + "org_id":"5cc03283d07e7f00019404b3", + "for_api":"", + "for_plan":"5ce4086ce845260001c1e1f5", + "apply_policies":[ + "5ce4086ce845260001c1e1f5" + ], + "by_user":"5ce4090ee845260001c1e1f6", + "fields":{ + "oauth_app_description":"First OAuth Client", + "oauth_use_case":"" + }, + "approved":true, + "date_created":"2019-05-21T14:39:45.139Z", + "version":"v2", + "jwt_secret":"", + "certificate":"", + "oauth_info":{ + "redirect_uri":"http://httpbin.org;" + }, + "portal_developer":{ + "id":"5ce4090ee845260001c1e1f6", + "email":"mark+5@tyk.io", + "date_created":"2019-05-21T14:19:57.99Z", + "inactive":false, + "org_id":"5cc03283d07e7f00019404b3", + "keys":{ + "55b1ba47":[ + "5ce4086ce845260001c1e1f5" + ] + }, + "subscriptions":{ + "5ce4086ce845260001c1e1f5":"55b1ba47" + }, + "fields":{ + + }, + "nonce":"", + "sso_key":"", + "oauth_clients":{ + "5ce4086ce845260001c1e1f5":[ + { + "client_id":"00a3d6916da6448381ea6c254937eda9", + "secret":"ZGU4OWRlZjUtMzA0NS00Njk0LTljYTYtNDJmODY0ZWI1NWUz", + "redirect_uri":"http://localhost", + "app_description":"Application details", + "use_case":"", + "date_created":"2019-06-04T07:21:35.97Z" + } + ] + }, + "password_max_days":0, + "password_updated":"2019-10-23T10:01:56.25Z", + "PWHistory":[ + + ], + "last_login_date":"2019-10-23T10:02:14.117Z" + }, + "catalogue_entry":{ + "name":"Portal OAuth API", + "short_description":"", + "long_description":"", + "show":true, + "api_id":"", + "policy_id":"5ce4086ce845260001c1e1f5", + "documentation":"", + "version":"v2", + "is_keyless":false, + "config":{ + "id":"", + "org_id":"", + "signup_fields":[ + + ], + "key_request_fields":[ + + ], + "require_key_approval":false, + "redirect_on_key_request":false, + "redirect_to":"", + "enable_multi_selection":false, + "disable_login":false, + "disable_signup":false, + "disable_auto_login":false, + "catalogue_login_only":false, + "oauth_usage_limit":-1, + "email":"", + "mail_options":{ + "mail_from_name":"", + "mail_from_email":"", + "email_copy":{ + "welcome_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "key_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "reset_password_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + } + } + }, + "override":false, + "HashKeys":false + }, + "fields":{ + + }, + "auth_type":"oauth" + }, + "catalogues":[ + { + "name":"Portal OAuth API", + "short_description":"", + "long_description":"", + "show":true, + "api_id":"", + "policy_id":"5ce4086ce845260001c1e1f5", + "documentation":"", + "version":"v2", + "is_keyless":false, + "config":{ + "id":"", + "org_id":"", + "signup_fields":[ + + ], + "key_request_fields":[ + + ], + "require_key_approval":false, + "redirect_on_key_request":false, + "redirect_to":"", + "enable_multi_selection":false, + "disable_login":false, + "disable_signup":false, + "disable_auto_login":false, + "catalogue_login_only":false, + "oauth_usage_limit":-1, + "email":"", + "mail_options":{ + "mail_from_name":"", + "mail_from_email":"", + "email_copy":{ + "welcome_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "key_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "reset_password_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + } + } + }, + "override":false, + "HashKeys":false + }, + "fields":{ + + }, + "auth_type":"oauth" + } + ] + }, + { + "id":"5ce40a67e845260001c1e1f8", + "org_id":"5cc03283d07e7f00019404b3", + "for_api":"", + "for_plan":"5ce4086ce845260001c1e1f5", + "apply_policies":[ + "5ce4086ce845260001c1e1f5" + ], + "by_user":"5ce4090ee845260001c1e1f6", + "fields":{ + + }, + "approved":true, + "date_created":"2019-05-21T14:25:43.934Z", + "version":"v2", + "jwt_secret":"", + "certificate":"", + "portal_developer":{ + "id":"5ce4090ee845260001c1e1f6", + "email":"mark+5@tyk.io", + "date_created":"2019-05-21T14:19:57.99Z", + "inactive":false, + "org_id":"5cc03283d07e7f00019404b3", + "keys":{ + "55b1ba47":[ + "5ce4086ce845260001c1e1f5" + ] + }, + "subscriptions":{ + "5ce4086ce845260001c1e1f5":"55b1ba47" + }, + "fields":{ + + }, + "nonce":"", + "sso_key":"", + "oauth_clients":{ + "5ce4086ce845260001c1e1f5":[ + { + "client_id":"00a3d6916da6448381ea6c254937eda9", + "secret":"ZGU4OWRlZjUtMzA0NS00Njk0LTljYTYtNDJmODY0ZWI1NWUz", + "redirect_uri":"http://localhost", + "app_description":"Application details", + "use_case":"", + "date_created":"2019-06-04T07:21:35.97Z" + } + ] + }, + "password_max_days":0, + "password_updated":"2019-10-23T10:01:56.25Z", + "PWHistory":[ + + ], + "last_login_date":"2019-10-23T10:02:14.117Z" + }, + "catalogue_entry":{ + "name":"Portal OAuth API", + "short_description":"", + "long_description":"", + "show":true, + "api_id":"", + "policy_id":"5ce4086ce845260001c1e1f5", + "documentation":"", + "version":"v2", + "is_keyless":false, + "config":{ + "id":"", + "org_id":"", + "signup_fields":[ + + ], + "key_request_fields":[ + + ], + "require_key_approval":false, + "redirect_on_key_request":false, + "redirect_to":"", + "enable_multi_selection":false, + "disable_login":false, + "disable_signup":false, + "disable_auto_login":false, + "catalogue_login_only":false, + "oauth_usage_limit":-1, + "email":"", + "mail_options":{ + "mail_from_name":"", + "mail_from_email":"", + "email_copy":{ + "welcome_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "key_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "reset_password_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + } + } + }, + "override":false, + "HashKeys":false + }, + "fields":{ + + }, + "auth_type":"oauth" + }, + "catalogues":[ + { + "name":"Portal OAuth API", + "short_description":"", + "long_description":"", + "show":true, + "api_id":"", + "policy_id":"5ce4086ce845260001c1e1f5", + "documentation":"", + "version":"v2", + "is_keyless":false, + "config":{ + "id":"", + "org_id":"", + "signup_fields":[ + + ], + "key_request_fields":[ + + ], + "require_key_approval":false, + "redirect_on_key_request":false, + "redirect_to":"", + "enable_multi_selection":false, + "disable_login":false, + "disable_signup":false, + "disable_auto_login":false, + "catalogue_login_only":false, + "oauth_usage_limit":-1, + "email":"", + "mail_options":{ + "mail_from_name":"", + "mail_from_email":"", + "email_copy":{ + "welcome_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "key_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "reset_password_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + } + } + }, + "override":false, + "HashKeys":false + }, + "fields":{ + + }, + "auth_type":"oauth" + } + ] + }, + { + "id":"5ce40a12e845260001c1e1f7", + "org_id":"5cc03283d07e7f00019404b3", + "for_api":"", + "for_plan":"5ce4086ce845260001c1e1f5", + "apply_policies":[ + "5ce4086ce845260001c1e1f5" + ], + "by_user":"5ce4090ee845260001c1e1f6", + "fields":{ + "oauth_app_description":"First OAuth client", + "oauth_use_case":"" + }, + "approved":true, + "date_created":"2019-05-21T14:24:18.631Z", + "version":"v2", + "jwt_secret":"", + "certificate":"", + "oauth_info":{ + "redirect_uri":"http://httpbin.org;" + }, + "portal_developer":{ + "id":"5ce4090ee845260001c1e1f6", + "email":"mark+5@tyk.io", + "date_created":"2019-05-21T14:19:57.99Z", + "inactive":false, + "org_id":"5cc03283d07e7f00019404b3", + "keys":{ + "55b1ba47":[ + "5ce4086ce845260001c1e1f5" + ] + }, + "subscriptions":{ + "5ce4086ce845260001c1e1f5":"55b1ba47" + }, + "fields":{ + + }, + "nonce":"", + "sso_key":"", + "oauth_clients":{ + "5ce4086ce845260001c1e1f5":[ + { + "client_id":"00a3d6916da6448381ea6c254937eda9", + "secret":"ZGU4OWRlZjUtMzA0NS00Njk0LTljYTYtNDJmODY0ZWI1NWUz", + "redirect_uri":"http://localhost", + "app_description":"Application details", + "use_case":"", + "date_created":"2019-06-04T07:21:35.97Z" + } + ] + }, + "password_max_days":0, + "password_updated":"2019-10-23T10:01:56.25Z", + "PWHistory":[ + + ], + "last_login_date":"2019-10-23T10:02:14.117Z" + }, + "catalogue_entry":{ + "name":"Portal OAuth API", + "short_description":"", + "long_description":"", + "show":true, + "api_id":"", + "policy_id":"5ce4086ce845260001c1e1f5", + "documentation":"", + "version":"v2", + "is_keyless":false, + "config":{ + "id":"", + "org_id":"", + "signup_fields":[ + + ], + "key_request_fields":[ + + ], + "require_key_approval":false, + "redirect_on_key_request":false, + "redirect_to":"", + "enable_multi_selection":false, + "disable_login":false, + "disable_signup":false, + "disable_auto_login":false, + "catalogue_login_only":false, + "oauth_usage_limit":-1, + "email":"", + "mail_options":{ + "mail_from_name":"", + "mail_from_email":"", + "email_copy":{ + "welcome_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "key_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "reset_password_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + } + } + }, + "override":false, + "HashKeys":false + }, + "fields":{ + + }, + "auth_type":"oauth" + }, + "catalogues":[ + { + "name":"Portal OAuth API", + "short_description":"", + "long_description":"", + "show":true, + "api_id":"", + "policy_id":"5ce4086ce845260001c1e1f5", + "documentation":"", + "version":"v2", + "is_keyless":false, + "config":{ + "id":"", + "org_id":"", + "signup_fields":[ + + ], + "key_request_fields":[ + + ], + "require_key_approval":false, + "redirect_on_key_request":false, + "redirect_to":"", + "enable_multi_selection":false, + "disable_login":false, + "disable_signup":false, + "disable_auto_login":false, + "catalogue_login_only":false, + "oauth_usage_limit":-1, + "email":"", + "mail_options":{ + "mail_from_name":"", + "mail_from_email":"", + "email_copy":{ + "welcome_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "key_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "reset_password_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + } + } + }, + "override":false, + "HashKeys":false + }, + "fields":{ + + }, + "auth_type":"oauth" + } + ] + } + ], + "Pages":1 +} +``` + +### List a Single Key Request + +| **Property** | **Description** | +| :------------ | :-------------------------- | +| Resource URL | `/api/portal/requests/{id}`| +| Method | GET | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```http +GET /api/portal/requests/KEYID HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +```json expandable +{ + "id":"5cf61bff0313b300010b89ac", + "org_id":"5cc03283d07e7f00019404b3", + "for_api":"", + "for_plan":"5ce4086ce845260001c1e1f5", + "apply_policies":[ + "5ce4086ce845260001c1e1f5" + ], + "by_user":"5ce4090ee845260001c1e1f6", + "fields":{ + "oauth_app_description":"Application details", + "oauth_use_case":"" + }, + "approved":true, + "date_created":"2019-06-04T07:21:35.97Z", + "version":"v2", + "jwt_secret":"", + "certificate":"", + "oauth_info":{ + "redirect_uri":"http://localhost" + }, + "portal_developer":{ + "id":"5ce4090ee845260001c1e1f6", + "email":"mark+5@tyk.io", + "date_created":"2019-05-21T14:19:57.99Z", + "inactive":false, + "org_id":"5cc03283d07e7f00019404b3", + "keys":{ + "55b1ba47":[ + "5ce4086ce845260001c1e1f5" + ] + }, + "subscriptions":{ + "5ce4086ce845260001c1e1f5":"55b1ba47" + }, + "fields":{ + + }, + "nonce":"", + "sso_key":"", + "oauth_clients":{ + "5ce4086ce845260001c1e1f5":[ + { + "client_id":"00a3d6916da6448381ea6c254937eda9", + "secret":"ZGU4OWRlZjUtMzA0NS00Njk0LTljYTYtNDJmODY0ZWI1NWUz", + "redirect_uri":"http://localhost", + "app_description":"Application details", + "use_case":"", + "date_created":"2019-06-04T07:21:35.97Z" + } + ] + }, + "password_max_days":0, + "password_updated":"2019-10-23T10:01:56.25Z", + "PWHistory":[ + + ], + "last_login_date":"2019-10-23T10:02:14.117Z" + }, + "catalogue_entry":{ + "name":"Portal OAuth API", + "short_description":"", + "long_description":"", + "show":true, + "api_id":"", + "policy_id":"5ce4086ce845260001c1e1f5", + "documentation":"", + "version":"v2", + "is_keyless":false, + "config":{ + "id":"", + "org_id":"", + "signup_fields":[ + + ], + "key_request_fields":[ + + ], + "require_key_approval":false, + "redirect_on_key_request":false, + "redirect_to":"", + "enable_multi_selection":false, + "disable_login":false, + "disable_signup":false, + "disable_auto_login":false, + "catalogue_login_only":false, + "oauth_usage_limit":-1, + "email":"", + "mail_options":{ + "mail_from_name":"", + "mail_from_email":"", + "email_copy":{ + "welcome_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "key_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "reset_password_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + } + } + }, + "override":false, + "HashKeys":false + }, + "fields":{ + + }, + "auth_type":"oauth" + }, + "catalogues":[ + { + "name":"Portal OAuth API", + "short_description":"", + "long_description":"", + "show":true, + "api_id":"", + "policy_id":"5ce4086ce845260001c1e1f5", + "documentation":"", + "version":"v2", + "is_keyless":false, + "config":{ + "id":"", + "org_id":"", + "signup_fields":[ + + ], + "key_request_fields":[ + + ], + "require_key_approval":false, + "redirect_on_key_request":false, + "redirect_to":"", + "enable_multi_selection":false, + "disable_login":false, + "disable_signup":false, + "disable_auto_login":false, + "catalogue_login_only":false, + "oauth_usage_limit":-1, + "email":"", + "mail_options":{ + "mail_from_name":"", + "mail_from_email":"", + "email_copy":{ + "welcome_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "key_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + }, + "reset_password_email":{ + "enabled":false, + "subject":"", + "body":"", + "sign_off":"", + "hide_token_data":false + } + } + }, + "override":false, + "HashKeys":false + }, + "fields":{ + + }, + "auth_type":"oauth" + } + ] +} +``` + +### Create a Key Request + +| **Property** | **Description** | +| :------------ | :----------------------------- | +| Resource URL | `/api/portal/requests` | +| Method | POST | +| Type | None | +| Body | Developer Object | +| Param | None | + +#### Sample Request + +```http expandable +POST /api/portal/requests HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "key": "", + "org_id": "5e9d9544a1dcd60001d0ed20", + "for_api": "", + "for_plan": "5ead7120575961000181867e", + "apply_policies": [ + "5ead7120575961000181867e" + ], + "remove_policies": [], + "by_user": "5efdbdb749960c000137f589", + "fields": {}, + "approved": true, + "date_created": "2020-07-02T12:47:35.953Z", + "version": "v2", + "jwt_secret": "", + "certificate": "", + "portal_developer": { + "id": "5efdbdb749960c000137f589", + "email": "portal-developer@example.org", + "date_created": "2020-07-02T10:57:59.878Z", + "inactive": false, + "org_id": "5e9d9544a1dcd60001d0ed20", + "keys": { + "2ac7b253c36e210d": [ + "5ead7120575961000181867e" + ] + }, + "subscriptions": { + "5ead7120575961000181867e": "2ac7b253c36e210d" + }, + "fields": {}, + "nonce": "", + "sso_key": "", + "password_max_days": 0, + "password_updated": "2020-07-02T10:57:59.951Z", + "PWHistory": [], + "last_login_date": "2020-07-02T12:47:31.837Z" + }, + "catalogue_entry": { + "name": "Swagger Petstore", + "short_description": "", + "long_description": "", + "show": true, + "api_id": "", + "policy_id": "5ead7120575961000181867e", + "documentation": "5efdbdb849960c000137f58a", + "version": "v2", + "is_keyless": false, + "config": { + "id": "", + "org_id": "", + "signup_fields": [], + "key_request_fields": [], + "require_key_approval": false, + "redirect_on_key_request": false, + "redirect_to": "", + "enable_multi_selection": false, + "disable_login": false, + "disable_signup": false, + "disable_auto_login": false, + "catalogue_login_only": false, + "oauth_usage_limit": -1, + "email": "", + "mail_options": { + "mail_from_name": "", + "mail_from_email": "", + "email_copy": { + "welcome_email": { + "enabled": false, + "subject": "", + "body": "", + "sign_off": "", + "hide_token_data": false + }, + "key_email": { + "enabled": false, + "subject": "", + "body": "", + "sign_off": "", + "hide_token_data": false + }, + "reset_password_email": { + "enabled": false, + "subject": "", + "body": "", + "sign_off": "", + "hide_token_data": false + } + } + }, + "override": false, + "HashKeys": false + }, + "fields": {}, + "auth_type": "authToken" + }, + "catalogues": [{ + "name": "Swagger Petstore", + "short_description": "", + "long_description": "", + "show": true, + "api_id": "", + "policy_id": "5ead7120575961000181867e", + "documentation": "5efdbdb849960c000137f58a", + "version": "v2", + "is_keyless": false, + "config": { + "id": "", + "org_id": "", + "signup_fields": [], + "key_request_fields": [], + "require_key_approval": false, + "redirect_on_key_request": false, + "redirect_to": "", + "enable_multi_selection": false, + "disable_login": false, + "disable_signup": false, + "disable_auto_login": false, + "catalogue_login_only": false, + "oauth_usage_limit": -1, + "email": "", + "mail_options": { + "mail_from_name": "", + "mail_from_email": "", + "email_copy": { + "welcome_email": { + "enabled": false, + "subject": "", + "body": "", + "sign_off": "", + "hide_token_data": false + }, + "key_email": { + "enabled": false, + "subject": "", + "body": "", + "sign_off": "", + "hide_token_data": false + }, + "reset_password_email": { + "enabled": false, + "subject": "", + "body": "", + "sign_off": "", + "hide_token_data": false + } + } + }, + "override": false, + "HashKeys": false + }, + "fields": {}, + "auth_type": "authToken" + }] +} +``` + +#### Sample Response + +```json +{ + "Status": "OK", + "Message": "5efde61749960c000137f590", + "Meta": null +} +``` + +### Update a Key Request + +| **Property** | **Description** | +| :------------ | :-------------------------- | +| Resource URL | `/api/portal/requests/{id}`| +| Method | UPDATE | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```http expandable +UPDATE /api/portal/requests/5efde61749960c000137f590 HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 + +{ + "key": "", + "org_id": "5e9d9544a1dcd60001d0ed20", + "for_api": "", + "for_plan": "5ead7120575961000181867e", + "apply_policies": [ + "5ead7120575961000181867e" + ], + "remove_policies": [], + "by_user": "5efdbdb749960c000137f589", + "fields": {}, + "approved": true, + "date_created": "2020-07-02T12:47:35.953Z", + "version": "v2", + "jwt_secret": "", + "certificate": "", + "portal_developer": { + "id": "5efdbdb749960c000137f589", + "email": "portal-developer@example.org", + "date_created": "2020-07-02T10:57:59.878Z", + "inactive": false, + "org_id": "5e9d9544a1dcd60001d0ed20", + "keys": { + "2ac7b253c36e210d": [ + "5ead7120575961000181867e" + ] + }, + "subscriptions": { + "5ead7120575961000181867e": "2ac7b253c36e210d" + }, + "fields": {}, + "nonce": "", + "sso_key": "", + "password_max_days": 0, + "password_updated": "2020-07-02T10:57:59.951Z", + "PWHistory": [], + "last_login_date": "2020-07-02T12:47:31.837Z" + }, + "catalogue_entry": { + "name": "Swagger Petstore", + "short_description": "", + "long_description": "", + "show": true, + "api_id": "", + "policy_id": "5ead7120575961000181867e", + "documentation": "5efdbdb849960c000137f58a", + "version": "v2", + "is_keyless": false, + "config": { + "id": "", + "org_id": "", + "signup_fields": [], + "key_request_fields": [], + "require_key_approval": false, + "redirect_on_key_request": false, + "redirect_to": "", + "enable_multi_selection": false, + "disable_login": false, + "disable_signup": false, + "disable_auto_login": false, + "catalogue_login_only": false, + "oauth_usage_limit": -1, + "email": "", + "mail_options": { + "mail_from_name": "", + "mail_from_email": "", + "email_copy": { + "welcome_email": { + "enabled": false, + "subject": "", + "body": "", + "sign_off": "", + "hide_token_data": false + }, + "key_email": { + "enabled": false, + "subject": "", + "body": "", + "sign_off": "", + "hide_token_data": false + }, + "reset_password_email": { + "enabled": false, + "subject": "", + "body": "", + "sign_off": "", + "hide_token_data": false + } + } + }, + "override": false, + "HashKeys": false + }, + "fields": {}, + "auth_type": "authToken" + }, + "catalogues": [{ + "name": "Swagger Petstore", + "short_description": "", + "long_description": "", + "show": true, + "api_id": "", + "policy_id": "5ead7120575961000181867e", + "documentation": "5efdbdb849960c000137f58a", + "version": "v2", + "is_keyless": false, + "config": { + "id": "", + "org_id": "", + "signup_fields": [], + "key_request_fields": [], + "require_key_approval": false, + "redirect_on_key_request": false, + "redirect_to": "", + "enable_multi_selection": false, + "disable_login": false, + "disable_signup": false, + "disable_auto_login": false, + "catalogue_login_only": false, + "oauth_usage_limit": -1, + "email": "", + "mail_options": { + "mail_from_name": "", + "mail_from_email": "", + "email_copy": { + "welcome_email": { + "enabled": false, + "subject": "", + "body": "", + "sign_off": "", + "hide_token_data": false + }, + "key_email": { + "enabled": false, + "subject": "", + "body": "", + "sign_off": "", + "hide_token_data": false + }, + "reset_password_email": { + "enabled": false, + "subject": "", + "body": "", + "sign_off": "", + "hide_token_data": false + } + } + }, + "override": false, + "HashKeys": false + }, + "fields": {}, + "auth_type": "authToken" + }] +} +``` + +#### Sample Response + +```json +{ + "Status": "OK", + "Message": "5efde61749960c000137f590", + "Meta": null +} +``` + +### Delete a Key Request + +| **Property** | **Description** | +| :------------ | :-------------------------- | +| Resource URL | `/api/portal/requests/{id}`| +| Method | DELETE | +| Type | None | +| Body | None | +| Param | None | + +#### Sample Request + +```http +DELETE /api/portal/requests/5efde61749960c000137f590 HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response + +```json +{ + "Status": "OK", + "Message": "Data deleted", + "Meta": null +} +``` + +### Approve Key Requests + +| **Property** | **Description** | +| :------------ | :----------------------------- | +| Resource URL | `/api/portal/requests/approve/{id}`| +| Method | PUT | +| Type | None | +| Body | Developer Object | +| Param | None | + +#### Sample Request + +```http +PUT /api/portal/requests HTTP/1.1 +Host: localhost +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +``` + +#### Sample Response - Authentication Token + +```json +{ + "RawKey": "eyJvcmciOiI1ZTlkOTU0NGExZGNkNjAwMDFkMGVkMjAiLCJpZCI6ImQ0NzIzOWUxMjg3NTRjMGM5MTQ4MzYzMjg2YjhlZDQ2IiwiaCI6Im11cm11cjY0In0=", + "Password": "" +} +``` + +**Note:** If you're using *Authentication Token* as your *authentication mode*, the `RawKey` value is the actual API Key used to send authenticated requests. + +#### Sample Response - Basic Authentication + +```json +{ + "RawKey": "ffeLySpZR5", + "Password": "XJSm3gZIeDdk" +} +``` + +**Note:** If you're using Basic *Basic Authentication* as your *authentication mode*, the `RawKey` value is the username used to send authenticated requests. \ No newline at end of file diff --git a/tyk-cloud.mdx b/tyk-cloud.mdx new file mode 100644 index 00000000..c51a4ef8 --- /dev/null +++ b/tyk-cloud.mdx @@ -0,0 +1,2795 @@ +--- +title: "Set Up Tyk Cloud" +'og:description': "This page serves as a comprehensive guide to migrating workloads to Tyk Cloud" +sidebarTitle: "Tyk Cloud" +tags: ['Tyk Cloud', 'Migration'] +--- + +import { ButtonLeft } from '/snippets/ButtonLeft.mdx'; +import { ResponsiveGrid } from '/snippets/ResponsiveGrid.mdx'; + +## What is Tyk Cloud + +Tyk cloud is a fully managed service that makes it easy for API teams to create, secure, publish and maintain APIs at any scale, anywhere in the world. Tyk Cloud includes everything you need to manage your global API ecosystem: [Tyk Gateways](/tyk-oss-gateway), [Tyk Dashboard](/tyk-dashboard), [Tyk Developer Portal](/tyk-developer-portal) and [Universal Data Graph](/api-management/data-graph#overview). + +- **No need to wrestle with infrastructure:** You can be up and running within a few clicks. No need for complex deployments or large infrastructure teams. +- **Flexible deployment options:** Whether you're a startup or a large enterprise, Tyk Cloud has deployment options to suit your needs. You can scale and manage your API ecosystem easily and efficiently. The control plane is hosted by Tyk in the cloud, in one of the 5 regions available. Meanwhile, the data planes, composed of Tyk Gateways and Redis for temporary storage, can be either hosted by Tyk or managed by you on your infrastructure. +- **Geographical freedom:** Tyk Cloud allows you to select your preferred AWS location as your home region, ensuring your data and Tyk Gateways are live and secured in the region that suits you best. +- **Designed for platform teams:** With Tyk Cloud, you can use [role-based access control (RBAC)](https://tyk.io/blog/how-to-manage-multiple-teams-with-tyk-cloud/) to manage your team structure, as well as [multiple environments and multiple organizations](https://tyk.io/blog/how-to-manage-multiple-teams-with-tyk-cloud/). + + +Start using Tyk on our servers. Nothing to install: + +<ButtonLeft href="https://tyk.io/sign-up/" color="green" content="Tyk Cloud Free trial" /> + + +### Why Tyk Cloud? + +#### Next Level SaaS + +Tyk is cloud native and has always been a true multi-cloud product and now we’re taking it to the next level with our next level SaaS platform, Tyk Cloud. + +Now you don’t need to worry about vendor lock-in, or complex deployments, you can benefit from being able to optimize your platforms across and between a myriad of providers such as AWS, Google Cloud etc. + +#### Quick deployments Wherever you need them + +Want to handle Govt API traffic in Singapore? With just a few clicks your data and Tyk Gateways are live and secured in the AWS Gov cloud. Want to add local Gateways in Australia to improve performance and resilience? It’s just a click away. Want to deploy your management layer to your own hosted servers rather than a cloud provider? We make it simple. + +The new Tyk Cloud platform allows you to quickly setup the full Tyk Enterprise API Management platform, simply choosing the regions where you wish to locate your gateways and where you wish your data to reside, resulting in immediate and secure data sovereignty. + +Seamlessly wire your environments between cloud providers, and your own infrastructure, anywhere in the world at the click of a button, not only eliminating lock-in but making it possible to expand your platform to cater for the changing needs of your clients. + +#### Designed for Enterprises + +Tyk Cloud is designed for Enterprises who may have multi-organizations and multi-teams, so you can combine or isolate your platform and the underlying providers, the choice is completely yours! + +To make it even simpler, Tyk Cloud is pre-configured so you can be up and running within a few clicks, no laborious tasks for your internal teams and best practice configuration and security is delivered out of the box. + +### Where is Tyk Cloud hosted? + +Tyk Cloud is currently available to auto-deploy on AWS. +Paid plans and Enterprise trials allow users to select one of 6 AWS locations as their home region as well as the locations of their Cloud Data Planes. The 6 AWS regions to choose from are: +- aws-ap-southeast-1, Singapore +- aws-eu-central-1, Frankfurt, Germany +- aws-eu-west-2, London, UK +- aws-us-east-1, N. Virginia, USA +- aws-us-west-2, Oregon, USA +- aws-ap-southeast-2, Australia + +## Quick Start Tyk Cloud + + +<Note> +The Tyk Cloud trial is limited to 48 hours. After this period, your data will be deleted. +The Tyk Cloud trial does not include access to [Hybrid deployments](#deploy-hybrid-gateways) or the [Developer Portal](/portal/overview/intro). +To try out these capabilities, please get in touch for a [guided evaluation](https://tyk.io/guided-evaluation/) with our team. +</Note> + + +Welcome to the [Tyk Cloud Platform](/tyk-cloud)! +This guide will lead you through the following steps: +1. Signing up with [Tyk Cloud ](/tyk-cloud). +2. Creating your first [API](/api-management/gateway-config-introduction) using the [Tyk Dashboard](/tyk-dashboard). +3. Setting up a [Policy](/api-management/policies#what-is-a-security-policy) and Key to secure your APIs. + +No installation required! + +### Steps for Setting up Tyk Cloud + +1. **Create Tyk Cloud Account** + +To create Tyk Cloud account follow this [guide](/getting-started/create-account). + +2. **Get started with your first API with Tyk Dashboard** + +To create your API using Tyk Dashboard follow this [guide](/getting-started/configure-first-api). + +## Tyk Cloud Feature Setups + +<ResponsiveGrid> + + +<Card title="Configuration" href="/tyk-cloud#add-custom-authentication"> +**Python custom plugins** + +Implement your own custom logic with Python based plugins +</Card> + +<Card title="Configuration" href="/tyk-cloud#configure-custom-domains"> +**Using custom domains** + +Configure custom domain for your Dashboard and Developer Portal +</Card> + +<Card title="Administration" href="/tyk-cloud#managing-environments"> +**Manage environments** + +Create and manage multiple environments within your Tyk Cloud organization +</Card> + +<Card title="Administration" href="/tyk-cloud#managing-control-planes"> +**Manage deployments** + +Create and manage your Cloud Control Plane and Cloud Data Plane deployments +</Card> + +<Card title="Administration" href="/tyk-cloud#managing-teams-and-users"> +**Manage teams & users** + +Create teams in your organization, define roles and manage user access +</Card> + +<Card title="Account" href="/tyk-cloud#manage-accounts-and-billing"> +**Manage billing** + +Upgrade your subscription, billing details or card information +</Card> + + +</ResponsiveGrid> + +## Comprehensive Tyk Cloud Setup + +This section walks you through how to start using Tyk Cloud, creating organization, environment and users before creating an API. If you are in a hurry, try the [Quick Start guide](#quick-start-tyk-cloud) for a 5 min version of this tutorial. + +* Creating your Tyk Cloud account +* Your first Organization +* Creating your first Team and Environment +* Configuring and deploying your Control Plane and creating your Cloud Data Plane +* Adding and testing your first API + + + + <Note> + `Comprehensive Tyk Cloud Setup` requires access to `Tyk Cloud Console` (management UI). Please note that free trial users do not have access to the `Tyk Cloud Console`. To obtain the required access, contact [support](https://tyk.io/contact/). + </Note> + + +At the end of this process you will have a simple API set up via a Tyk Dashboard and you'll see analytics for this API on the Tyk Activity Dashboard. + +Depending on your initial requirements in terms of Environments, Teams and Users, the setup process should take between 15 to 30 minutes. + +### Prerequisites + +The following information would be useful so you can set up Tyk Cloud as quickly as possible: + +* Team member information including their email address and the role you plan to assign to them. +* We have some specific terminology used within Tyk Cloud. It would be useful to checkout our [Glossary](#glossary) so you understand what we are referring to. + +### Hierarchy + +This diagram shows how _Organization, Teams, Environments, Control Planes and Cloud Data Planes_ fit in with each other and which object contains which: + +<img src="/img/cloud/Onboarding_Diagram_2-1_Ara.png" alt="Hierarchy of Organization, Teams, Environments, Control Planes and Cloud Data Planes" /> + +### Complete Cloud Setup Tasks + +#### Create an Account + +You can use Tyk Cloud to manage your APIs effectively and with minimal effort. This page explains how to create an account, in order to start doing so. + +##### What happens when you create your Tyk Cloud account? + +When you create your Tyk Cloud account, we do the following for you: + +* Assign the account creator as a [Billing admin](#user-roles-in-tyk-cloud) for the Organization. This user role allows you to manage the billing and plans for your org. You can also add other billing admins as required. +* Assign the new account to our [48 hours free trial plan](https://tyk.io/sign-up/#cloud) + +##### Creating your first account + +[Start here](https://tyk.io/sign-up/#cloud). + +* To create your account, you will have to fill in first-level details like your first name, last name and email. +* Then, set up a password for your Tyk Cloud account. +* Following that check the box at the bottom of the page to confirm that you have read and accepted our [Terms and Conditions](https://tyk.io/software-as-a-service-agreement/). +* Finally, click **Create new account** +* After completing the Account Creation form, click **Start Organization Setup**. + + +#### Set Up Your Organisation + +Now that you have created the new Tyk Cloud account with your basic details, it is time to set up your organization. This page will tell you how to set up your organization and also about the two ways of setting it up. + +##### What is an organization? + +* An organization is the main entity for all your data (Environments, APIs, Users, etc) +* An Organization is connected to a single region and once connected, cannot be changed. + +##### Steps to set up your organization + +* **Step 1 - Name your Organization:** Give your organization a name. This is up to you, but most users use their company name. + +* **Step 2 - Select a Home Region:** Select a region from the drop-down list where your [Control Plane](#glossary) will be deployed and your data stored. The number of regions available will depend on your license. Further regions can be added as an upgrade option. + + + + <Note> + Tyk Cloud can currently be deployed across 2 AWS regions in the USA plus UK, Germany and Singapore. If you have any concerns about Brexit impacting the way you store data you should read [AWS regularly updated Brexit statement](https://aws.amazon.com/compliance/gdpr-center/brexit/). + </Note> + + +##### Types of Setups + +You can now select how to configure your deployment. + +**Option 1: Demo Setup** + +Our demo setup will quickly configure your first deployment setup automatically, creating your first team, Cloud Control Plane and Cloud Data Plane. + +**Option 2: Manual Setup** + +This setup option gives you full control on creating the following: + +* Teams +* Environments +* Configuration and deployment of Control Planes and Cloud Data Planes + +For a manual setup you'll get started by [setting up your first team](#create-your-first-team). + +#### Create Your First Team + +Following organization setup, you will have to set up your team(s) on Tyk Cloud. This page will tell you all about the process. + +##### What is a team? + +* A team is a sub-grouping inside an organization. +* Inside a team, you can define users(team members) and roles(permissions that can be applied to a user or a team of users). + +##### Steps to set up your team + +After creating your Organization you'll land on the success screen. Click **Get Started**. + +* **Step 1 - Name your Team:** Give your [Team](#glossary) a name. You may find it useful to reflect the names used within your organization. + +* **Step Two - Invite your Users:** Invite your [users](#glossary) to your team. You'll only need their email address and which of the available [roles](#glossary) you want to assign to them. This step is optional and can be completed within the dashboard later. + +##### User Roles in Tyk Cloud + +Out of the box, the following roles are setup: + +* **Team member:** They can manage deployment activity for the team they are added to. +* **Team admin:** They can manage deployment activity and users for the team they are added to. +* **Organization admin:** They can manage deployment activity and users for a single organization. + +Next you'll create an [Environment](#configure-environment-and-deployments). + +#### Configure Environment and Deployments + +An Environment allows you to group deployments together. In this step we will create an Environment and configure our first Control Plane and Cloud Data Plane deployments. + +##### What is an environment? + +An environment is a grouping of β€˜deployments’ that can have multiple Control Planes and Cloud Data Planes. + +##### Steps to set up your environment + +* **Step 1 - Name your Environment:** Give your [Environment](#glossary) a name. You may find it useful to reflect the names used within your organization such as Development, Production etc. + +* **Step 2 - Name your Control Plane:** Give your [Control Plane](#glossary) a name. Again, this is up to you and you may already have an infrastructure you want to re-create in Tyk Cloud. + +* **Step 3 - Configure your first Cloud Data Plane:** Select the region you want to locate your [Cloud Data Plane](#glossary) in from the drop-down list. Your Cloud Data Plane is not confined to the same region as your Organization and Control Plane but the amount of regions you have to choose from can be limited depending on your subscription plan. Give your Cloud Data Plane a name. + + + + <Note> + You need to have at least one Cloud Data Plane with a *Deployed* status connected to your Control Plane. + </Note> + + +* **Step 4 - Deployment:** + +1. Click [Deploy Control Plane and Create a Cloud Data Plane](#glossary). You can watch your Control Plane being deployed and your Cloud Data Plane being created. You will then be taken to the Control Plane overview screen within the Tyk Cloud dashboard. +2. From your Control Plane overview you will see the Cloud Data Plane is in a **Not Deployed** state. Click on your Cloud Data Plane to open its overview. +3. In the top right of your Cloud Data Plane overview, click **Not Deployed** and choose **Deploy** from the drop-down. +4. With your Cloud Data Plane successfully deployed, make a note of the tags assigned to your Cloud Data Plane. One tag is "edge" and the other is the location of your Cloud Data Plane. You'll add a tag when creating your API. + +Here's a video on how to set up your Tyk Cloud Environment. + +<iframe width="560" height="315" src="https://www.youtube.com/embed/DxoLm0vgsP8" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + +Next you'll [set up your first API](#deploy-and-add-your-first-api) from the Tyk Dashboard. + +#### Deploy and Add Your First API + +Your onboarding is now complete! The next step will be to setup a very basic API to demonstrate how APIs are managed within Tyk Cloud. + + +<Warning> +Warning + +In Tyk Gateway release 5.3.0, Tyk OAS APIs gained feature maturity. Tyk Dashboard will automatically migrate any pre-5.3.0 Tyk OAS APIs to the feature mature standard when you upgrade to 5.3.0 or later. Feature mature Tyk OAS APIs may not work with pre-5.3.0 versions of Tyk Gateway. + +It is not possible to rollback to previous versions of Tyk components with Tyk OAS APIs created in 5.3.0. + +For further details, please refer to the [release notes](/developer-support/release-notes/gateway#530-release-notes) for Tyk Gateway v5.3.0. +</Warning> + + +##### Steps to add an API in Tyk Cloud + +* **Step 1 - Access the Dashboard:** Go to the Control Plane overview and click the dashboard link in the Ingress list. You'll be redirected to the Tyk Dashboard for your [Control Plane](#glossary). +* **Step 2 - Add a New API:** Click the APIs menu item and then click **Add New API**. +* **Step 3 - Core Settings:** + + 1. Give Your API a name - We'll use "my app" for the rest of this Getting Started journey. + 2. Scroll down to the **Target URL** setting and use the URL https://httpbin.org/ + 3. Then scroll down to the Authentication section and select **Open(Keyless)** to keep things simple for this demo. + + + + <Warning> + Ensure you configure a valid API Listen path. Root ("/") listen paths are **not** supported on Tyk Cloud deployments prior to version v3.2.0. + </Warning> + + +* **Step 4 - Advanced Options:** + + 1. Click the **Advanced Options** tab of the API Designer. + 2. Scroll to the **Segment Tags (Node Segmentation)** setting and add the cloud data plane tag (edge) you saw when creating the Cloud Data Plane. + +<img src="/img/cloud/edge_segment_tags.png" alt="Segment Tags" /> + + +<Note> +**Note:** + +**How Segment Tags work in Tyk Cloud?** + +When a Cloud Data Plane is deployed, the tag 'edge' and a location tag are automatically generated for the Cloud Data Plane. You use these tags to connect your API to the appropriate Cloud Data Plane. It works as follows: + +1. Add the **edge** tag to your API to connect it to all Cloud Data Planes within the Control Plane. +2. Add the location tag to your API to connect it to only Cloud Data Planes with that location within the Control Plane. +</Note> + + + +<Warning> +All APIs must be connected to a Cloud Data Plane by adding the appropriate tag in the *Segment Tags (Node Segmentation)* in the API Designer. +</Warning> + + +* **Step 5 - Save Your API:** Click **Save** from the API Designer. Your API will now be added to the APIs list and as the next step you'll access your API from the Gateway Ingress. + +Watch our video on Adding an API to your Tyk Cloud Dashboard. + +<iframe width="560" height="315" src="https://www.youtube.com/embed/OtQMNKwfXwo" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + +Want to learn more from one of our team? + +<ButtonLeft href="https://tyk.io/book-a-demo/" color="green" content="Book a demo" /> + + +#### Test Your API + +Your first API has been added. What's next? Testing it! This page shows how you can test an API that you have added to Tyk Cloud, to ensure that it’s functioning correctly. You'll now access the API you setup in [Task 5](#deploy-and-add-your-first-api) from the Cloud Data Plane within Tyk Cloud. + +##### Steps to test your API + +* **Step 1 - Access the Gateway Ingress:** From the Cloud Data Plane overview, copy the Ingress link and open it in a browser tab. You will get a 404 error. + +* **Step 2 - Append the URL with your API:** You created a API named **my app** in [Task 5](#deploy-and-add-your-first-api). Add `/my-app/` to the end of the URL. You should be taken to [https://httpbin.org/](https://httpbin.org/), which you added as the **Target URL** for the API in [Task 5](#steps-to-add-an-api-in-tyk-cloud). + + +Next you'll [view the analytics](#view-analytics) for your API in the Dashboard. + +#### View Analytics + +We have now created and tested our API. How do we know that they are performing well? This page walks you through how to then view your API analytics so that you can ensure your APIs are performing perfectly. + +##### Steps to check your API analytics + +* **Step 1 - Access the Dashboard:** You'll now look at the analytics for the API you created in [Task 5](#deploy-and-add-your-first-api).If you're not still in the Tyk Dashboard for your Control Plane, click the dashboard link in the Control Plane Ingress list. Click the Gateway Dashboard menu item and you can see the successful calls made to your API from the Cloud Data Plane you created. + +* **Step 2 - Create an Error:** From the Cloud Data Plane, make a call that will throw an error. For example, use `me-app` instead of `my-app`. You should see the error displayed in the Analytics. + + +#### Review Your Setup + +This summary page explains the steps required to implement Tyk Cloud, which enables you to manage your APIs seamlessly. + +We've covered the following to get you started with the Tyk Cloud way of managing Tyk deployments: + +* We've created a new Organization +* We've added a Team +* We've added an Environment, including a Control Plane and an associated Cloud Data Plane and deployed them +* We've added a very simple API to the Control Plane Dashboard and tested it via the first Cloud Data Plane +* We've seen the data from the Gateways displayed in the Analytics section of the Control Plane Dashboard + +We have actually only scratched the surface of what is possible with Tyk Cloud. + +What to go through next: + +* Managing your Deployments +* Adding [Plugins and Middleware](#configure-plugins) to your Control Plane + +## Import Existing APIs + +Tyk supports importing both API Blueprint and Swagger (OpenAPI) JSON definitions from either the Gateway or the Dashboard. Tyk will output the converted file to to `stdout`. Below are the commands you can use to get Tyk to switch to command mode and generate the respective API definitions for both API Blueprint and Swagger files. + +### API Blueprint is being Deprecated + +Our support for API Blueprint is being deprecated. We have been packaging [aglio](https://github.com/danielgtaylor/aglio) in our Docker images for the Dashboard which enables rendering API Blueprint Format in the portal. This module is no longer maintained and is not compatible with newer NodeJS. If you wish to continue using this feature, you can do so by installing the module yourself in your Dockerfile. The imapct of this change is that our Docker images will no longer contain this functionality. + +As a work around, you can do the following: + +* Create API Blueprint in JSON format using the Apiary [Drafter](https://github.com/apiaryio/drafter) tool +* Convert API Blueprint to OpenAPI (Swagger) using the Apiary [API Elements CLI](https://github.com/apiaryio/api-elements.js/tree/master/packages/cli) tool. + +### Import APIs via the Gateway + +#### Using API Blueprint + + +<Note> +See [note](#api-blueprint-is-being-deprecated) above regarding deprecation of support for API Blueprint. +</Note> + + +Tyk supports an easy way to import Apiary API Blueprints in JSON format using the command line. + +Blueprints can be imported and turned into standalone API definitions (for new APIs) and also imported as versions into existing APIs. + +It is possible to import APIs and generate mocks or to generate Allow Lists that pass-through to an upstream URL. + +All imported Blueprints must be in the JSON representation of Blueprint's markdown documents. This can be created using Apiary's [Snow Crash tool](https://github.com/apiaryio/snowcrash). + +Tyk outputs all new API definitions to `stdout`, so redirecting the output to a file is advised in order to generate new definitions to use in a real configuration. + +**Importing a Blueprint as a new API:** + +Create a new definition from the Blueprint: + +```{.copyWrapper} +./tyk --import-blueprint=blueprint.json --create-api --org-id=<id> --upstream-target="http://widgets.com/api/" +``` + +**Importing a definition as a version in an existing API:** + +Add a version to a definition: + +```{.copyWrapper} +./tyk --import-blueprint=blueprint.json --for-api=<path> --as-version="version_number" +``` + +**Creating your API versions as a mock** + +As the API Blueprint definition allows for example responses to be embedded, these examples can be imported as forced replies, in effect mocking out the API. To enable this mode, when generating a new API or importing as a version, simply add the `--as-mock` parameter. + +#### Using Swagger (OpenAPI) + +Tyk supports importing Swagger documents to create API definitions and API versions. Swagger imports do not support mocking though, so sample data and replies will need to be added manually later. + +**Importing a Swagger document as a new API** + +Create a new definition from Swagger: + +```{.copyWrapper} +./tyk --import-swagger=petstore.json --create-api --org-id=<id> --upstream-target="http://widgets.com/api/" +``` + +<Note> +When creating a new definition from an OAS 3.0 spec, you will have to manually add the listen path after the API is created. +</Note> + + + +**Importing a Swagger document as a version into an existing API** + +Add a version to a definition: + +```{.copyWrapper} +./tyk --import-swagger=petstore.json --for-api=<path> --as-version="version_number" +``` + +**Mocks** + +Tyk supports API mocking using our versioning `use_extended_paths` setup, adding mocked URL data to one of the three list types (white_list, black_list or ignored). In order to handle a mocked path, use an entry that has `action` set to `reply`: + +```json expandable +"ignored": [ + { + "path": "/v1/ignored/with_id/{id}", + "method_actions": { + "GET": { + "action": "reply", + "code": 200, + "data": "Hello World", + "headers": { + "x-tyk-override": "tyk-override" + } + } + } + } +], +``` + +See [Tyk Classic API versioning](/api-management/gateway-config-tyk-classic#tyk-classic-api-versioning) for more details. + +### Import APIs via the Dashboard API + +#### Import API - Swagger + +| **Property** | **Description** | +| :------------ | :------------------------- | +| Resource URL | `/api/import/swagger/` | +| Method | POST | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```{.json} expandable +POST /api/import/swagger/ +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +{ + "swagger": "{swagger data...}", + "insert_into_api": false, + "api_id": "internal API id", + "version_name": "yourversionname", + "upstream_url": "yourupstreamurl" +} +``` + +Parameters: + +* `insert_into_api`: If set to `true` the import will replace an existing API. Setting to `false` will import into a new API. +* `api_id`: The internal MongoDB object id for your API. +* `version_name`: Your versioning convention name for the imported API. +* `upstream_url`: The URL the API is served by. + + +**Sample Response** + +``` +{ + "Status": "OK", + "Message": "API Imported", + "Meta": "new_api_id" +} + +``` + + +#### Import API - Blueprint + +| **Property** | **Description** | +| :------------ | :--------------------------- | +| Resource URL | `/api/import/blueprint/` | +| Method | POST | +| Type | None | +| Body | None | +| Param | None | + +**Sample Request** + +```{.json} expandable +POST /api/import/blueprint/ +Host: localhost:3000 +authorization:7a7b140f-2480-4d5a-4e78-24049e3ba7f8 +{ + "blueprint": "{blueprint data...}", + "insert_into_api": false, + "api_id": "internal API id", + "as_mock": false, + "version_name": "yourversionname", + "upstream_url": "yourupstreamurl" +} +``` + +Parameters: + +* `insert_into_api`: If set to `true` the import will replace an existing API. Setting to `false` will import into a new API. +* `api_id`: The internal MongoDB object id for your API. +* `as_mock`: If set to true, enables our mocking support for Blueprint imported API. See **Mocks** above for more details. +* `version_name`: Your versioning convention name for the imported API. +* `upstream_url`: The URL the API is served by. + + +**Sample Response** + +``` +{ + "Status": "OK", + "Message": "API Imported", + "Meta": "new_api_id" +} + +``` + + + +### Import APIs via the Dashboard UI + +1. **Select "APIs" from the "System Management" section** + +<img src="/img/2.10/apis_menu.png" alt="API listing" /> + +2. **Click "IMPORT API"** + +<img src="/img/2.10/import_api_button.png" alt="Add API button location" /> + +Tyk supports the following import options: + +1. From an Existing Tyk API definition +2. From a Apiary Blueprint (JSON) file +3. From a Swagger/OpenAPI (JSON only) file +4. From a SOAP WSDL definition file (new from v1.9) + +To import a Tyk Definition, just copy and paste the definition into the code editor. + +For Apiary Blueprint and Swagger/OpenAPI, the process is the same. For example: + +Click the "From Swagger (JSON)" option from the pop-up + +<img src="/img/2.10/import_api_json.png" alt="Import popup" /> + +For WSDL: + +<img src="/img/2.10/import_api_wsdl.png" alt="Import WSDL" /> + +3. **Enter API Information** + +You need to enter the following information: + +* Your **Upstream Target** +* A **Version Name** (optional) +* An optional **Service Name** and **Port** (WSDL only) +* Copy code into the editor + +4. **Click "Generate API"** + +Your API will appear in your APIs list. If you select **EDIT** from the **ACTIONS** drop-down list, you can see the endpoints (from the [Endpoint Designer](/api-management/dashboard-configuration#endpoint-designer)) that have been created as part of the import process. + +### Creating a new API Version by importing an API Definition using Tyk Dashboard + +As well as importing new APIs, with Tyk, you can also use import to create a new version of an existing Tyk Classic API. + +1. Open the API Designer page and select Import Version from the **Options** drop-down. + +<img src="/img/oas/import-api-version.png" alt="Import API Version Drop-Down" /> + +2. Select either OpenAPI (v2.0 or 3.0) or WSDL/XML as your source API + +3. You need to add a new **API Version Name**. **Upstream URL** is optional. + +<img src="/img/oas/import-api-version-config.png" alt="Import API Version Configuration" /> + +4. Click **Import API**. + +<img src="/img/oas/import-api-button.png" alt="Import API" /> + +5. Select the **Versions** tab and your new version will be available. +6. Open the **Endpoint Designer** for your API and select your new version from **Edit Version**. +7. You will see all the endpoints are saved for your new version. + +<img src="/img/oas/version-endpoints.png" alt="Version Endpoints" /> + + +## Additional Cloud Configuration + +### Manage Environments and Deployments + +This section covers the administration of the various components of your Tyk Cloud installation: + +* [Managing Organizations](#set-up-your-organisation) +* [Managing Environments](#manage-environments-and-deployments) +* [Managing Control Planes](#managing-control-planes) +* [Managing Cloud Data Planes](#managing-cloud-data-plane) + +It also covers links to how to start [managing your APIs](#manage-apis) via the Tyk Dashboard, accessible from your Control Plane. + +#### Organisations Overview + +Your Organization is your "container" for all your Environments, Control Planes and Cloud Data Planes. When you setup your Organization when [creating your account](/getting-started/create-account), you assign it to a Home Region where all your data is stored. You cannot change this home region after creating your organization. + +**Organization Overview Screen** + +If you are an Organization Admin, when you log in you will see the Overview screen for the Organization you are connected to. If you are a team admin or team member you will see the Team Overview Screen. The Organization Overview screen displays the following info: + +* Quick Stats +* All Teams +* All Deployments +* All Environments + + +**Quick Stats** + +<img src="/img/admin/tyk-cloud-org-overview.png" alt="Quick Stats" /> + +This section gives you an "at a glance" overview of your organization. This section is designed to show what your plan entitles your organization to and how much of your entitlement is currently used in relation to Teams, Control Planes, Cloud Data Plane Deployments and the distribution of those deployments across the available entitlement regions. + +**Teams** + +<img src="/img/admin/tyk-cloud-org-teams.png" alt="Teams" /> + +This section shows the number of teams created within the organization, the number of environments the team is assigned to, and the Control Plane and Deployed Cloud Data Planes within those environments. + +**Deployments** + +The default view for this section is Group by Control Plane and shows all deployments across all teams. + +<img src="/img/admin/tyk-cloud-org-deployments.png" alt="Deployments Grouped by Control Plane" /> + +**Environments** + +The Environments section shows the environments created within your organization, the team they belong to and active deployments within each environment. + +<img src="/img/admin/org_admin_environments.png" alt="Environments" /> + +#### Managing Environments + +Environments are used to group your [Control Plane](#glossary) and [Cloud Data Planes](#glossary) into logical groups. For example you may want to create environments that reflect different departments of your organization. + + +<Note> +The number of Environments you can create is determined by your [plan](#select-a-payment-plan) +</Note> + + +##### Prerequisites + +The following [user roles](#assign-user-roles) can perform Environment Admin tasks: + +- Org Admin +- Team Admin + +You should also have created a team to assign to any new environment. + +##### Adding a New Environment + +1. From the Environments screen, click **Add Environment** +2. Select the team you want to assign to the Environment +3. Give your new Environment a name +4. Click **Create** + +##### Editing an Existing Environment + +An Org Admin can perform the following: + +- Rename an Environment +- Delete an Environment + +1. Click the environment Name from your list + +<img src="/img/admin/tyk-cloud-edit-env.png" alt="Edit Environment Name" /> + +2. Click Edit + +<img src="/img/admin/tyk-cloud-env-screen.png" alt="Env Edit Screen" /> + +3. You can now rename the environment, or delete it from your organization + +<img src="/img/admin/tyk-cloud-rename-delete.png" alt="Delete or Rename Env" /> + + +<Warning> +Deleting an environment will also delete all the Control Planes and Cloud Data Planes associated with it +</Warning> + + +#### Managing Control Planes + +Control Planes are situated in your Organization's home region and provide links to an instance of the [Tyk Dashboard](/tyk-dashboard) and the [Developer Portal](/tyk-developer-portal). The Dashboard is where you perform all your API tasks. The developer portal allows your 3rd party developers access to your APIs. Cloud Data Planes are then connected to your Control Planes. + + +##### Prerequisites + +All [user roles](#assign-user-roles) can edit, deploy, undeploy, restart, redeploy all deployments within a team. Only the Organization Admin and the Team Admin can create or delete deployments. + +##### Adding a new Control Plane + +Watch our video on setting up a Control Plane and a Cloud Data Plane. + +<iframe width="560" height="315" src="https://www.youtube.com/embed/JqXXEDplrr8" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + + +<Note> +The number of Control Planes you can add is dependent on your [plan](#select-a-payment-plan) +</Note> + + +1. From the Deployments screen click **Add Deployment** (you can also add a Deployment from within an Environment overview) +2. Enter a name for the new Control Plane +3. Select Control Plane from the Type drop-down list +4. Select the Bundle Channel and Version +5. (Optional) Enter a [custom domain](#configure-custom-domains) if required +6. (Optional) Enable [plugins](#configure-plugins) if required + +##### Edit Control Planes + +You can edit the following Control Plane settings: +* Change the Control Plane name +* Add a [custom domain](#configure-custom-domains) +* Change the Bundle Channel and Bundle Version +* Enable [plugins](#configure-plugins) + + + + <Note> + The use of custom domains is dependent on your [plan](#select-a-payment-plan) + </Note> + + +To edit an existing Control Plane: + +1. From the Deployments screen, click the **Control Plane Name** from the list +2. Select **Edit** from the Deployed drop-down list + +<img src="/img/admin/cp-edit.png" alt="Edit drop-down" /> + +##### Upgrade Control Planes + +To upgrade an existing Control Plane: + +1. Go to the **Control Plane settings** using the _Edit Control Planes_ instructions and scroll down to the **Version** section. +2. Select a **Bundle Channel**. + +<img src="/img/admin/cp-edge-upgrade-channel.png" alt="Bundle channel drop-down" /> + +3. Next, select a **Bundle Version**. + +<img src="/img/admin/cp-edge-upgrade-version.png" alt="Bundle version drop-down" /> + +4. To apply your changes, click the **"Save and Re-Deploy"** button located at the top right. After a few seconds, you will be redirected to the overview page of the Control Plane and a **"Deploying"** indicator button will appear. + +<img src="/img/admin/cp-edge-upgrade-deploying.png" alt="Deploying notification" /> + +5. A **"Deployed"** button indicates a successful upgrade. + +<img src="/img/admin/cp-edge-upgrade-deployed.png" alt="Deployed notification" /> + +#### Managing Cloud Data Plane + +Cloud Data Planes do all the heavy lifting of actually managing your requests: traffic proxying, access control, data transformation, logging and more. + + +##### Prerequisites + +All [user roles](#assign-user-roles) can edit, deploy, undeploy, restart, redeploy all deployments within a team. Only the Organization Admin and the Team Admin can create or delete deployments. + + +##### Adding a new Cloud Data Plane + +Watch our video on setting up a Control Plane and a Cloud Data Plane. + +<iframe width="560" height="315" src="https://www.youtube.com/embed/JqXXEDplrr8" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + + +<Note> +The number of Cloud Data Planes you can add is dependent on your [plan](#select-a-payment-plan) +</Note> + + +1. From the Deployments screen click **Add Deployment** +2. Enter a name for the new Gateway +3. Select Cloud Data Plane from the Type drop-down list +4. Select the Bundle Channel and Version +5. (Optional) Enter a [custom domain](#configure-custom-domains) if required +6. (Optional) Enable [plugins](#configure-plugins) if required + +##### Edit Cloud Data Planes + +You can edit the following Control Plane settings: +* Change the Gateway name +* Add a [custom domain](#configure-custom-domains) +* Change the Bundle Channel and Bundle Version +* Enable [plugins](#configure-plugins) + + + + <Note> + The use of custom domains is dependent on your [plan](#select-a-payment-plan) + </Note> + + +To edit an existing Cloud Data Plane: + +1. On the Deployments screen, expand the Control Plane and click on the Cloud Data Plane to access the Cloud Data Plane overview screen. +2. Select **Edit** from the Deployed drop-down list + +<img src="/img/admin/cp-edit.png" alt="Cloud Data Plane drop-down" /> + + +##### Upgrade Cloud Data Planes + +To upgrade an existing Cloud Data Plane: + +1. Go to the **Cloud Data Plane settings** using the _Edit Cloud Data Planes_ instructions and scroll down to the **Version** section. +2. Select a **Bundle Channel**. + +<img src="/img/admin/cp-edge-upgrade-channel.png" alt="Bundle channel drop-down" /> + +3. Next, select a **Bundle Version**. + +<img src="/img/admin/cp-edge-upgrade-version.png" alt="Bundle version drop-down" /> + +4. To apply your changes, click the **"Save and Re-Deploy"** button located at the top right. After a few seconds, you will be redirected to the overview page of the Control Plane and a **"Deploying"** indicator button will appear. + +<img src="/img/admin/cp-edge-upgrade-deploying.png" alt="Deploying notification" /> + +5. A **"Deployed"** button indicates a successful upgrade. + +<img src="/img/admin/cp-edge-upgrade-deployed.png" alt="Deployed notification" /> + +#### Deploy Hybrid Gateways + +[Tyk Cloud](https://tyk.io/cloud/) hosts and manages the control planes for you. You can deploy the data planes across multiple locations: + +- as [Cloud Gateways](#managing-cloud-data-plane): Deployed and managed in *Tyk Cloud*, in any of our available regions. These are SaaS gateways, so there are no deployment or operational concerns. +- as Hybrid Gateways: This is a self-managed data plane, deployed in your infrastructure and managed by yourself. Your infrastructure can be a public or private cloud, or even your own data center. + +This page describes the deployment of hybrid data planes and how to connect them to Tyk Cloud, in both Kubernetes and Docker environments. + +##### Prerequisites + +* Tyk Cloud Account, register here if you don't have one yet: <ButtonLeft href="https://tyk.io/sign-up/#cloud" color="green" content="free trial" /> +* A Redis instance for each data plane, used as ephemeral storage for distributed rate limiting, token storage and analytics. You will find instructions for a simple Redis installation in the steps below. +* No incoming firewall rules are needed, as the connection between Tyk Hybrid Gateways and Tyk Cloud is always initiated from the Gateways, not from Tyk Cloud. + +##### Tyk Hybrid Gateway configuration + +The hybrid gateways in the data plane connect to the control plane in Tyk Cloud using the *Tyk Dashboard* API Access Credentials. Follow the guides below to create the configuration that we will be used in later sections to create a deployment: + +Login to your Tyk Cloud account deployments section and click on `ADD HYBRID DATA PLANE` + + <img src="/img/hybrid-gateway/tyk-cloud-hybrid-configuration-home.png" alt="Tyk Cloud hybrid configuration home" /> + +Fill in the details and then click _SAVE DATA PLANE CONFIG_ + + <img src="/img/hybrid-gateway/tyk-cloud-save-hybrid-configuration.png" alt="Save Tyk Cloud hybrid configuration home" /> + +This will open up a page with the data plane configuration details that we need. + + <img src="/img/hybrid-gateway/tyk-cloud-hybrid-masked-details.png" alt="Save Tyk Cloud hybrid configuration masked details" /> + +Those details are: +| | Docker | Helm | +| :-------------------------------------- | :------------------- | :------------------------ | +| key | api_key | gateway.rpc.apiKey | +| org_id | rpc_key | gateway.rpc.rpcKey | +| data_planes_connection_string (mdcb) | connection_string | gateway.rpc.connString | + +You can also click on _OPEN DETAILS_ + + <img src="/img/hybrid-gateway/tyk-cloud-hybrid-open-details.png" alt="Tyk Cloud hybrid open for details" /> + +This will reveal instructions that you can use to connect your hybrid data plane to Tyk Cloud. + +<img src="/img/hybrid-gateway/tyk-cloud-hybrid-revealed-instructions.png" alt="Tyk Cloud hybrid detailed instructions" /> + + +##### Deploy with Docker + +**1. In your terminal, clone the demo application [Tyk Gateway Docker](https://github.com/TykTechnologies/tyk-gateway-docker) repository** + +```bash +git clone https://github.com/TykTechnologies/tyk-gateway-docker.git +``` + + +**2. Configure Tyk Gateway and its connection to Tyk Cloud** + +You need to modify the following values in [tyk.hybrid.conf](https://github.com/TykTechnologies/tyk-gateway-docker#hybrid) configuration file: + +* `rpc_key` - Organization ID +* `api_key` - Tyk Dashboard API Access Credentials of the user created earlier +* `connection_string`: MDCB connection string +* `group_id`*(optional)* - if you have multiple data planes (e.g. in different regions), specify the data plane group (string) to which the gateway you are deploying belongs. The data planes in the same group share one Redis. + + +```json expandable +{ +"rpc_key": "<ORG_ID>", +"api_key": "<API-KEY>", +"connection_string": "<MDCB-INGRESS>:443", +"group_id": "dataplane-europe", +} +``` + +* *(optional)* you can enable sharding to selectively load APIs to specific gateways, using the following: + +```json expandable +{ + "db_app_conf_options": { + "node_is_segmented": true, + "tags": ["qa", "uat"] + } +} +``` + +**3. Configure the connection to Redis** + +This example comes with a Redis instance pre-configured and deployed with Docker compose. If you want to use another Redis instance, make sure to update the `storage` section in [tyk.hybrid.conf](https://github.com/TykTechnologies/tyk-gateway-docker#hybrid): + +```json expandable +{ + "storage": { + "type": "redis", + "host": "tyk-redis", + "port": 6379, + "username": "", + "password": "", + "database": 0, + "optimisation_max_idle": 2000, + "optimisation_max_active": 4000 + } +} +``` + +**4. Update docker compose file** + +Edit the `<docker-compose.yml>` file to use the [tyk.hybrid.conf](https://github.com/TykTechnologies/tyk-gateway-docker#hybrid) that you have just configured. + +From: + +```yml +- ./tyk.standalone.conf:/opt/tyk-gateway/tyk.conf +``` +To: + +```yml +- ./tyk.hybrid.conf:/opt/tyk-gateway/tyk.conf +``` + +**5. Run docker compose** + +Run the following: + +```bash +docker compose up -d +``` + +You should now have two running containers, a Gateway and a Redis. + +**6. Check that the gateway is up and running** + +Call the /hello endpoint using curl from your terminal (or any other HTTP client): + +```bash +curl http://localhost:8080/hello -i +```` + +Expected result: + +```http +HTTP/1.1 200 OK +Content-Type: application/json +Date: Fri, 17 Mar 2023 12:41:11 GMT +Content-Length: 59 + +{"status":"pass","version":"4.3.3","description":"Tyk GW"} +``` + +##### Deploy in Kubernetes with Helm Chart + +**Prerequisites** + +* [Kubernetes 1.19+](https://kubernetes.io/docs/setup/) +* [Helm 3+](https://helm.sh/docs/intro/install/) +* Connection details to remote control plane from the above [section](#deploy-hybrid-gateways). + +The following quick start guide explains how to use the [Tyk Data Plane Helm chart](/product-stack/tyk-charts/tyk-data-plane-chart) to configure Tyk Gateway that includes: +- Redis for key storage +- Tyk Pump to send analytics to Tyk Cloud and Prometheus + +At the end of this quickstart Tyk Gateway should be accessible through service `gateway-svc-hybrid-dp-tyk-gateway` at port `8080`. Pump is also configured with Hybrid Pump which sends aggregated analytics to Tyk Cloud, and Prometheus Pump which expose metrics locally at `:9090/metrics`. + +**1. Set connection details** + +Set the below environment variables and replace values with connection details to your Tyk Cloud remote control plane. See the above [section](#deploy-hybrid-gateways) on how to get the connection details. + +```bash +MDCB_UserKey=9d20907430e440655f15b851e4112345 +MDCB_OrgId=64cadf60173be90001712345 +MDCB_ConnString=mere-xxxxxxx-hyb.aws-euw2.cloud-ara.tyk.io:443 +MDCB_GroupId=your-group-id +``` + +**2. Then use Helm to install Redis and Tyk** + +```bash expandable +NAMESPACE=tyk +APISecret=foo +REDIS_BITNAMI_CHART_VERSION=19.0.2 + +helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ +helm repo update + +helm upgrade tyk-redis oci://registry-1.docker.io/bitnamicharts/redis -n $NAMESPACE --create-namespace --install --version $REDIS_BITNAMI_CHART_VERSION + +helm upgrade hybrid-dp tyk-helm/tyk-data-plane -n $NAMESPACE --create-namespace \ + --install \ + --set global.remoteControlPlane.userApiKey=$MDCB_UserKey \ + --set global.remoteControlPlane.orgId=$MDCB_OrgId \ + --set global.remoteControlPlane.connectionString=$MDCB_ConnString \ + --set global.remoteControlPlane.groupID=$MDCB_GroupId \ + --set global.secrets.APISecret="$APISecret" \ + --set global.redis.addrs="{tyk-redis-master.$NAMESPACE.svc.cluster.local:6379}" \ + --set global.redis.passSecret.name=tyk-redis \ + --set global.redis.passSecret.keyName=redis-password +``` + +**3. Done!** + +Now Tyk Gateway should be accessible through service `gateway-svc-hybrid-dp-tyk-gateway` at port `8080`. Pump is also configured with Hybrid Pump which sends aggregated analytics to Tyk Cloud, and Prometheus Pump which expose metrics locally at `:9090/metrics`. + +For the complete installation guide and configuration options, please see [Tyk Data Plane Chart](/product-stack/tyk-charts/tyk-data-plane-chart). + + +##### Remove hybrid data plane configuration + + +<Warning> +Please note the action of removing a hybrid data plane configuration cannot be undone. + +To remove the hybrid data plane configuration, navigate to the page of the hybrid data plane you want to remove and click _OPEN DETAILS_ +</Warning> + + + + <img src="/img/hybrid-gateway/tyk-cloud-hybrid-open-details.png" alt="Tyk Cloud hybrid open for details" /> + +Then click on _REMOVE DATA PLANE CONFIGS_ + + <img src="/img/hybrid-gateway/tyk-cloud-hybrid-remove-configs.png" alt="Tyk Cloud hybrid remove configs" /> + +Confirm the removal by clicking _DELETE HYBRID DATA PLANE_ + + <img src="/img/hybrid-gateway/tyk-cloud-hybrid-confirm-config-removal.png" alt="Tyk Cloud hybrid confirm removal of configs" /> + +#### Manage APIs + +You can manage your APIs in *Tyk Dashboard* UI. To access it, click on your desired Control Plane name in the [Deployments](https://dashboard.cloud-ara.tyk.io/deployments) screen and then on the *MANAGE APIS* button + +From there you have access to the full scope of Tyk API management functionality, including: + +* [Adding APIs](/api-management/gateway-config-managing-classic#create-an-api) to Tyk, including REST and GraphQL APIs +* Applying Quotas and Rate limits via [Security Policies](/api-management/gateway-config-managing-classic#secure-an-api) and [Keys](/api-management/gateway-config-managing-classic#access-an-api) +* [Securing](/api-management/security-best-practices#securing-apis-with-tyk) your APIs +* Viewing granular [Analytics](/api-management/dashboard-configuration#traffic-analytics) for your Tyk managed APIs +* [Transform traffic](/api-management/traffic-transformation) with the Tyk API Designer +* Add integration options such as [SSO](/api-management/external-service-integration#single-sign-on-sso) and [3rd Party IdentityProviders](/api-management/external-service-integration) +* [Adding Segment Tags](#faqs) + + +#### Secure Your APIs + +If you decide to use Tyk Cloud to protect your APIs, you need to make APIs accessible to your Tyk Cloud Data Planes so that Tyk can connect your clients to them. A common question that arises is, β€œhow do I secure my APIs (backend services)?”. + +Here are the most popular ways to secure your APIs. + +**1. Mutual TLS or Client authorization** + +- This is the most secure method to protect your APIs. With Client authorization, you need to add your Tyk Gateway certificates to an allow-list in all your backends and they will then accept access requests only from clients that present these pre authorized certificates. There are a few limitations with this approach: + + a. Depending on your setup, you might need to add it to every backend service. If you have a Load Balancer (LB), then it can be set at the LB level. + + b. Sometimes the LBs (like Application Load Balancers) do not support mTLS and then you need to find other solutions, like Request Signing (below). Another option that might be possible, is to front your services or your LB with an L7 API Gateway (Like Tyk!) to do mTLS with the Tyk Cloud Data Planes on Tyk Cloud. + +- You need to be able to update the list in case certificates expire or get revoked. + +**2. Request Signing** + +Tyk can [sign the request with HMAC or RSA](/developer-support/release-notes/archived#hmac-request-signing), before sending it to the API target. This is an implementation of an [RFC Signing HTTP Messages(draft 10)](https://datatracker.ietf.org/doc/html/draft-cavage-http-signatures-10). This RFC was designed to provide authenticity of the digital signature of the client. In our flow, the Tyk Cloud Data Planes, as the client, using a certificate private key, will add a header signature to the request. The API, using a pre-agreed public key (based on a meaningful keyId identifier) will verify the authenticity of the request coming from your Tyk Cloud Data Plane. + A limitation is that the APIs or LB need to implement this signature verification and be able to update the certificates as mentioned in Mutual TLS or Client authorization (above). + +**3. IP Whitelisting** + + Each Tyk Cloud organization is dedicated to an IP range which is unique to them. This allows you to restrict access to your APIs to only API requests coming from your Tyk Cloud organization. + +IP Whitelisting is susceptible to IP Spoofing, and it is recommended to be combined with an API Key in secure environments. + +In order to find your organization’s IP range, please open a support ticket with our support team, which is available to all paying customers. + +**4. Post plugin with OAuth flow** + +The custom plugin approach is mentioned last because it involves writing a bit of code. However, if your appetite allows for it, custom plugins offer the most flexibility of all these solutions. You can use Tyk’s custom plugins to execute an OAuth flow, for example, between Tyk (as the client) and your authorization server, and inject a Bearer token into the request. The backend service will need to validate the bearer as usual. You can write [custom plugins](#configure-plugins) in a variety of languages. + +**Where to Authenticate?** + +No matter which option or combination of options you choose, it is best to keep this authentication layer outside your application logic. This glue code should be placed in your ingress, whatever that might be. By keeping this logic outside your application, you keep a separation between the business logic and the boilerplate code. You can even use the Tyk Open Source API Gateway as an ingress to protect your APIs, and it is compatible with all the methods mentioned above. + + + +### Managing Teams and Users + +This section covers the following: + +- [Managing Teams](#managing-teams) +- [Managing Users](#managing-teams-and-users) +- Available Tyk Cloud [User Roles](#user-roles-in-tyk-cloud) +- [Tyk Cloud Single Sign-On (SSO)](#configure-single-sign-on-sso) + +#### Managing Teams + +The following [user roles](#user-roles-in-tyk-cloud) can perform existing Team Admin tasks: + +* Organization Admin - Can manage all teams in the organization they are a member of. +* Team Admin - Can only manage the team they are a member of. + +For an existing team, you can: + +* Change the team name +* Create or delete a team (Organization Admin only) +* Invite and manage users in a team + +##### Change the Team Name + +1. From the Teams screen, select the team name. +2. Click **Edit**. +3. Change the existing name for the team. +4. Click **Save**. + +##### Create a New Team + +You need to be a [Organization Admin](#assign-user-roles) to create a new team. + +1. From the Admin > Teams screen, click **Add Team**. +2. Enter a name for the new team that will be added to the organization. +3. Click **Create**. + +##### Delete a Team + +You need to be a [Organization Admin](#assign-user-roles) to delete a team. + +1. From the Teams screen, select the team name. +2. Click **Edit**. +3. Click **Delete Team**. +4. You'll be asked to confirm the deletion. Click **Delete Team** from the dialog box to confirm, or click **Cancel**. + +You can now invite users to your new team. See [Managing Users](#managing-teams-and-users) for more details. + + +#### Managing Users + +The following [user roles](#user-roles-in-tyk-cloud) can perform existing User Admin tasks: + +* [Organization Admin](#assign-user-roles) - Can manage all users in the organization they are a member of. +* [Team Admin](#assign-user-roles) - Can only manage the users of the team they are a member of. + + + + <Note> + Organization Admins, Team Admins and Team Members are responsible for managing the Tyk Cloud organization hierarchy and deploying/managing stacks, as well as having access to the Tyk Dashboard to manage APIs. Users of Tyk Cloud are usually DevOps, Architects and sometimes Engineers or Managers. + + You can also [add users to the Tyk Dashboard](/api-management/user-management#manage-tyk-dashboard-users) itself instead of inviting them as Tyk Cloud users. These users would likely be your API Developers and Engineers who manage the APIs. + </Note> + + +##### Invite a new user to your team + +1. From the Teams screen, select the team name. +2. Click **Invite User**. +3. Complete the form for the new user. + +##### Editing Existing Users + +1. Select the team with the user you want to edit. +2. Click the user name from the team user list. +3. You can change the following details + * Change the team they are a member of. + * Change the user role assigned to them. +4. Click Save to update the user info. + +##### Delete a User + +1. Select the team with the user you want to edit. +2. Click the user name from the team user list. +3. Click **Delete** +4. You'll be asked to confirm the deletion. Click **Delete User** from the pop-up box to confirm, or click **Cancel**. + + +#### Assign User Roles + +This section defines the different user roles within Tyk Cloud, so that you can see at a glance what each role does and manage your account accordingly. + +##### User Roles within Tyk Cloud + +We have the following user roles defined in Tyk Cloud for your team members + +* Billing Admin +* Organization Admin +* Team Admin +* Team Member + +Billing Admins are responsible for the billing management of the Tyk Cloud account. Organization Admins, Team Admins and Team Members are responsible for managing the Tyk Cloud organization hierarchy and deploying/managing stacks, as well as having access to the Tyk Dashboard to manage APIs. Users of Tyk Cloud are usually DevOps, Architects and sometimes Engineers or Managers. + +You can [add users to the Tyk Dashboard](/api-management/user-management#manage-tyk-dashboard-users) itself instead of inviting them as Tyk Cloud users. These users would likely be your API Developers and Engineers who manage the APIs. + +##### Use Case Matrix + +The following table shows the scope for each user role. + + +| Use Case | Billing Admin | Org Admin | Team Admin | Team Members | +| :--------------------------------------------------- | :--------------- | :----------- | :------------ | :-------------- | +| Create a new account | X | | | | +| Create a new organization | X | | | | +| Managing a new account | X | | | | +| Managing an organization entitlement | X | | | | +| Ability to create other billing admins | X | | | | +| Editing organization name | X | X | | | +| Create team / delete | | X | | | +| Future - Edit team entitlements | | X | | | +| Invite, delete, edit org admins and team admins | | X | | | +| Invite, delete, edit team members | | X | X | | +| Create new environments | | X | X | | +| Delete / change environments | | X | X | | +| View environments | | X | X | X | +| Create and delete cloud data planes | | X | X | | +| Create and delete control planes | | X | X | | +| View deployments | | X | X | X | +| Deploy, undeploy, redeploy, restart a deployment. | | X | X | X | +| Create and manage basic SSO | | X | X | | +| Upload plugins to the File Server | | X | X | X | +| Delete plugins from File Server | | X | X | X | +| Viewing Analytics | | X | X | X | + +##### Initial Tyk Cloud Account Roles + +The user who signs up for the initial Tyk Cloud account is uniquely assigned to two roles: + +1. Org admin of the organization +2. Billing admin of the account + +This is the only occasion where a user can be assigned to 2 roles. So, for example, if you invite a user to be a Team Admin, that is the only role (and team) they can belong to. For a user to be invited as a Billing admin, they can't have an existing Tyk Cloud account. + + +<Note> +This functionality may change in subsequent releases. +</Note> + + +**Tyk System Integration User (do not delete)** + +When you click your Control Plane Dashboard link from your Tyk Cloud Deployments Overview screen, you are automatically logged in to your Dashboard. This is due to a default Tyk Integration user that is created as part of the Control Plane deployment process. This user has a first name of `Tyk System Integration` and a last name of `User (do not delete)`. As the last name infers, you should not delete this user or your access to the Dashboard will be broken from your Tyk Cloud Installation. + + +#### Configure Single Sign-On (SSO) + +**What is SSO?** +Single Sign-On (SSO) is an authentication process that empowers users to access various services and applications using a single set of credentials. This streamlines the login experience by eliminating the need to remember multiple usernames and passwords for different platforms. + +These credentials are securely stored with an Identity Provider(IdP). An Identity Provider (IdP) is a service that stores and manages digital identities. Companies can use these services to allow their employees or users to connect with the resources they need. + +**Pre-requisites** + +<Note> +This functionality is a preview feature and may change in subsequent releases. + +To be able to configure Single Sign-On authentication, an SSO entitlement needs to be enabled in the subscription plan. +If you are interested in getting access, contact your account manager or reach out to our [support@tyk.io](<mailto:support@tyk.io?subject=Tyk Cloud Single sign on>) +</Note> + + +##### Add new SSO profile +To add a new SSO profile, login to Tyk Cloud, navigate to the _Profile_ list and click on the _ADD PROFILE_ button. + + <img src="/img/cloud/cloud-sso-profile-list.png" alt="Tyk Cloud SSO profile list" /> + +Populate the form with all the mandatory fields and click the _ADD PROFILE_ button. + + <img src="/img/cloud/cloud-sso-add-profile-form.png" alt="Tyk Cloud SSO add profile form" /> + +The table below explains the fields that should be completed: +| Field name | Explanation | +| :---------------------- | :--------------------------------------------------------------------------------------------------------------------------------- | +| Provider name | Used to distinguish between different SSO providers. | +| Client ID | Used for client authentication with the IdP provider. The value can be found in your chosen IdP provider's configuration. | +| Client Secret | Used for client authentication with the IdP provider. The value can be found in your chosen IdP provider's configuration. | +| Discovery URL | Used to read your IdP's openid configuration. This URL can be found in your chosen IdP provider's configuration. | +| Default User Group ID | The ID of the user group that new users are allocated to by default upon registration. | +| Only registered users | A check-box that defines which users are allowed to use SSO. If checked, only users who are already registered in Tyk Cloud are allowed to login with SSO. If un-checked, new users will be added to Tyk Cloud in the _Default_ user group upon successful registration. | + + +As illustrated below an authentication and callback URL will be generated, once the new profile has been added. You need to add these URLs to the configuration of your chosen IdP provider. +The Auth URL is your custom URL that can be used to start the SSO login flow whereas Callback URL is the URL that the SSO provider will callback to confirm successful authentication. + + <img src="/img/cloud/cloud-sso-add-config-details.png" alt="Tyk Cloud SSO example of filled form" /> + +##### Edit SSO profile + +To update/re-configure SSO profile, login to Tyk Cloud, navigate to _Profile_ list and click on the profile that you would like to update. + + <img src="/img/cloud/cloud-sso-edit-select.png" alt="Tyk Cloud SSO edit selection" /> + +Edit the fields you would like to change and then click on the _SAVE PROFILE_ button. + + <img src="/img/cloud/cloud-sso-save-edit.png" alt="Tyk Cloud SSO save edit selection" /> + +##### Delete SSO profile + + +<Warning> +Please note the action of deleting an SSO profile cannot be undone. + +To delete an SSO profile, login to Tyk Cloud, navigate to _Profile_ list and click on the profile you would like to remove. +</Warning> + + + <img src="/img/cloud/cloud-sso-delete-select.png" alt="Tyk Cloud SSO delete selection" /> + +On the profile page, click on the _DELETE PROFILE_ button. + + <img src="/img/cloud/cloud-sso-delete-click.png" alt="Tyk Cloud SSO delete accept" /> + +On the confirmation window, confirm by clicking the _DELETE PROFILE_ button. + + <img src="/img/cloud/cloud-sso-delete-confirm.png" alt="Tyk Cloud SSO delete confirm" /> + +After profile deletion, the authentication URL will not be available anymore. + + +### Manage Accounts and Billing +This section covers the following: + +* The available [Tyk Cloud Plans](#select-a-payment-plan) +* Adding [Payment Methods](#add-payment-methods) +* How to [upgrade from the free trial plan](#upgrade-your-free-trial) +* [Managing Billing Admins](#managing-billing-admin) on your account +* What to do if your account goes into [retirement](#retire-your-account) + +#### Select a Payment Plan + +Our plans cover every use case scenario, from a free trial, to a global enterprise ready plan. You can also purchase addons to increase your functionality. For details on our available plans and pricing go to [Tyk Cloud Pricing](https://tyk.io/price-comparison/). + +Here's an overview of all of the available plans: + +| **Plan** | **Who's it for?** | **About** | +| :----------------- | :----------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 48 hours free trial <p>[Free Support SLA](/developer-support/support)</p> | This is for POC’s and those testing the Tyk platform. | Tyk Cloud has 48 hours free trial. You can always request a longer trial period or talk to support if you need help. | +| Starter <p>[Standard Support SLA](/developer-support/support)</p> | For single teams with low traffic, mostly small businesses that manage few APIs. | This plan includes all of the features of the Tyk Stack. You can have **[custom domains](#configure-custom-domains)** and **[plugins](#configure-plugins)**, along with management of upto 5 APIs. Standard support is provided.| +| Launch <p>[Standard Support SLA](/developer-support/support)</p> | For single teams with low traffic, mostly small businesses. | This plan includes all of the features of the Tyk Stack. You can have **[custom domains](#configure-custom-domains)** and **[plugins](#configure-plugins)** along with management of unlimited APIs. Standard support is provided. | +| Grow <p>[Standard Support SLA](/developer-support/support)</p> | For single teams with high traffic. | This plan includes all of the features of the Tyk Stack. In this plan, you have **[Hybrid Gateways](#deploy-hybrid-gateways)** as an add on, along with standard support. | +| Scale <p>[Enhanced Support SLA](/developer-support/support)</p> | For global organizations with multiple teams, requiring gateway deployments in multiple locations. | This plan includes all of the features of the Tyk Stack. **Enhanced(silver) support** will be provided. | + + +<ButtonLeft href="https://tyk.io/sign-up/#cloud" color="green" content="Get started with Cloud free trial" /> + +**Available add ons** + +You can purchase the following addons, depending on your plan. + +- Additional Control Planes +- Additional Cloud Data Planes +- Additional Users +- Additional Teams +- Additional Gateway Region (Enterprise Plans only) +- SLA support (varies according to your plan) + +**Boostable Overages** + +Your selected plan comes with limited throughput per month. Overages allow your consumption to go over the monthly limit, which can be an easy way to deal with for example seasonal or unexpected spikes in traffic. The overage level will be automatically charged based on your throughput on a monthly basis. + +| Launch | Entitlement $1,800 | Overage 1 $2,700 | Overage 2 $3,240 | Overage 3 $3,780 | Overage 4 $4,320 | +| --------------- | :----------------: | :--------------: | :--------------: | :--------------: | :--------------: | +| Calls (at 10kb) | 15m | 18.75m | 22.5m | 26.5m | 30m | +| Throughput | 150GB | 187.5GB | 225GB | 262.5GB | 300GB | + +| Grow | Entitlement $3,800 | Overage 1 $5,700 | Overage 2 $6,840 | Overage 3 $7,980 | Overage 4 $9,120 | +| --------------- | :----------------: | :--------------: | :--------------: | :--------------: | :--------------: | +| Calls (at 10kb) | 100m | 125m | 150m | 175m | 200m | +| Throughput | 1TB | 1.25TB | 1.5TB | 1.75TB | 2TB | + +| Scale | Entitlement $6,800 | Overage 1 $10,200 | Overage 2 $12,240 | Overage 3 $14,280 | Overage 4 $16,320 | +| --------------- | :----------------: | :---------------: | :---------------: | :---------------: | :---------------: | +| Calls (at 10kb) | 1b | 1.25b | 1.5b | 1.75b | 2tb | +| Throughput | 10TB | 12.5TB | 15TB | 17.5TB | 20TB | + +**Changing plans** + +You can upgrade or downgrade your current plan at any point in your billing cycle and your payment will be pro-rata'd to reflect the new payment. + +**Downgrading plan requirements** + +If you downgrade your plan, the new plan may have less entitlements than your current plan. You will need to restructure your organization to comply with the new plan entitlements before the downgraded billing starts. + +##### Checking the Tyk Cloud status + +If you want to check if there are issues with the environments and any upcoming down times, you can go to the [Tyk Status](https://status.tyk.io/) page. + + +#### Add Payment Methods + +This section provides a step-by-step guide on how to add a payment method to your Tyk Cloud account, ensuring uninterrupted access to your API management services. + +**Adding a payment method to your account** + +**Note:** You must have *Billing Admin* user rights to add a payment method. + +Follow these steps: + +1. Ensure you are logged in to *Tyk Cloud UI* as a Billing Admin user. +2. Navigate to <a href="https://account.cloud-ara.tyk.io/payment-method" className="external-links" target="_blank" rel="noopener">ACCOUNT & BILLING --> Payment Method</a>. If you lack the necessary user rights, you will be directed to the main [OPERATIONS](https://dashboard.cloud-ara.tyk.io/) screen (the main login page). +3. Enter your card details and click *Save*. +4. You'll see a confirmation that the payment method was successfully added. + + + + <Note> + **Note about card payments** + + Currently, *Tyk Cloud* exclusively supports card payments. For alternative payment methods, please [contact us](https://tyk.io/contact/). + </Note> + + +**Payment Method Maintenance** + +As a *Billing Admin* user, you have the ability to edit or delete an existing payment method. Deleting a payment method without adding a new one will result in your plan going into [retirement](#retire-your-account) at the end of your current billing cycle. + +#### Upgrade Your Free Trial + +This section explains how you can upgrade your free trial of Tyk Cloud to a full account, to continue enjoying the benefits of Tyk Cloud. + +**My free trial is coming to an end. What are my options?** + +Every new Tyk Cloud account is assigned to a 48 hour free trial. You have the following options: + +* Upgrade to a paid plan at any stage of the free trial period. +* Use the free trial period and upgrade after it expires + +**What happens if my free trial expires?** + +If your free trial ends without you upgrading, your account enters what we call [retirement](#glossary). + +**What does upgrading a free trial account involve?** + +To upgrade your free trial, you (as a Billing Admin) need to: +* Add a [payment method](#add-payment-methods) to your organization +* Select a new [plan](#select-a-payment-plan) from our list + +**I've trialled more than what my desired paid plan allows.** + +During the free trial we give you the same access as our Enterprise Global plan. When you come to the end of your free trial, you may want to subscribe to a plan such as 'Proof of Concept' which only allows 1 Environment, Cloud Control Plane and Cloud Data Plane. If you had an excess of these set up during your free trial, you would need to remove the appropriate amount of Environments etc from your Organization in order to start the paid plan. But don't worry, we'll let you know what you need to remove when you go to purchase a plan. + + +#### Managing Billing Admin + +This page explains what a Tyk Cloud billing admin can do as part of your API management process, giving you complete control over your API management. + +As a Billing Admin you can perform the following: + +* Add, edit and delete [payment methods](#add-payment-methods) +* Add further users as Billing Admins +* Upgrade or downgrade plans + +##### Adding a new Billing Admin + + +<Note> +To be added as a Billing Admin, a user cannot have an existing Tyk Cloud account. +</Note> + + +**Prerequisites** + +To add a new Billing Admin team member requires you to have one of the following roles: + +* Be an existing Billing Admin +* Be the account creator Organization Admin (this user also has the Billing Admin role assigned to them) + +1. Select **Account & Billing** from the Admin menu (if you only have Billing Admin permissions you will automatically be logged into the Account and Billing area). + +<img src="/img/admin/tyk-cloud-account-billing-menu.png" alt="Account & Billing menu" /> + +2. Select **Billing Admins** from the Accounts & Billing menu + +<img src="/img/admin/billing-admins.png" alt="Billing Admins menu" /> + +3. Click **Invite Billing Admin** + +<img src="/img/admin/invite-billing-admin.png" alt="Invite Billing Admin" /> + +4. Complete the Billing Admin form and click **Send Invite** + +##### Removing Billing Admin Access + +For this release, removing a billing Admin is not allowed. We can remove a Billing Admin manually, so contact your Account Manager if you need to remove a Billing Admin user. + +#### Retire Your Account + +This section explains what it means when your Tyk Cloud account goes into retirement and what your options are when it does, from account reinstatement to closure. + +Your plan will go into [retirement](#glossary) in the following scenarios: + +* Your subscription is manually canceled by a Billing Admin. +* Your periodic subscription payment fails. +* You are on a Free Trial (5 weeks) and have not signed up to a plan before the expiration of the Free Trial. + +**What does retirement mean?** + +When a plan goes into retirement, it means your Organization, Teams and any Environmenmts and APIs you manage are suspended for a grace period of up to 30 days and you won't be able to add or edit, only remove. + +**How can I end retirement?** + +Your Billing Admin needs to do one of the following: + +* Renew a subscription that was manually canceled. +* Update your payment details and any outstanding payments are cleared. +* Update a Free Trial to a paid plan, and payment is successful. + +**What happens at the end of the 30 day retirement period?** + +At the end of the 30 day retirement period if you have not restored or created a relevant subscription, all your data will be deleted. + + + + +### Configure Developer Portal from Dashboard + +After deploying your Control Plane, you need to perform some initial configuration for the Developer Portal to prevent seeing any `Page Not Found` errors when trying to access the portal. You need to set up a Home page from the Control Plane Dashboard. + +Watch our video on configuring your Tyk Cloud Developer Portal. + +<iframe width="560" height="315" src="https://www.youtube.com/embed/8_SPUake84w" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + +1. From the Control Plane Dashboard, select **Pages** from the **Portal Management** menu +2. Click **Add Page** + +<img src="/img/getting-started/create-account-portal-pages.png" alt="Add Portal Page" /> + +3. In the Settings, give your page a name and slug. Below we've called it Home +4. Select **Check to make this page the Home page** +5. Select **Default Home page template** from the Page type drop-down list +6. You can leave the Registered Fields sections for now + +<img src="/img/getting-started/portal-home-page-settings.png" alt="Portal Home page settings" /> + +7. Click **Save**. + +You should now be able to access your Portal from **Open Your Portal** from the **Your Developer Portal** menu. + +<img src="/img/getting-started/portal_menu.png" alt="Portal Menu" /> + +#### Further Portal Configuration + +Our Developer Portal is completely customizable. See [Portal Customization](/tyk-developer-portal/customise) for more details. + +### Configure Custom Domains + +You can set up Tyk Cloud to use a custom domain. Using custom domains is available on our free trial and all our paid [plans](https://tyk.io/price-comparison/). You can use a custom domain for both your **Control Planes** and **Cloud Data Planes**. + + +<Note> +**note** + +Wild cards are not supported by Tyk Cloud in custom domain certificates +</Note> + + +#### Custom Domains with Control Planes + +* Currently, you can only use **one custom domain** per Control Plane deployment. +* The custom domain in this case ties to a **Tyk Developer Portal**. Please set up a **CNAME DNS** record such that it points to the "Portal" ingress as displayed on your Control Plane deployment page. + +#### Custom Domains with Cloud Data Planes + +You can set multiple custom domains on a Cloud Data Plane. In this instance please set up your CNAME DNS records such that they point to the only ingress displayed on your Cloud Data Plane deployment page. + +Note: While you can set multiple custom domains for a Cloud Data Plane, a single custom domain cannot be used for multiple Cloud Data Planes. + +#### How to set up a Custom Domain + +In this example we are going to set up a custom domain called `Cloud Data Plane.corp.com` for a Cloud Data Plane deployment. + +1. Create a CNAME DNS record `edge.corp.com` that points to your Cloud Data Plane ingress (e.g. `something-something.aws-euw2.cloud-ara.tyk.io`). +2. From your Cloud Data Plane deployment, select **Edit** from the Status drop-down. + +<img src="/img/2.10/edge-dropdown.png" alt="Cloud Data Plane drop-down" /> + +3. Enter `edge.corp.com` in the Custom Domains field. + +<img src="/img/2.10/edge_custom_domain.png" alt="Cloud Data Plane Custom Domain" /> + +4. Click **Save and Re-deploy**. + +<img src="/img/2.10/save_redeploy.png" alt="Save and Re-Deploy" /> + +#### How our Custom Domain functionality works + +When you point your custom domain to your deployment, we use [Let\'s Encrypt\'s](https://letsencrypt.org/docs/challenge-types/#http-01-challenge) **HTTP01 ACME** challenge type, which verifies ownership by accessing your custom CNAME on your Control Plane or Cloud Data Plane deployment. For example - `something-something.aws-euw2.cloud-ara.tyk.io` above. + +### Configure Plugins + +This section explains that you can use plugins with Tyk Cloud and links to details of Python, JSVM and Golang based plugins. + +Tyk Cloud allows you to take advantage of Tyk's plugin architecture that allows you to write powerful middleware. For this version of Tyk Cloud, we support the use of Python, JavaScript Middleware and Golang based plugins. + +For more details, see: +* [Python Plugins](/api-management/plugins/rich-plugins#overview) +* [JSVM](/api-management/plugins/javascript#) +* [Golang](#configure-plugins) + +Next you'll set up an Tyk Cloud Control Plane to use a Python Authentication Plugin. + +#### Setup Control Plane + + +This page explains how to set up a control plane with plugins to customize it on Tyk Cloud, so that you can ensure your API management solution is as effective as possible. + +**What do I need to do to use Plugins?** + +<img src="/img/plugins/plugins_enable.png" alt="Plugins Settings" /> + +1. You need to enable Plugins on a Control Plane and on a Cloud Data Plane. +2. You need to enter Provider details to enable you to store and access your plugins. For this version of Tyk Cloud, we are supporting Amazon AWS S3. If you haven't got an AWS S3 account, go to [https://aws.amazon.com/s3/](https://aws.amazon.com/s3/) and set one up. You will need the following details to configure SW3 within your Control Plane: + * Your AWS Key ID + * Your AWS Secret + * Your AWS Region + + + + <Note> + For this release of Tyk Cloud, you need to enter your AWS Region manually. You also need to consider that uploading a custom plugin bundle to Tyk Cloud results in a new bucket being created for each bundle uploaded. It also requires that Tyk Cloud has permissions in the form of an AWS IAM policy to have create rights on AWS. + </Note> + + +**AWS IAM Policy** + +**What is an IAM Policy?** + +- A policy is an entity that, when attached to an identity or resource, defines their permissions. IAM policies define permissions for an action regardless of the method that you use to perform the operation. + +- We have included a sample IAM policy that you need to create in AWS to allow the plugin bundle to work. For more information on creating IAM policies, see the [AWS Documentation](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create-console.html). + + + + <Warning> + We recommend you restrict your IAM user as much as possible before sharing the credentials with any 3rd party, including Tyk Cloud. See [IAM User Permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_change-permissions.html) for more details. + </Warning> + + +```.json expandable +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "s3:CreateBucket", + "s3:ListBucket", + "s3:GetBucketLocation", + "s3:DeleteBucket" + ], + "Resource": "arn:aws:s3:::mserv-plugin-*" + }, + { + "Effect": "Allow", + "Action": "s3:ListAllMyBuckets", + "Resource": "*" + }, + { + "Sid": "VisualEditor1", + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:DeleteObject" + ], + "Resource": "arn:aws:s3:::mserv-plugin-*/*" + } + ] +} +``` + +Next you'll [set up the Python authentication code bundle](/tyk-cloud#create-a-python-code-bundle). + + +#### Uploading your Bundle + +This section walks you through uploading your bundle as part of the process of Python custom authentication on Tyk Cloud, so that you can ensure your API management solution is as effective as possible. + +**How do I upload my bundle file to my Amazon S3 bucket?** + +We are going to use a Tyk CLI tool called **mservctl**. This acts as a file server for our plugins. You use it to push your plugin bundle to your S3 bucket. Your Tyk Cloud Tyk Gateway will use **MServ** to retrieve your bundle, instead of connecting directly into S3. + +**Prerequisites** + +1. You need to install the mserv binary according to your local environment from the following repo - https://github.com/TykTechnologies/mserv/releases. Linux and MacOS are supported. + +2. From your Control Plane you need the following settings. + +<img src="/img/plugins/fileserver_settings.png" alt="File Server Settings" /> + + * Your Tyk Cloud Control Plane Ingress File Server Endpoint (1) + * Your File Server API Key (2) + +**How does mservctl work?** + +You create a config file (in YAML) that contains your Control Plane settings that connects to your S3 bucket. You then use a `push` command to upload your `bundle.zip` file to your bucket. + +**mservctl settings - Mac** + +To run `mservctl` from your local machine, from the binary directory, run: + +```.bash +./mservctl.macos.amd64 +``` +**mservctl settings - Linux** + +To run `mservctl` from your local machine, from the binary directory, run: + +```.bash +./mservctl.linux.amd64 +``` + +The help for mservctl will be displayed. We will be using the config file options for this tutorial. + +```.bash expandable +$ mservctl help +mservctl is a CLI application that enables listing and operating middleware in an Mserv instance. +Use a config file (by default at $HOME/.mservctl.yaml) in order to configure the Mserv to use with the CLI. +Alternatively pass the values with command line arguments, e.g.: +$ mservctl list -e https://remote.mserv:8989 +Set TYK_MSERV_LOGLEVEL="debug" environment variable to see raw API requests and responses. +Usage: + mservctl [command] +Available Commands: + delete Deletes a middleware from mserv + fetch Fetches a middleware record from mserv + help Help about any command + list List middleware in mserv + push Pushes a middleware to mserv +Flags: + --config string config file (default is $HOME/.mservctl.yaml) + -e, --endpoint string mserv endpoint + -h, --help help for mservctl + -t, --token string mserv security token +Use "mservctl [command] --help" for more information about a command. +``` + + +<Note> +You may have to change the CHMOD settings on the binary to make it executable. (`chmod +x <filename>`). On MacOS you may also need to change your security settings to allow the binary to run. +</Note> + + +**Creating the mserv config file** + +1. Create a file (we'll call it `python-demo.mservctl.yaml`) +2. Copy your Control Plane File Server endpoint URL and use it for your `endpoint` flag. Remember to prepend it with `https://`. +3. Copy your File Server API Key and use it for your `token` flag + +Your `python-demo.mservctl.yaml` config file should now look like this: + +```.yaml +endpoint: https://agreeable-native-mgw.usw2.ara-staging.tyk.technology/mserv +token: eyJvcmciOiI1ZWIyOGUwY2M3ZDc4YzAwMDFlZGQ4ZmYiLCJpZCI6ImVmMTZiNGM3Y2QwMDQ3Y2JhMTAxNWIyOTUzZGRkOWRmIiwiaCI6Im11cm11cjEyOCJ9 +``` + +**Uploading To Your S3 Bucket** + +1. We are going to use the MacOS binary here, just substitute the binary name for the Linx version if using that OS. Note we have our YAML config file in the same directory as our bundle.zip file. Run the following mserv `push` command: + +```.bash +./mservctl.macos.amd64 --config ~/my-tyk-plugin/python-demo.mservctl.yaml push ~/my-tyk-plugin/bundle.zip +``` +2. You should get confirmation that your middleware has been uploaded to your S3 bucket. + +```.bash +INFO[0000] Using config file:/Users/marksouthee/my-tyk-plugin/python-demo.mservctl.yaml app=mservctl +Middleware uploaded successfully, ID: 9c9ecec1-8f98-4c3f-88cd-ca3c27599e6b +``` +3. You will notice that the middleware uploaded has been given an ID. We are going to use that ID with an API that allows you to specify specific middlware. You can also check the contents of the middleware you have just uploaded using the mservctl `list` command. Run: + +```.bash +./mservctl.macos.amd64 --config ~/my-tyk-plugin/python-demo.mservctl.yaml list +``` + +4. You will see the list of middleware you have pushed to your S3 Bucket + +```.bash +INFO[0000] Using config file:/Users/marksouthee/my-tyk-plugin/python-demo.mservctl.yaml app=mservctl + ID ACTIVE SERVE ONLY LAST UPDATE + + 9c9ecec1-8f98-4c3f-88cd-ca3c27599e6b true false 2020-05-20T15:06:55.901Z + ``` +5. If you use the -f flag with the list command, you will see the functions within your middleware listed: + +```.bash +./mservctl.macos.amd64 --config ~/my-tyk-plugin/python-demo.mservctl.yaml list -f +``` +6. As you can see, the 2 middleware hooks specified within your `manifest.json` are returned: + +```.bash +INFO[0000] Using config file:/Users/marksouthee/my-tyk-plugin/python-demo.mservctl.yaml app=mservctl + ID ACTIVE SERVE ONLY LAST UPDATE FUNCTION TYPE + + 9c9ecec1-8f98-4c3f-88cd-ca3c27599e6b true false 2020-05-20T15:06:55.901Z + MyPostMiddleware Post + MyAuthMiddleware CustomKeyCheck +``` expandable + +Next you will [create an API](#test-middleware) from our Control Plane and see our middleware in action. + +#### Test Middleware + +This section explains how to test out your Python custom authentication on Tyk Cloud, to ensure that it’s working properly. + +**Testing our middleware with an API** + +You now have your middleware uploaded to your S3 bucket. We are now going to create an API from our Control Plane Dashboard and test it via Postman + +**Prerequisites** + +* A Postman API Client from https://www.postman.com/product/api-client/ +* Your mserv middleware ID +* The `auth` value token from your `middleware.py` code + +**Create your API** + +1. From your Control Plane in Tyk Cloud, click the *Ingress > Dashboard link* + +<img src="/img/plugins/control_plane_dashboard_link.png" alt="Dashboard Link" /> + +2. From the Dashboard screen, click **APIs** from the System Management menu + +<img src="/img/plugins/apis_menu.png" alt="APIs Menu" /> + +3. Click **Add New API** +4. From the API Designer, enter the following in the **Core Settings** tab: + * From the API Settings section, give your API a name. We'll name this example "test" + * Scroll down to the Authentication section and select **Custom authentication (Python, CoProcess and JSVM plugins)** from the drop-down menu + * Select the **Allow query parameter as well as header** option +5. From the Advanced Settings tab enter the following: + * In the Plugin Options, enter the **Plugin Bundle ID** as returned by mservctl. In our example `9c9ecec1-8f98-4c3f-88cd-ca3c27599e6b` + * To propagate your API to all your Cloud Data Plane Tyk Gateways connected to your Control Plane, you need to add the tag **edge** in the **API Segment Tags section** +6. Click **Save**. + +You now have an API called "test" which has as its target the httpbin test site. + +**Testing Your API** + +You now need to test your API to show how the Python Authorization middleware works. We are going to use the Postman client for our testing. + +1. First, a quick test. Copy the URL of your Cloud Data Plane (Note the "edge" tag in the tags column) and paste it in a browser tab. You should get a **404 page not found error**. +2. Then add the "test" endpoint to the URL in your browser tab, so in our example `uptight-paddle-gw.usw2.ara.app/test/`. You should now see a **403 "error: "forbidden"**. This is because your API has Authentication enabled and you haven't provided the credentials yet. +3. Open up your Postman client: + * Paste your Gateway URL with the API appended to the request - (`uptight-paddle-gw.usw2.ara.app/test/`) + * Click **Send**. You'll see the **403 "error: "forbidden response"** again + * In the Headers section in Postman, select **Authorization** from the Key column. Add some random text in the Value field and click **Send**. You should again see the **403 error**. + * Now replace the random text with the `auth` value from your Python code. In our example `47a0c79c427728b3df4af62b9228c8ae` and click **Send** again. + * You should now see the **HTTPB** in test page + +<img src="/img/plugins/postman_success.png" alt="Postman Success" /> + +4. As a further test of your plugin, you can add `get` to your API request in Postman. So in our example `uptight-paddle-gw.usw2.ara.app/test/get`. Click **Send**. This will return all the get requests, including headers. You should see the `x-tyk-request: "something"` which is the post middleware hook you set up in the Python code. + +<img src="/img/plugins/postman_all_get_requests.png" alt="Postman All Get Requests" /> + + +#### Create a Python Code Bundle + +This section demonstrates how to create a Python code bundle as part of the custom authentication process for Tyk Cloud, so that you can ensure your API management solution is as effective as possible. + + +**What do I need to do to create my Plugin?** + +* You need to create the Python code bundle on your locally installed Gateway (not an Tyk Cloud Cloud Data Plane stack). +* You will create 2 files, a manifest file (`manifest.json`) and the python file (`middleware.py`) +* You then create a zipped bundle via our Tyk CLI tool that is built in to your local Gateway instance. + +**Creating the Plugin bundle** + +**Step 1: Create your working directory** + +The first step is to create a directory for your plugin bundle files: + +```.copyWrapper +mkdir ~/my-tyk-plugin +cd ~/my-tyk-plugin +``` + +**Step 2: Creating the Manifest File** + +The manifest file contains information about your plugin file structure and how we expect it to interact with the API that will load it. This file should be named `manifest.json` and needs to have the following contents: + +```.json +{ + "custom_middleware": { + "auth_check": { + "name": "MyAuthMiddleware" + }, + "pre": [ + { + "name": "MyAuthMiddleware" + } + ], + "driver": "python" + }, + "file_list": [ + "middleware.py" + ] +} +``` expandable +**File description** + +| File | Description | +| :------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| custom_middleware | contains the middleware settings like the plugin driver you want to use (driver) and the hooks that your plugin will expose. We use the **auth_check** for this tutorial. For other hooks see [here](/api-management/plugins/rich-plugins#coprocess-dispatcher---hooks). | +| file_list | contains the list of files to be included in the bundle. The CLI tool expects to find these files in the current working directory. | +| name | references the name of the function that you implement in your plugin code: **MyAuthMiddleware** | +| middleware.py | an additional file that contains the main implementation of our middleware. | + +**Step 3: Creating the middleware.py file** + +* You import decorators from the Tyk module that gives us the Hook decorator, and we import [Tyk Python API helpers](/api-management/plugins/rich-plugins#tyk-python-api-methods) + +* You implement a middleware function and register it as a hook. The input includes the request object, the session object, the API meta data and its specification. The hook checks the authorization header for a specified value. In this tutorial we have called it `Authorization`. + +```.python +from tyk.decorators import * +from gateway import TykGateway as tyk + +@Hook +def MyAuthMiddleware(request, session, metadata, spec): + auth = request.get_header('Authorization') + if not auth: + auth = request.object.params.get('authorization', None) + + if auth == '47a0c79c427728b3df4af62b9228c8ae': + session.rate = 1000.0 + session.per = 1.0 + metadata["token"] = auth + return request, session, metadata + +@Hook +def MyPostMiddleware(request, session, spec): + tyk.log("This is my post middleware", "info") + request.object.set_headers["x-tyk-request"] = "something" + return request, session + ``` expandable + +**File description** + +| File | Description | +| :--------------------------- | :-------------------------------------------------------------------------------- | +| `MyAuthMiddleware` @hook | checks for a value. If it is found it is treated as your authentication token. | +| `MyPostMiddleware` @hook | adds a header to the request. In this tutorial `something` | | + +**Step 4: Create the Plugin Bundle** + +* You create a bundle to cater for a number of plugins connected to the one API, and using a bundle makes this more manageable. + +* To bundle your plugin we run the following command in your working directory where your manifest.json and plugin code is. + +```.bash +docker run \ + --rm \ + -v $(pwd):/cloudplugin \ + --entrypoint "/bin/sh" -it \ + -w "/cloudplugin" \ + tykio/tyk-gateway:v3.1.2 \ + -c '/opt/tyk-gateway/tyk bundle build -y' +``` expandable + +* A plugin bundle is a packaged version of the plugin, it may also contain a cryptographic signature of its contents. The -y flag tells the Tyk CLI tool to skip the signing process in order to simplify this tutorial. For more information on the Tyk CLI tool, see [here](/api-management/plugins/overview#how-plugin-bundles-work). +* You should now have a `bundle.zip` file in the plugin working directory. +* Next you will configure [uploading your plugin bundle file](#uploading-your-bundle) to your Amazon S3 bucket. + + +#### Add Custom Authentication + +This section introduces the process of configuring a custom authentication plugin, so that you can override the default Tyk authentication mechanism with your own authentication logic. + +**What are we going to do?** + +We are going to configure an Tyk Cloud Control Plane to use a custom authentication plugin built in Python. + +**What do I need to configure the Tyk Cloud Control Plane?** + +Here are the requirements: + +1. Firstly you will need a local Tyk Gateway installation that allows you to create your Python plugin bundle. We recommend [installing our Self-Managed version on Ubuntu Bionic 18.04](/tyk-self-managed/install#install-tyk-on-debian-or-ubuntu). +2. Ensure you have a currently stable [Python 3.x version](https://www.python.org/downloads/) installed +3. You need install the build tools `apt-get install -y build-essential` +4. Install our required modules: + +```{.copyWrapper} +apt install python3 python3-dev python3-pip +pip3 install protobuf grpcio +``` expandable + + +### Deploy Legacy Hybrid Gateways + + +<Warning> +`tyk-hybrid` chart is deprecated. Please use our [Tyk Data Plane helm chart](#deploy-hybrid-gateways) instead. + +We recommend that all users to migrate to the `tyk-data-plane` Chart. Please review the [Configuration](/product-stack/tyk-charts/tyk-data-plane-chart#configuration) section of the new helm chart and cross-check with your existing configurations while planning for migration. +</Warning> + + +1. **Add the Tyk official Helm repo `tyk-helm` to your local Helm repository** + +```bash +helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ +helm repo update +``` + +The helm charts are also available on [ArtifactHub](https://artifacthub.io/packages/helm/tyk-helm/tyk-hybrid). + +2. **Then create a new namespace that will be hosting the Tyk Gateways** + +```bash +kubectl create namespace tyk +``` + +3. **Get the default values.yaml for configuration** + +Before proceeding with installation of the chart we need to set some custom values. First save the full original values.yaml to a local copy: + +```bash +helm show values tyk-helm/tyk-hybrid > values.yaml +``` expandable + +4. **Configure Tyk Gateway and its connection to Tyk Cloud** + +You need to modify the following values in your custom `values.yaml` file: + +* `gateway.rpc.apiKey` - Tyk Dashboard API Access Credentials of the user created earlier +* `gateway.rpc.rpcKey` - Organization ID +* `gateway.rpc.connString` - MDCB connection string +* `gateway.rpc.group_id`*(optional)* - if you have multiple data plane (e.g. in different regions), specify the data plane group (string) to which the gateway you are deploying belong. The data planes in the same group share one Redis instance. +* `gateway.sharding.enabled` and `gateway.sharding.tags`*(optional)* - you can enable sharding to selectively load APIs to specific gateways, using tags. By default, sharding is disabled and the gateway will load all APIs. + +5. **Configure the connection to Redis** + +You can connect the gateway to any Redis instance already deployed (as DBaaS or hosted in your private infrastructure). + +In case you don't have a Redis instance yet, here's how to deploy Redis in Kubernetes using Bitnami Helm charts. + +```bash +helm install tyk-redis bitnami/redis -n tyk --version 19.0.2 +``` + + +<Note> +Please make sure you are installing Redis versions that are supported by Tyk. Please refer to Tyk docs to get list of [supported versions](/tyk-self-managed#redis). +</Note> + + +Follow the notes from the installation output to get connection details and password. + +```bash + Redis(TM) can be accessed on the following DNS names from within your cluster: + + tyk-redis-master.tyk.svc.cluster.local for read/write operations (port 6379) + tyk-redis-replicas.tyk.svc.cluster.local for read-only operations (port 6379) + + export REDIS_PASSWORD=$(kubectl get secret --namespace tyk tyk-redis -o jsonpath="{.data.redis-password}" | base64 --decode) +``` + +You need to modify the following values in your custom `values.yaml` file: + +* `redis.addrs`: the name of the Redis instance including the port as set by Bitnami `tyk-redis-master.tyk.svc.cluster.local:6379` +* `redis.pass`: password set in redis (`$REDIS_PASSWORD`). Alternatively, you can use --set flag to set it during helm installation. For example `--set redis.pass=$REDIS_PASSWORD`. + + +6. **Install Hybrid data plane** + +Install the chart using the configured custom values file: + +```bash +helm install tyk-hybrid tyk-helm/tyk-hybrid -f values.yaml -n tyk +``` + +You should see the prompt: + +```bash +At this point, Tyk Hybrid is fully installed and should be accessible. +``` + + +7. **Check that the installation was successful** + +The hybrid data planes are not yet visible in Tyk Cloud (coming soon!). Here is how you can check that the deployment was successful. + +Run this command in your terminal to check that all pods in the `tyk` namespace are running: + +```bash +kubectl get pods -n tyk +```` + +**Expected result:** + +```bash +NAME READY STATUS RESTARTS AGE +gateway-tyk-hybrid-54b6c498f6-2xjvx 1/1 Running 0 4m27s +tyk-redis-master-0 1/1 Running 0 47m +tyk-redis-replicas-0 1/1 Running 0 47m +tyk-redis-replicas-1 1/1 Running 0 46m +tyk-redis-replicas-2 1/1 Running 0 46m +``` + +Note: if you are using a Redis instance hosted somewhere else, then no Redis pods will appear here. + +Run this command in your terminal to check that the services were correctly created: + +```bash +kubectl get service -n tyk +```` + +**Expected result:** + +```bash +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +gateway-svc-tyk-hybrid NodePort 10.96.232.123 <none> 443:32668/TCP 44m +tyk-redis-headless ClusterIP None <none> 6379/TCP 47m +tyk-redis-master ClusterIP 10.109.203.244 <none> 6379/TCP 47m +tyk-redis-replicas ClusterIP 10.98.206.202 <none> 6379/TCP 47m +``` + +Note: IP adresses might differ on your system. + + +Finally, from your terminal, send an HTTP call to the /hello endpoint of the gateway `gateway-svc-tyk-hybrid`: + +Note: you may need to port forward if you're testing on a local machine, e.g. `kubectl port-forward service/gateway-svc-tyk-hybrid -n tyk 8080:443` + +```bash +curl http://hostname:8080/hello -i +``` + +**Expected result:** + +```bash +HTTP/1.1 200 OK +Content-Type: application/json +Date: Fri, 17 Mar 2023 10:35:35 GMT +Content-Length: 234 + +{ + "status":"pass", + "version":"4.3.3", + "description":"Tyk GW", + "details":{ + "redis": {"status":"pass","componentType":"datastore","time":"2023-03-15T11:39:10Z"}, + "rpc": {"status":"pass","componentType":"system","time":"2023-03-15T11:39:10Z"}} +} +``` + +### Tyk Cloud Monitor Metrics + +This section explains the various metrics that are monitored by Tyk Cloud. + +#### Tyk Cloud Throughput +Tyk Cloud counts the total request/response sizes for traffic transferred through a deployment. Throughput metrics are displayed for the current day. These are calculated as the difference between the throughput usage at the current time and the throughput at last midnight. + +External traffic is subject to billing, while internal traffic is exempt. The monitoring service aggregates traffic between different services: + +<img src="/img/cloud/tyk-cloud-monitoring-priced-traffic.png" alt="Monitoring Traffic Pricing" /> + +**Billed traffic** + - Traffic between user β†’ Control Plane + - Traffic between user β†’ Cloud Data Plane + - Traffic between user β†’ Enterprise Developer Portal + - Traffic between user β†’ Mserv (plugin upload) + - Traffic between Control Plane β†’ Cloud Data Plane cross region + - Traffic between Cloud Data Plane β†’ Mserv cross region + - Traffic between Control Plane β†’ Portal cross region + +**Unbilled traffic** + - Hybrid traffic is currently not counted + - Traffic between Control Plane β†’ Cloud Data Plane in the same region + - Traffic between Cloud Data Plane β†’ Mserv in the same region + - Traffic between Control Plane β†’ Portal in the same region + +#### Tyk Cloud Storage +When a client makes a request to a Tyk Gateway deployment, the details of the request and response are captured and [stored in Redis](/api-management/dashboard-configuration#traffic-analytics). Tyk Pump processes the records from Redis and forwards them to MongoDB. Finally, Tyk Cloud reads that data from MongoDB and displays its size(bytes) in the _Storage_ section of _Monitoring_. + + + +### Tyk Cloud Telemetry + +Telemetry in Tyk Cloud enables distributed tracing of your APIs, allowing you to track and analyze how requests flow through your system. +This trace data helps you understand request paths, identify bottlenecks, and troubleshoot issues by providing detailed insights into each request's journey through your API infrastructure. + +We support distributed tracing for `Cloud Data Plane` deployments. You can enable it while creating or updating after setting up telemetry. + +Since this functionality is powered by Tyk Gateway's OpenTelemetry integration, we recommend reviewing our comprehensive [OpenTelemetry documentation](/api-management/logs-metrics#opentelemetry) +to understand the underlying architecture, best practices for implementation, and how to maximize the value of distributed tracing in your API infrastructure. The documentation provides detailed insights into configuration options, sampling strategies. + +#### Available Telemetry Providers + +Tyk Cloud integrates with these monitoring platforms: + +- [Datadog](#configure-providers) +- [Dynatrace](#configure-providers) +- [New Relic](#configure-providers) +- [Elastic](#configure-providers) +- [Custom](#configure-providers) + + + + + <Note> + Before diving into the configuration details, please note that Telemetry is an add-on feature in Tyk Cloud. + To enable this capability for your account, please contact our [support team](https://support.tyk.io/). + Our team will help you activate this feature and ensure you have access to all the Telemetry options. + </Note> + + +#### Enabling Telemetry in Tyk Cloud + +Configuring telemetry in Tyk cloud is a two step process. +1. Configure a provider/backend at organization level. +2. Enable/Disable telemetry option while creating/updating a `Cloud Data Plane`. + + + + <Note> + Before diving into the configuration details, please note that Telemetry is an add-on feature in Tyk Cloud. + To enable this capability for your account, please contact our [support team](https://support.tyk.io/). + Our team will help you activate this feature and ensure you have access to all the Telemetry options. + </Note> + + +##### Steps for Configuring Telemetry Provider in Tyk Cloud + +1. **Choosing Your Provider** + + In the `Tyk Cloud Console`, select `Telemetry` option. You'll see a grid displaying all supported backends/providers. Click on your preferred backend/provider to begin the configuration process. + + + + <Note> + Only a single provider/backend can be configured at any given time. + </Note> + + + <img src="/img/cloud/telemetry-exports.png" alt="Tyk Cloud Telemetry providers" /> + +2. **Configuring Basic Elements** + + Every telemetry backend shares these fundamental settings: + + 1. **Connection Toggle:** This switch activates or deactivates your telemetry export. When enabled, Tyk will start sending monitoring data to your chosen platform. + + 2. **Sampling Rate:** This setting determines what percentage of your API traffic data gets sent to your monitoring platform. The default is 10%, which means Tyk will send data for one out of every ten API calls. + +3. **Configuring Optional Settings** + + Beyond the basic settings, you can customize your telemetry with two optional features: + + 1. **Tags to Add to the Traces :** + + Add custom labels to your telemetry data to make it easier to analyze. For example: + ``` + environment:production + region:europe + team:api-gateway + ``` + + 2. **Fields to Filter :** + + Specify which data fields should be excluded from your telemetry. This is useful for ensuring sensitive information doesn't get sent to your monitoring platform. + +4. **Configuring Provider-Specific Configuration** + + Each monitoring platform has its own requirements for connection. Let's explore what you'll need for each: + + <a id="configure-providers"></a> + + <Tabs> + + <Tab title="Datadog"> + + - **Provider Site:** Enter your Datadog URL (like us5.datadoghq.com). To obtain your URL, refer to the following [docs](https://docs.datadoghq.com/getting_started/site/#access-the-datadog-site). + - **API Key:** Enter your Datadog API key. To obtain your API Key, refer to the following [docs](https://docs.datadoghq.com/account_management/api-app-keys/#add-an-api-key-or-client-token). + + Example: A Datadog setup might look like: + ``` + Provider Site: us5.datadoghq.com + API Key: your-datadog-api-key + ``` + + <img src="/img/cloud/telemetry-datadog.png" alt="Tyk Cloud Telemetry Datadog" /> + + </Tab> + + <Tab title="Dynatrace"> + + - **Provider Endpoint:** Enter Your Dynatrace environment URL. To obtain your URL, refer to the following [docs](https://docs.dynatrace.com/docs/discover-dynatrace/get-started/monitoring-environment#environment-id). + - **API Token:** Enter your Dynatrace access token. To obtain your Access Token, refer to the following [docs](https://docs.dynatrace.com/docs/manage/identity-access-management/access-tokens-and-oauth-clients/access-tokens#create-api-token). + + Example configuration: + ``` + Provider Endpoint: https://<YOUR-ENVIRONMENT-STRING>.live.dynatrace.com/api/v2/otlp + API Token: your-dynatrace-token + ``` + + <img src="/img/cloud/telemetry-dynatrace.png" alt="Tyk Cloud Telemetry Dynatrace" /> + + </Tab> + + <Tab title="New Relic"> + + - **Provider Endpoint:** Your New Relic HTTP endpoint. To obtain your endpoint, refer to the following [docs](https://docs.newrelic.com/docs/opentelemetry/best-practices/opentelemetry-otlp/#configure-endpoint-port-protocol). + - **API Token:** Your New Relic license key. To obtain your license key, refer to the following [docs](https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/). + + Example configuration: + ``` + Provider Endpoint: https://security-api.newrelic.com/security/v1 + API Token: your-newrelic-api-key + ``` + + <img src="/img/cloud/telemetry-newrelic.png" alt="Tyk Cloud Telemetry NewRelic" /> + + </Tab> + + <Tab title="Elastic"> + + - **Provider Endpoint:** Your Elastic APM server address. + - **Secret Token:** Your Elastic APM authentication token. To obtain your token, refer to the following [docs](https://www.elastic.co/guide/en/observability/current/apm-secret-token.html#apm-create-secret-token). + + Example setup: + ``` + Provider Endpoint: https://your-elastic-cluster:8200 + Secret Token: your-elastic-secret-token + ``` + + <img src="/img/cloud/telemetry-elastic.png" alt="Tyk Cloud Telemetry Elastic" /> + + </Tab> + + <Tab title="Custom"> + + For when you need to connect to a different monitoring system: + + - **Exporter:** Choose gRPC/HTTP + - **Provider Endpoint:** Your monitoring system URL + - **Authorization:** Configure how Tyk should authenticate with your system + + Example custom configuration: + ``` + Exporter: gRPC/HTTP + Provider Endpoint: grpc://your-collector:4317 + Authorization Header Name: Authorization + Authorization Header Value: Bearer your-token + ``` + + <img src="/img/cloud/telemetry-custom.png" alt="Tyk Cloud Telemetry Custom" /> + + </Tab> + + </Tabs> + +--- + +##### Configure Telemetry Export in Cloud Data Plane Deployments + +When creating a new Cloud Data Plane deployment or editing an existing one, you can configure telemetry export settings. These settings are specific to Cloud Data Plane deployments only and allow you to monitor API performance through your chosen telemetry provider. + +When you modify any general telemetry settings in Tyk Cloud, these changes don't take immediate effect. +Your Cloud Data Planes need to be redeployed to activate the new telemetry configuration. + +<br /> + +<h6 id="configuration-options">Configuration Options</h6> +<img src="/img/cloud/telemetry-enable.png" alt="Tyk Cloud Telemetry Enable" /> + +1. **Enable Datadog Connection** + - Toggle switch to enable/disable Datadog monitoring for this specific Cloud Data Plane deployment + +2. **Sampling Rate Override** + - Choose what percentage of API traffic to monitor (default: 10%) + + + + <Note> + The sampling level can be configured at both the organization level (while setting up the provider) and the `Cloud Data Plane`. The configuration at the Cloud Data Plane will override the organization-level settings. + </Note> + + +2. **Verifying Cloud Data Plane Configuration** + + As shown in the below image, you should observe `Telemetry Export` to be enabled: + + <img src="/img/cloud/tyk-cloud-data-plane-enable-telemetry.png" alt="Tyk Cloud Data Plane Telemetry Enabled" /> + + +### Tyk Dashboard Audit Logs + +Tyk Cloud provides comprehensive audit logging capabilities to track and monitor all administrative actions performed within your Tyk Dashboard. This feature is essential for compliance and security. + +#### What are Audit Logs? + +Audit logs capture detailed records of all requests made to endpoints under the `/api` route in your Tyk Dashboard. These logs include information about: + +- User actions and administrative operations +- API changes and configurations +- Authentication and authorisation events +- System access and modifications +- Response status codes and timestamps + +#### Enabling Audit Logs for Control Plane Deployments + + +<Note> +The audit log feature is available for Control Plane versions v5.7.0 or later. +</Note> + + +##### How to Enable Audit Logging + +1. **Contact Your Account Manager**: Audit logging must be enabled at the subscription level. Reach out to your Tyk account manager to add this feature to your plan. + +2. **Enable via Tyk Cloud UI**: Once the feature is available in your subscription, you can enable audit logging directly from the Tyk Cloud console: + - Navigate to your Control Plane deployment + - Select **Edit** from the deployment options + - Enable the **Audit Logging** option + - Save and redeploy your Control Plane + +Audit logs will be stored in your Control Plane's database for easy access and management. + +##### Viewing and Accessing Audit Logs + +Once audit logging is enabled, you can retrieve the logs via the Tyk Dashboard API. + +For details on the API endpoints and usage, see [Retrieving Audit Logs via API](/api-management/dashboard-configuration#retrieving-audit-logs-via-api). + +#### Storage Size Caps + +Tyk Cloud enforces audit log storage size caps based on your subscription terms: + +- **Storage Limits**: A size cap is applied to audit logs based on your subscription plan +- **Automatic Cleanup**: When the storage limit is reached, the oldest logs are automatically removed to make space for new entries. + +### Tyk Cloud MDCB Supported versions + +This section lists the supported MDCB version for hybrid setup + +| Dashboard | Gateway | MDCB | +| :--------- | :------- | :------ | +| v5.2.0 | v5.2.0 | v2.4.0 | +| v5.1.2 | v5.1.2 | v2.3.0 | +| v5.1.1 | v5.1.1 | v2.3.0 | +| v5.1.0 | v5.1.0 | v2.3.0 | +| v5.0.5 | v5.0.5 | v2.2.0 | +| v5.0.4 | v5.0.4 | v2.2.0 | +| v5.0.3 | v5.0.3 | v2.2.0 | +| v5.0.2 | v5.0.2 | v2.2.0 | +| v5.0.1 | v5.0.1 | v2.1.1 | +| v5.0.0 | v5.0.0 | v2.1.1 | +| v4.3.3 | v4.3.3 | v2.1.0 | +| v4.3.2 | v4.3.2 | v2.0.4 | +| v4.3.1 | v4.3.1 | v2.0.4 | +| v4.3.0 | v4.3.0 | v2.0.4 | +| v4.2.4 | v4.2.4 | v2.0.3 | +| v4.2.3 | v4.2.3 | v2.0.3 | +| v4.0.10 | v4.0.10 | v2.0.4 | +| v4.0.9 | v4.0.9 | v2.0.3 | +| v4.0.8 | v4.0.8 | v2.0.3 | +| v3.2.3 | v3.2.3 | v1.8.1 | +| v3.0.9 | v3.0.9 | v1.7.11 | + +### Track Usage + +##### How to check metrics +Login to Tyk Cloud and click on *Monitoring* within the *Operations* menu. Enable *Throughput* to display throughput metrics. + +<img src="/img/cloud/tyk-cloud-monitoring-throughput.png" alt="Monitoring Throughput" /> + +Enable *Storage* to display storage metrics. + +<img src="/img/cloud/tyk-cloud-monitoring-storage.png" alt="Monitoring Storage" /> + +You can also optionally filter for metrics by date. + +<img src="/img/cloud/tyk-cloud-monitoring-filtering-by-date.png" alt="Monitoring Metric Filtering" /> + +Here you can see the metrics broken down per environment and a list of the top 5 control and cloud data planes. + +<img src="/img/cloud/tyk-cloud-monitoring-break-down.png" alt="Monitoring Metric break down" /> + +### Troubleshooting Tyk Cloud + +#### FAQs + +**Q1: Is a Cloud Data Plane Deployment considered highly available? Do I need to deploy multiple Cloud Data Planes to a single Data Center?** + +A: On a Production plan and higher there are at least two Gateway instances at all times, usually in different +availability zones within a single region (in cloud provider terms). + +**Q2: What are the performance benchmarks of a single Cloud Data Plane?** + +A: In Phase 2 we plan to allow users to choose from a pool of "runtimes" that provide different performance targets, so +they'll be able to have a Tyk Cloud environment with Cloud Data Planes that can sustain more load and then another environment +(e.g. for testing) that sustains less. + +**Q3: How can I geo-load balance across multiple Cloud Data Planes? Why should I want to?** + +A: The use case to deploy multiple Cloud Data Planes is either segregating regional traffic and/or segregating APIs. +This doesn't necessarily concern High Availability. + +The number of actual Gateway instances within a single Cloud Data Plane deployment varies, auto-scales and load balances depending +on the plan. + +If you deploy several Cloud Data Planes and want to use it to e.g. geo-load balance it's currently your responsibility to put such +a system into place, but we have plans to help with this in later phases. + +**Q4: What instance sizes/VMs does a Gateway run on?** + +A: You won't need to worry. We abstract all the resources and only provide performance "targets". See Q2. + +**Q5: Can I connect my own Hybrid Gateways?** + +A: Yes. The MDCB Ingress URL is given in the Control Plane details page, which allows for connecting a Hybrid Gateway. + +**Q6: Can we use SSO in the Dashboard and/or Portal?** + +A: Yes, as of v3.0.0, TIB is integrated into Tyk Dashboard, meaning that once a Control Plane is deployed, a user can +go into the Identity Management section of Tyk Dashboard and setup SSO with their IdP for both the Dashboard and +Developer Portal. + +**Q7: How do I view Gateway/Dashboard logs for troubleshooting?** + +A: This will be exposed in later phases per deployment. + +**Q8: How do Segment tags work with Tyk Cloud?** + +A: When a Cloud Data Plane is deployed, the tag 'edge' and a location tag are automatically generated for the Cloud Data Plane. You use these tags to connect your API to the appropriate Cloud Data Plane. It works as follows: + +* Add the **edge** tag to your API to connect it to all Cloud Data Planes within the Control Plane. +* Add the location tag to your API to connect it to only Cloud Data Planes with that location within the Control Plane. + +To add either of the tags, see [Adding an API](#steps-to-add-an-api-in-tyk-cloud) in the Getting Started section. + + +<Warning> +You must add one of the above tags to any API you add to your Control Plane Dashboard. +</Warning> + + +#### Glossary + +**Account** + +The highest level container for one or more Organizations. + +**Organization** + +The main entity for all your data (Environments, APIs, Users, etc). An Organization is connected to a single region and once connected, cannot be changed. + +**Team** + +A sub-grouping within an Organization. + +**User** + +A person who is a member of a Team with a set of permissions. + +**Role** + +A set of data and access permissions that can be applied to a user or team of users. See [User Roles](#user-roles-in-tyk-cloud) for more details. + +**Profile** + +The place that holds personal information for a user. + +**Subscription** + +A set of allowances assigned to an Organization (made up of plan+addons+settings). + +**Plan** + +A portion of allowances (without add-ons) that feed into the main subscription. + +**Operations** + +The place to manage all deployments for an Organization or Team. + +**Environment** + +A grouping of 'deployments' that can have multiple Control Planes and Cloud Data Planes. + +**Stack** + +The high level name for the set of configurations making up different types of deployments. + +**Control Plane** + +A deployment type: A single management layer for data in one region (where all the data lives). + +**Cloud Data Plane** + +A deployment type: Additional workers with varying functionality that are linked to the main control plane and can be deployed in a different region from the Control Plane. + +**Instance** + +Used to control traffic and scale in a Tyk Gateway. + +**Dashboard** + +The Tyk Analytics Dashboard to manage APIs and services. + +**Retirement** + +Where an Organization has expired due to either a subscription failure or cancelation and is now within a "retirement" period of 30 days, during which an [Billing Admin](#assign-user-roles) can reinstate full functionality by updating or creating a new subscription. + +**Deploy** + +Deploy a not yet deployed state (a first time deployment). + +**Undeploy** + +Temporarily remove a deployed status but keep all data and configuration. + +**Redeploy** + +Redeploy from a deployed state. Used for troubleshooting. + +**Destroy** + +Permenantly remove a deployment and all associated data and configurations. + +**Create** + +Date and time of time a deployment was initially created. + +**Add** + +Add a new 'user' or 'team' etc. + +**Remove** + +Remove things that have been added e.g. users and teams. + +**Update** + +Saving a change to a configuration. + +**Edit** + +Changing configuration or information on any of the deployments or other resources such as users or teams. + +**Deployed** + +A deployment that is currently deployed. + +**Undeployed** + +A deployment that was deployed but has been undeployed. + +**Not Deployed** + +A deployment that has never been deployed. + +**Destroyed** + +A deployment that has been permenantly deleted and will not be visible in the operations console. + +**Unsuccessful** + +When there has been an error in the deployment process. + +**Deploying** + +When a deployment is being deployed. + +**Undeploying** + +When a deployment is being undeployed. + +See [User Roles](#user-roles-in-tyk-cloud) for more details + +**Super Administrator** + +Can do everything across all organizations + +**Organization Admin** + +Can do everything within the scope of the one organization they have access to. + +**Team Admin** + +Can do everything within the scope of the one team they have access to. + +**Team Member** + +Can only view and manage the overview, environments and deployments. diff --git a/tyk-components.mdx b/tyk-components.mdx new file mode 100644 index 00000000..ab856553 --- /dev/null +++ b/tyk-components.mdx @@ -0,0 +1,242 @@ +--- +title: "Tyk Components" +'og:description': "Explaining Tyk Components" +tags: ['Tyk API Management', 'Getting Started', 'Tutorials'] +order: 5 +sidebarTitle: "Tyk Components" +--- + +import { ResponsiveGrid } from '/snippets/ResponsiveGrid.mdx'; + +# Understanding Tyk’s API Management Components + +Tyk offers a full ecosystem of API management tools, supporting every stage of the API lifecycleβ€”from development and deployment to monitoring and scaling. With Tyk’s components, teams can seamlessly integrate, secure, and scale APIs across different environments, facilitating robust and reliable API operations. + +--- + +## Tyk’s API Management Components Overview + +Below is a detailed introduction to each of Tyk’s components and their individual roles within an API management strategy. For more information, follow the links provided to explore specific documentation on each component. + + +<ResponsiveGrid> + + +<Card title="Tyk Gateway" href="/tyk-components#tyk-gateway"> +The open-source core of Tyk that handles request routing, rate-limiting, and caching. +</Card> + +<Card title="Tyk Dashboard" href="/tyk-components#tyk-dashboard"> +A powerful interface for managing and monitoring APIs, users, and permissions. +</Card> + +<Card title="Tyk Enterprise Developer Portal" href="/tyk-components#tyk-enterprise-developer-portal"> +A customizable portal for enabling API discovery and user self-service. +</Card> + +<Card title="Tyk Multi Data Centre Bridge" href="/tyk-components#tyk-multi-data-centre-bridge-mdcb"> +Enables centralized API management across distributed data centers for global reach and compliance. +</Card> + +<Card title="Tyk Pump" href="/tyk-components#tyk-pump"> +Aggregates and exports API analytics from Gateway to enable insights and advanced monitoring. +</Card> + +<Card title="Tyk Operator" href="/tyk-components#tyk-operator"> +Integrates with Kubernetes for seamless API management within containerized environments. +</Card> + +<Card title="Tyk Streams" href="/tyk-components#tyk-streams"> +Enables real-time event streaming for APIs, ideal for push notifications and live data. +</Card> + +<Card title="Tyk Sync" href="/tyk-components#tyk-sync"> +Synchronizes API configurations across environments for consistency and streamlined deployments. +</Card> + +<Card title="Tyk Identity Broker" href="/tyk-components#tyk-identity-broker"> +Facilitates single sign-on (SSO) and integrates external identity providers with APIs. +</Card> + +<Card title="Tyk Helm Charts" href="/tyk-components#tyk-helm-charts"> +Provides templated configurations to simplify deploying Tyk in Kubernetes. +</Card> + +<Card title="Universal Data Graph" href="/tyk-components#universal-data-graph-udg"> +Combines multiple data sources into a unified API, supporting GraphQL and REST. +</Card> + + +</ResponsiveGrid> + +--- + +### Tyk Gateway + +The **[Tyk Gateway](/tyk-oss-gateway)** is the core of Tyk’s API management platform. This open-source, high-performance API gateway handles crucial API tasks: + +- **Traffic Management**: Routes requests to backend services, balancing loads and ensuring high availability. +- **Security and Authentication**: Supports OAuth2, JWT, OpenID Connect, and other protocols to secure APIs. +- **Rate Limiting and Quotas**: Controls traffic volume to prevent abuse and ensure fair usage. +- **Analytics**: Tracks API usage patterns and provides insights into performance and engagement. + +The Gateway is the primary component managing and securing all API interactions, making it an essential part of any API management strategy. + +--- + +### Tyk Dashboard + +The **[Tyk Dashboard](/tyk-dashboard)** is a GUI interface for managing, monitoring, and configuring APIs. It integrates with the Tyk Gateway and offers centralized control for all API operations: + +- **Design APIs**: Set up endpoints, security policies, and access controls. +- **Monitor API Usage**: View real-time analytics to track API performance and identify trends. +- **User and Team Management**: Configure multi-tenancy and manage team-based access control. + +The Tyk Dashboard simplifies API management, enabling teams to effectively administer API policies and monitor activity. + +--- + +### Tyk Enterprise Developer Portal + +The **[Tyk Developer Portal](/tyk-developer-portal)** provides a self-service interface for developers to access and interact with APIs. Key functionalities include: + +- **API Documentation**: Supplies developers with detailed API documentation, making integration easier. +- **Self-Service Access**: Developers can register, request API keys, and manage subscriptions independently. +- **Community Engagement**: Features like FAQs and forums foster a collaborative developer community. + +The Developer Portal streamlines API adoption by offering developers a comprehensive platform for API discovery and self-service. + +--- + +### Tyk Multi Data Centre Bridge (MDCB) + +The **[Tyk Multi Data Centre Bridge (MDCB)](/tyk-multi-data-centre)** enables centralized control over API configurations across distributed regions or data centers. It’s ideal for global API infrastructure, offering: + +- **Centralized Management**: Controls API configurations across different regions from a single interface. +- **Data Isolation**: Supports regulatory compliance by keeping data within specified regions. +- **Enhanced Resilience**: Ensures high availability for distributed API deployments. + +MDCB is designed for organizations with extensive geographic reach, supporting both global distribution and local compliance. + +--- + +### Tyk Pump + +**[Tyk Pump](/tyk-pump)** is a lightweight tool that aggregates API analytics from the Tyk Gateway and exports them to various storage systems for analysis: + +- **Data Collection**: Continuously gathers real-time API analytics. +- **Data Export**: Integrates with multiple backends like Splunk, Prometheus, ElasticSearch, and others. +- **Actionable Insights**: Enables advanced monitoring and analysis for data-driven decision-making. + +With Tyk Pump, you can leverage powerful insights into API usage, helping teams optimize API performance and improve user experience. + +--- + +### Tyk Operator + +The **[Tyk Operator](/api-management/automations/operator#what-is-tyk-operator)** is a Kubernetes-native API management solution, allowing teams to deploy and manage APIs as Kubernetes resources: + +- **Kubernetes Integration**: Simplifies API management within Kubernetes environments. +- **Declarative Configuration**: Supports automated, scalable, and resilient API deployments. +- **CI/CD Integration**: Ideal for development teams with continuous integration and deployment workflows. + +For teams operating within Kubernetes, Tyk Operator integrates seamlessly, providing a consistent way to manage API resources. + +--- + +### Tyk Streams + +**[Tyk Streams](/api-management/event-driven-apis#)** is Tyk’s real-time data streaming tool for APIs, enabling applications to receive data as events happen: + +- **Real-Time Data**: Pushes live data to clients in real time. +- **Event-Driven Architecture**: Triggers immediate responses to data changes. +- **Scalability**: Manages high-frequency data flows, scaling to meet demand. + +Tyk Streams is ideal for use cases like financial services, IoT, and live applications requiring rapid data updates. + +--- + +### Tyk Sync + +**[Tyk Sync](/api-management/automations/sync)** facilitates configuration synchronization, helping teams manage API configurations across different environments: + +- **Configuration Consistency**: Synchronizes API settings across development, staging, and production. +- **DevOps Compatibility**: Integrates with CI/CD pipelines for seamless deployment. +- **Backup and Restore**: Maintains backups of API configurations to prevent data loss. + +Tyk Sync ensures that API configurations are consistent and reliable across environments, supporting streamlined operations and effective change management. + +--- + +### Tyk Identity Broker + +The **[Tyk Identity Broker](/api-management/external-service-integration#what-is-tyk-identity-broker-tib)** simplifies authentication by connecting APIs with external identity providers (IDPs), supporting single sign-on (SSO) capabilities: + +- **Authentication Integration**: Works with IDPs like Google, Microsoft, and LDAP. +- **Secure Access Control**: Manages API access for verified users. +- **Token-Based Access**: Issues tokens for authenticated users, enabling controlled API access. + +The Identity Broker centralizes identity and access management, making it easier to secure APIs and manage user authentication across multiple systems. + +--- + +### Tyk Helm Charts + +**[Tyk Helm Charts](/product-stack/tyk-charts/overview)** provide templated configurations for deploying Tyk components in Kubernetes, enabling efficient setup and management: + +- **Efficient Deployment**: Simplifies installation and configuration within Kubernetes clusters. +- **Version Control**: Manages versioning and consistency across deployments. +- **Standardization**: Ensures uniform setups across environments, improving reliability. + +Tyk Helm Charts are particularly useful for teams deploying Tyk in Kubernetes, streamlining installation and ensuring consistent configurations. + +--- + +### Universal Data Graph (UDG) + +The **[Universal Data Graph (UDG)](/api-management/data-graph#overview)** provides a single API endpoint for accessing data from multiple sources using GraphQL, offering: + +- **Data Aggregation**: Combines data from disparate sources into a unified API. +- **Customizable Data Access**: Clients can query only the data they need. +- **Secure Data Management**: Implements access controls to manage data permissions. + +The UDG simplifies complex data interactions, enabling developers to work with a unified GraphQL endpoint for various backend services. + +--- + +## How Tyk’s Components Work Together + +Tyk’s components create a unified API management ecosystem, covering all aspects of API lifecycle management: + +- **Traffic and Security**: The Tyk Gateway acts as the first line of defense, handling traffic and enforcing security protocols. +- **Configuration and Deployment**: Tyk Dashboard, Tyk Operator, and Tyk Sync provide tools for managing, deploying, and scaling APIs. +- **Developer Experience**: The Tyk Developer Portal and Identity Broker enhances the developer experience, promoting API discovery and user engagement. +- **Global Reach and Resilience**: MDCB ensures consistent API configurations and high availability across regions. +- **Data and Insights**: Tyk Gateway's native support for OpenTelemetry, combined with Tyk Pump, enables real-time monitoring and analytics. +- **Unified Access**: UDG and Tyk Streams unify API Management by enabling seamless, multi-protocol data integration and delivery. +- **Kubernetes Deployment**: Tyk Helm Charts simplifies deployment of Tyk on Kubernetes. + +# Conclusion + +Now that you’ve been introduced to the Tyk suite, you have a strong foundation in understanding how each component fits into a complete API management ecosystem. Whether you’re ready to start deploying APIs, securing them, or scaling across multiple environments, Tyk provides the tools and flexibility to bring your API projects to life. + +## Where to Go Next + +1. **[Getting Started with Tyk Gateway](/tyk-oss-gateway)** + Start by setting up the Tyk Gateway, where you’ll configure your first API and explore the foundational capabilities of traffic management, security, and monitoring. + +2. **[Set Up and Configure the Tyk Dashboard](/tyk-dashboard)** + Dive into the Tyk Dashboard to manage your API lifecycle from a user-friendly interface, allowing you to monitor, configure, and scale your APIs with ease. + +3. **[Explore API Security](/api-management/client-authentication)** + Secure your APIs with Tyk’s robust authentication options like OAuth2, JWT, and HMAC, and learn how to apply rate limiting and quota policies to protect your resources. + +4. **[Implement Multi-Region Deployments with MDCB](/tyk-multi-data-centre)** + If your infrastructure requires high availability and global reach, explore Tyk’s Multi Data Centre Bridge to deploy and manage APIs across different regions. + +5. **[Use Tyk Sync for Consistent Configuration Management](/api-management/automations/sync)** + For development teams working across environments, Tyk Sync offers a way to manage API configurations consistently, supporting CI/CD workflows and minimizing deployment errors. + +6. **[Explore Tyk Developer Portal for Enhanced Developer Experience](/tyk-developer-portal)** + Provide external and internal developers with easy access to your APIs by setting up the Developer Portal, where they can find documentation, request access, and get started quickly. + diff --git a/tyk-configuration-reference/kv-store.mdx b/tyk-configuration-reference/kv-store.mdx new file mode 100644 index 00000000..312eaada --- /dev/null +++ b/tyk-configuration-reference/kv-store.mdx @@ -0,0 +1,451 @@ +--- +title: "Using external Key Value storage with Tyk" +'og:description': "Learn how to use external Key-Value storage with Tyk Gateway, including Consul and Vault, to manage configuration data and secrets." +sidebarTitle: "Key Value Store" +--- + +## Introduction + +With Tyk Gateway you can store configuration data (typically authentication secrets or upstream server locations) in Key-Value (KV) systems such as [Vault](#vault), and [Consul](#consul) and then reference these values during configuration of the Tyk Gateway or APIs deployed on the Gateway. + +## When to use external Key-Value storage + +### Simplify migration of APIs between environments + +Easily manage and update secrets and other configuration across multiple environments (e.g., development, staging, production) without modifying the configuration files. + +### Ensure separation of concerns + +Securely store sensitive information like API keys, passwords, and certificates in a centralised location. Not everybody needs access to these secrets: authorized people can maintain them and just pass along the reference used to access them from the KV store. + +### Support per-machine variables + +Storing local settings within the Tyk Gateway's configuration file allows you to have per instance variables, such as a machine ID, and inject these into API requests and responses using [transformation middleware](/api-management/traffic-transformation). + +## How external Key-Value storage works + +There are two parts to external Key-Value storage - the KV store and the Tyk configuration object (API definition or Tyk Gateway config file). + +1. The key-value data that you wish to reference should be added to the storage +2. References should be included within the Tyk configuration object that identify the location (KV store) and Key +3. When Tyk Gateway initialises it will resolve any external KV references in its configuration, retrieving and applying the values from those references +4. When Tyk Gateway loads (or reloads) APIs it will resolve any external KV references in the API definitions, retrieving and applying the values from those references + +Most Key-Value references are only retrieved when the configuration object (Gateway or API) is loaded, as explained above: changes to the externally stored value will not be detected until a subsequent gateway hot-reload. + +The exception to this is for specific [transformation middleware](#transformation-middleware) where the value will be retrieved for each call to the API, during the processing of the API request or response. + + +<Note> +If Tyk Gateway cannot communicate with the KV store, it will log an error and will treat the referenced key value as empty, continuing to load the Gateway or API, or to process the transformation middleware as appropriate. +</Note> + + +## Supported storage options + +Tyk Gateway supports the following locations for storage of key-value data, providing flexibility to choose the most suitable approach for your deployment and the data you are storing: + +### Consul + +HashiCorp [Consul](https://www.consul.io) is a service networking solution that is used to connect and configure applications across dynamic, distributed infrastructure. Consul KV is a simple Key-Value store provided as a core feature of Consul that can be used to store and retrieve Tyk Gateway configuration across multiple data centers. +- to retrieve the value assigned to a `KEY` in Consul you will use `consul://KEY` or `$secret_consul.KEY` notation depending on the [location](#how-to-access-the-externally-stored-data) of the reference + +### Vault + +[Vault](https://vaultproject.io) from Hashicorp is a tool for securely accessing secrets. It provides a unified interface to any secret while providing tight access control and recording a detailed audit log. Tyk Gateway can use Vault to manage and retrieve sensitive secrets such as API keys and passwords. +- to retrieve the value assigned to a `KEY` in Vault you will use `vault://KEY` or `$secret_vault.KEY` notation depending on the [location](#how-to-access-the-externally-stored-data) of the reference + +**Tyk Gateway config file** + +The `secrets` section in the [Tyk Gateway configuration file](/tyk-oss-gateway/configuration#secrets) allows you to store settings that are specific to a single Tyk Gateway instance. This is useful for storing instance-specific configuration to be injected into API middleware or if you prefer using configuration files. +- to retrieve the value assigned to a `KEY` in the `secrets` config you will use `secrets://KEY` or `$secret_conf.KEY` notation depending on the [location](/tyk-oss-gateway/configuration#secrets) of the reference + +**Environment variables** + +Tyk Gateway can access data declared in environment variables. This is a simple and straightforward way to manage secrets, especially in containerised environments like Docker or Kubernetes. +- if you want to set the local "secrets" section (equivalent to the `secrets` section in the [Gateway config file](#how-to-access-the-externally-stored-data)) using environment variables, you should use the following notation: `TYK_GW_SECRETS=key:value,key2:value2` +- if you’re using a different key value secret store not explicitly supported by Tyk but can map it to `TYK_GW_SECRETS` +then this will allow you to access those KV data +- to retrieve the value assigned to an environment variable `VAR_NAME` you will use `env://VAR_NAME` or `$secret_env.VAR_NAME` notation depending on the [location](#how-to-access-the-externally-stored-data) of the reference + +## How to access the externally stored data + +You can configure Tyk Gateway to retrieve values from KV stores in the following places: +- Tyk Gateway configuration file (`tyk.conf`) +- API definitions + + + + <Note> + You can use keys from different KV stores (e.g. Consul and environment variables) in the same configuration object (Gateway config or API definition). + </Note> + + +### From the Tyk Gateway configuration file + +In Tyk Gateway's configuration file (`tyk.conf`), you can retrieve values from KV stores for the following [fields](/tyk-oss-gateway/configuration): +- `secret` +- `node_secret` +- `storage.password` +- `cache_storage.password` +- `security.private_certificate_encoding_secret` +- `db_app_conf_options.connection_string` +- `policies.policy_connection_string` +- `slave_options.api_key` + +To reference the *Value* assigned to a *Key* in one of the KV stores from the Gateway configuration file use the following notation: +- Consul: `consul://path/to/key` +- Vault: `vault://path/to/secret.key` +- `tyk.conf` secrets: `secrets://key` +- Environment variables: `env://key` + +For example, if you create a Key-Value pair in [Vault](#vault) with the *key* `shared-secret` in *secret* `gateway-dashboard` within directory `tyk-secrets/` then you could use the *Value* as the `node_secret` in your Gateway config by including the following in your `tyk.conf` file: +``` .json +{ + "node_secret":"vault://tyk-secrets/gateway-dashboard.shared-secret" +} +``` +When the Gateway starts, Tyk will read the *Value* from Vault and use this as the `node_secret`, which is used to [secure connection to the Tyk Dashboard](/tyk-oss-gateway/configuration#node_secret). + +Note that all of these references are read (and replaced with the values read from the KV location) on Gateway start when loading the `tyk.conf` file. + +### From API Definitions + +From Tyk Gateway v5.3.0 onwards, you can store [any string field](#other-string-fields) from the API definition in any of the supported KV storage options; for earlier versions of Tyk Gateway only the [Target URL and Listen Path](#target-url-and-listen-path) fields and [certain transformation middleware](#transformation-middleware) configurations were supported. + +#### Target URL and Listen Path + +To reference the *Value* assigned to a *Key* in one of the KV stores for **Target URL** or **Listen Path** use the following notation: +- Consul: `consul://path/to/key` +- Vault: `vault://path/to/secret.key` +- Tyk config secrets: `secrets://key` +- Environment variables: `env://key` + +These references are read (and replaced with the values read from the KV location) when the API is loaded to the Gateway (either when Gateway restarts or when there is a hot-reload). + +For example, if you define an environment variable (*Key*) `UPSTREAM_SERVER_URL` with the *Value* `http://httpbin.org/` then within your API definition you could use the *Value* for the Target URL for your Tyk OAS API as follows: +```json expandable +{ + "x-tyk-api-gateway": { + "upstream": { + "url": "env://UPSTREAM_SERVER_URL" + } + } +} +``` + +When the Gateway starts, Tyk will read the *Value* from the environment variable and use this as the [Target URL](/api-management/gateway-config-tyk-oas#upstream). + + +<Note> +Prior to Tyk Gateway v5.3.0, **environment variables** used for the Target URL or Listen Path had to be named `TYK_SECRET_{KEY_NAME}`. They were referred to in the API definition using `env://{KEY_NAME}` excluding the `TYK_SECRET_` prefix. +<br /><br /> +From v5.3.0 onward, environment variables can have any `KEY_NAME`, and the full name should be provided in the API definition reference. The pre-v5.3.0 naming convention is still supported for backward compatibility, but only for these two fields. +</Note> + + +#### Transformation middleware + +Key-value references can be included in the following middleware, with the values retrieved dynamically when the middleware is called (during processing of an API request or response): +- [request body transform](/api-management/traffic-transformation/request-body) +- [request header transform](/api-management/traffic-transformation/request-headers) +- [URL rewrite](/transform-traffic/url-rewriting#url-rewrite-middleware) +- [response body transform](/api-management/traffic-transformation/response-body) +- [response header transform](/api-management/traffic-transformation/response-headers) + +To reference the *Value* assigned to a *Key* in one of the KV stores from these middleware use the following notation: +- Consul: `$secret_consul.key` +- Vault: `$secret_vault.key` +- Tyk config secrets: `$secret_conf.key` +- Environment variables: `$secret_env.key` or `env://key` (see [here](#using-environment-variables-with-transformation-middleware)) + +This notation is used to avoid ambiguity in the middleware parsing (for example where a KV secret is used in a URL rewrite path). + +For example, if you create a Key-Value pair in Consul with the *Key* `user_id` then you could use the *Value* in the `rewriteTo` upstream address in the URL rewrite middleware for your Tyk OAS API by including the following in your API definition: +```json expandable +{ + "x-tyk-api-gateway": { + "middleware": { + "operations": { + "anythingget": { + "urlRewrite": { + "enabled": true, + "pattern": ".*", + "rewriteTo": "/api/v1/users/$secret_consul.user_id", + } + } + } + } + } +} +``` + +When a call is made to `GET /anything`, Tyk will retrieve the *Value* assigned to the `user_id` *Key* in Consul and rewrite the Target URL for the request to `/api/v1/users/{user_id}`. + +These references are read (and replaced with the values read from the KV location) during the processing of the API request or response. + +##### Using environment variables with transformation middleware + +There are some subtleties with the use of environment variables as KV storage for the transformation middleware. + +- **Request and Response Body Transforms** support only the `$secret_env.{KEY_NAME}` format. +- **Request and Response Header Transforms** support both `env://{KEY_NAME}` and `$secret_env.{KEY_NAME}` formats. +- **URL Rewrite** supports the `env://{KEY_NAME}` format for both the `pattern` and `rewriteTo` fields. The `rewriteTo` field also supports `$secret_env.{KEY_NAME}` format. + + + + <Note> + **Notes** + + Due to the way that Tyk Gateway processes the API definition, when you use transformation middleware with the `$secret_env` format, it expects the environment variable to be named `TYK_SECRET_{KEY_NAME}` and to be referenced from the API definition using `$secret_env.{KEY_NAME}`. + <br /><br /> + For example, if you create a Gateway environment variable `TYK_SECRET_NEW_UPSTREAM=http://new.upstream.com`, then in a request body transform middleware, you would reference it as follows: + + ```json expandable + { + "custom_url": "$secret_env.NEW_UPSTREAM" + } + ``` + + To configure the URL rewrite `rewriteTo` field using this variable you could use either: + + ````json + { + "rewriteTo": "env://TYK_SECRET_NEW_UPSTREAM" + } + ```` + or + ````json + { + "rewriteTo": "$secret_env.NEW_UPSTREAM" + } + ```` + </Note> + + +#### Other `string` fields + +To reference the *Value* assigned to a *Key* in one of the KV stores from the API Definition use the following notation: +- Consul: `consul://key` +- Vault: `vault://secret.key` +- Tyk config secrets: `secrets://key` +- Environment variables: `env://key` + +These references are read (and replaced with the values read from the KV location) when the API is loaded to the Gateway (either when Gateway restarts or when there is a hot-reload). + + +<Note> +**Notes** + +When accessing KV references from the `/tyk-apis` directory on Consul or Vault, you should not provide the `path/to/` the key except for Target URL and Listen Path (as described [above](#target-url-and-listen-path)). +</Note> + + +For example, if you create a Key-Value pair in the `secrets` section of the `tyk.conf` file with the *Key* `auth_header_name`: +```json +{ + "secrets": { + "auth_header_name": "Authorization" + } +} +``` + +Then within your API definition you could use the *Value* for the authentication header name as follows: +```json +{ + "x-tyk-api-gateway": { + "components": { + "securitySchemes": { + "authToken": { + "type": "apiKey", + "in": "header", + "name": "secrets://auth_header_name" + } + } + } + } +} +``` + +When the Gateway starts, Tyk will read the *Value* from the `secrets` section in the Gateway config file and use this to identify the header where Tyk Gateway should look for the [Authentication](/api-management/gateway-config-tyk-oas#authentication) token in requests to your Tyk OAS API. + +## Using Consul as a KV store + +HashiCorp [Consul](https://www.consul.io) is a service networking solution that is used to connect and configure applications across dynamic, distributed infrastructure. Consul KV is a simple Key-Value (KV) store provided as a core feature of Consul that can be used to store and retrieve Tyk Gateway configuration across multiple data centers. + +### How to configure Tyk to access Consul + +Configuring Tyk Gateway to read values from Consul is straightforward - you simply configure the connection in your Tyk Gateway config file (`tyk.conf`) by adding the `kv` section as follows: + +```json +{ + "kv": { + "consul": { + "address": "localhost:8025", + "scheme": "http", + "datacenter": "dc-1", + "http_auth": { + "username": "", + "password": "" + }, + "wait_time": 10, + "token": "", + "tls_config": { + "address": "", + "ca_path": "", + "ca_file": "", + "cert_file": "", + "key_file": "", + "insecure_skip_verify": false + } + } + } +} +``` expandable + +| Key | Description | +| :------------ | :------------------------------------------------------------------------------------------------------------- | +| address | The location of the Consul server | +| scheme | The URI scheme for the Consul server, e.g. `http` | +| datacenter | Consul datacenter (agent) identifier | +| http_auth | Username and password for Tyk to log into Consul using HTTP Basic Auth (if required by your Consul service) | +| wait_time | Limits how long a [watch will block](https://developer.hashicorp.com/consul/api-docs/features/blocking) in milliseconds (if enabled in your Consul service) | +| token | Used to provide a per-request access token to Consul (if required by your Consul service) | +| tls_config | Configuration for TLS connection to Consul (if enabled in your Consul service) | + +Alternatively, you can configure it using the equivalent [environment variables](/tyk-oss-gateway/configuration#kvconsuladdress). + +### Where to store data in Consul + +When you want to reference KV data from Tyk Gateway config or transform middleware, you can store your KV pairs wherever you like within the Consul KV store. You can provide the Consul path to the key in the reference using the notation appropriate to the calling [location](#consul). + +From Tyk Gateway 5.3.0, you can reference KV data from any `string` field in the API definition. For these you should create a folder named `tyk-apis` in the root of your Consul KV store and store all keys in a flat structure there (sub-directories not currently supported). You should not include the `tyk-apis` path in the reference so, for example, given a key-value pair `"foo":"bar"` stored in `tyk-apis` in Consul, you would reference this from the API definition using `consul://foo`. + +### How to access data stored in Consul + +The notation used to refer to a KV pair stored in Consul depends upon the location of the reference as follows. + +**Tyk Gateway configuration file** + +As described [here](#from-the-tyk-gateway-configuration-file), from Tyk Gateway's configuration file (`tyk.conf`) you can retrieve values from Consul using the following notation: +- `consul://path/to/KEY` + +**API definition** + +The **Target URL** and **Listen Path** key-value pairs can be stored in any directory in the Consul KV store as they are accessed using a different mechanism than other fields in the API definition. If storing these in a sub-directory, you can retrieve the values from Consul by providing the directory path within Consul KV using the following notation: +- `consul://path/to/KEY` + +For [certain transformation middleware](#transformation-middleware) because the secret resolution happens during the request context, a different notation is used to retrieve values from Consul: +- `$secret_consul.KEY` + +From Tyk Gateway v5.3.0 onwards, you can store KV pairs to be used in **any `string` field** in the API definition in the Consul KV store. You can retrieve these values from Consul, noting that you do not provide the directory path (`/tyk-apis`) when accessing data for *these* fields, using the following notation: +- `consul://KEY` + + +## Using Vault as a KV store + + +[Vault](https://vaultproject.io) from Hashicorp is a tool for securely accessing secrets. It provides a unified interface to any secret while providing tight access control and recording a detailed audit log. Tyk Gateway can use Vault to manage and retrieve sensitive secrets such as API keys and passwords. + +### How to configure Tyk to access Vault + +Configuring Tyk Gateway to read values from Vault is straightforward - you simply configure the connection in your Tyk Gateway config file (`tyk.conf`) by adding the `kv` section as follows: + +```json +{ + "kv": { + "vault": { + "address": "http://localhost:1023", + "agent_address": "", + "max_retries": 3, + "timeout": 30, + "token": "", + "kv_version": 2 + } + } +} +``` expandable + +| Key | Description | +| :-------------- | :-------------------------------------------------------------------------------------------------------- | +| address | The address of the Vault server, which must be a complete URL such as `http://www.vault.example.com` | +| agent_address | The address of the local Vault agent, if different from the Vault server, must be a complete URL | +| max_retries | The maximum number of attempts Tyk will make to retrieve the value if Vault returns an error | +| timeout | The maximum time that Tyk will wait for a response from Vault (in nanoseconds, if set to 0 (default) will be interpreted as 60 seconds) | +| token | The Vault root access token | +| kv_version | The version number of Vault, usually defaults to 2 | + +Alternatively, you can configure it using the equivalent [environment variables](/tyk-oss-gateway/configuration#kvvaulttoken). + +### How key-value data is stored in Vault + +In traditional systems secrets are typically stored individually, each with their own unique key. Vault, however, allows for a more flexible approach where multiple *keys* can be grouped together and stored under a single *secret*. This grouping allows for better organization and management of related secrets, making it easier to retrieve and manage them collectively. + +When retrieving data from Vault, you use the dot notation (`secret.key`) to access the *value* from a specific *key* within a *secret*. + +**Example of storing key value data in Vault** + +If you want to store a *secret* named `tyk` with a *key* `gw` and *value* `123` in Vault then, from the command line, you would: +1. Enable the `kv` secrets engine in Vault under the path `my-secret` using: + `vault secrets enable -version=2 -path=my-secret kv` +2. Create a secret `tyk` with the key `gw` and value `123` in Vault: + `vault kv put my-secret/tyk gw=123` + +To retrieve the secret from Vault using the command line you would use the following command (there is no need to append `/data` to the secret path): +```curl +curl \ + --header "X-Vault-Token: <your_vault_token>" \ + --request GET \ + https://vault-server.example.com/v1/my-secret/tyk?lease=true +``` + +This would return a response along these lines, note that the response contains all the keys stored in the secret (here there are also keys called `excited` and `foo`): +```yaml +{ + "request_id": "0c7e44e1-b71d-2102-5349-b5c60c13fb02", + "lease_id": "", + "lease_duration": 0, + "renewable": false, + "data": { + "gw": "123", + "excited": "yes", + "foo": "world", + }, + "metadata":{ + "created_time": "2019-08-28T14:18:44.477126Z", + "deletion_time": "", + "destroyed": false, + "version": 1 + }, + "auth": ... +} +``` + +As explained [below](#vault), you could retrieve this value from within your Tyk Gateway config file using: + `TYK_GW_SECRET=vault://my-secret/tyk.gw` + +### Where to store data in Vault + +When you want to reference KV data from Tyk Gateway config or transform middleware, you can store your Vault *secrets* wherever you like within the KV store. You can provide the Vault path to the key in the reference using the notation appropriate to the calling [location](#vault). + +From Tyk Gateway 5.3.0, you can reference KV data from any `string` field in the API definition. For these you should create a folder named `tyk-apis` in the root of your Vault KV store and store all *secrets* in a flat structure there (sub-directories not currently supported). You should not include the `tyk-apis` path in the reference so, for example, given a key-value pair `"foo":"bar"` stored in a secret named `my-secret` in `/tyk-apis` in Vault, you would reference this from the API definition using `vault://my-secret.foo`. + +### How to access data stored in Vault + +The notation used to refer to a key-value pair stored in Vault depends upon the location of the reference as follows. + +**Tyk Gateway configuration file** + +As described [here](#from-the-tyk-gateway-configuration-file), from Tyk Gateway's configuration file (`tyk.conf`) you can retrieve values from Vault using the following notation: +- `vault://path/to/secret.KEY` + +**API definition** + +The **Target URL** and **Listen Path** key-value pairs can be stored in any directory in the Vault KV store as they are accessed using a different mechanism than other fields in the API definition. If storing these in a sub-directory, you can retrieve the values from Vault by providing the directory path within Consul KV using the following notation: +- `vault://path/to/secret.KEY` + +For [certain transformation middleware](#transformation-middleware) because the secret resolution happens during the request context, a different notation is used to retrieve values from Vault: +- `$secret_vault.KEY` + +From Tyk Gateway v5.3.0 onwards, you can store KV pairs to be used in **any `string` field** in the API definition in the Vault KV store. You can retrieve these values from Vault, noting that you do not provide the directory path (`/tyk-apis`) when accessing data for *these* fields, using the following notation: +- `vault://KEY` diff --git a/tyk-configuration-reference/redis-cluster-sentinel.mdx b/tyk-configuration-reference/redis-cluster-sentinel.mdx new file mode 100644 index 00000000..380dcfa5 --- /dev/null +++ b/tyk-configuration-reference/redis-cluster-sentinel.mdx @@ -0,0 +1,330 @@ +--- +title: "Configure Redis Cluster and Sentinel" +'og:description': "This page provides guidance on configuring Tyk to work with Redis Cluster and Sentinel, including TLS encryption and troubleshooting tips." +sidebarTitle: "Redis" +tags: ['configuration', 'redis', 'cluster', 'sentinel', 'tyk-gateway', 'tyk-dashboard', 'tyk-pump'] +--- + +## Configure Redis Cluster + +Our Gateway, Dashboard and Pump all support integration with Redis Cluster. Redis Cluster allows data to be automatically sharded across multiple Redis Nodes. To setup Redis Cluster correctly, we recommend you read the [Redis Cluster Tutorial](https://redis.io/topics/cluster-tutorial). You must use the same settings across the Gateway, Dashboard and Pump. + + +<Note> +Redis Cluster operates differently from a Redis setup where one instance serves as the primary and others as replicas. +</Note> + + +### Supported Versions +- Tyk 5.3 supports Redis 6.2.x, 7.0.x, and 7.2.x +- Tyk 5.2.x and earlier supports Redis 6.0.x and Redis 6.2.x only. + + +### Redis Cluster and Tyk Gateway + +To configure the Tyk Gateway to work with your Redis Cluster, set `enable_cluster` to `true` and list your servers under `addrs` in your `tyk.conf` file. + + +<Note> +`addrs` is new in v2.9.3, and replaces `hosts` which is now deprecated. +</Note> + + +If you are using TLS for Redis connections, set `use_ssl` to `true`. + +```json expandable +"storage": { + "type": "redis", + "enable_cluster": true, + "addrs": [ + "server1:6379", + "server2:6380", + "server3:6381" + ], + "username": "", + "password": "", + "database": 0, + "optimisation_max_idle": 2000, + "optimisation_max_active": 4000, + "use_ssl": false +}, +``` + +### Redis Cluster and Tyk Dashboard + + +<Note> +`redis_addrs` is new in v1.9.3 for the Dashboard, and replaces `hosts` which is now deprecated. +</Note> + + + +```json expandable +"redis_addrs": [ + "server1:6379", + "server2:6380", + "server3:6381" + ], +"redis_use_ssl": true, +"enable_cluster": true +``` +To configure the Tyk Dashboard to work with your Redis Cluster, add the Redis address information to your `tyk_analytics.conf` file: + + +### Redis Cluster and Tyk Pump + +To configure the Tyk Pump to work with your Redis Cluster, set `enable_cluster` to `true` and list your servers under `addrs` in your `pump.conf` file. + + +<Note> +`addrs` is new in v2.9.3, and replaces `hosts` which is now deprecated. +</Note> + + + +```json expandable +"analytics_storage_config": { + "type": "redis", + "enable_cluster": true, + "addrs": [ + "server1:6379", + "server2:6380", + "server3:6381" + ], + "username": "", + "password": "", + "database": 0, + "optimisation_max_idle": 100, + "use_ssl": false +}, +``` + +### Redis Cluster with Docker + +For Redis clustered mode to work with Tyk using Docker and Amazon ElastiCache, follow these two steps: + +1. **Make sure cluster mode is enabled** + +Set the environment variable `TYK_GW_STORAGE_ENABLECLUSTER` to `true`. + +2. **Add all cluster endpoints to the config** + +Add all the Redis Cluster endpoints into Tyk, not just the primary. If Tyk can't see the whole cluster, then it will not work. + +For ElastiCache Redis, you can bypass having to list all your nodes, and instead just use the *configuration endpoint*, +this allows read and write operations and the endpoint will determine the correct node to target. + +If this does not work, you can still list out the hosts using an environment variable. To do so, set the environment variable: + +```{.copyWrapper} +TYK_GW_STORAGE_ADDRS="redis_primary1:port,redis_replica1:port,redis_primary2:port,redis_replica2:port,redis_primary3:port,redis_replica3:port" +``` + +It is important that Tyk can connect to all primary and replica instances. + +It is recommended to ensure that the connection pool is big enough. To do so, set the following environment variables: + +```{.copyWrapper} +TYK_GW_STORAGE_MAXIDLE=6000 +TYK_GW_STORAGE_MAXACTIVE=10000 +``` + +<Note> +These are suggested settings, please verify them by load testing. +</Note> + + +#### Redis Cluster with TLS + +If you are using TLS for Redis connections, set `use_ssl` to `true` for Gateway and Pump, and `redis_use_ssl` to `true` for the dashboard. +Redis supports [SSL/TLS encryption](https://redis.io/topics/encryption) from version 6 as an optional feature, enhancing the security of data in transit. Similarly, Amazon ElastiCache offers encryption in transit and at rest. To configure TLS or mTLS connections between an application and Redis, consider the following settings in Tyk's configuration files: + +- `storage.use_ssl`: Set this to true to enable TLS encryption for the connection. + +- `storage.ssl_insecure_skip_verify`: A flag that, when set to true, instructs the application not to verify the Redis server's TLS certificate. This is not recommended for production due to the risk of `man-in-the-middle` attacks. + +From **Tyk 5.3**, additional options are available for more granular control: + +- `storage.ca_file`: Path to the Certificate Authority (CA) file for verifying the Redis server's certificate. + +- `storage.cert_file` and `storage.key_file`: Paths to your application's certificate and private key files, necessary for mTLS where both parties verify each other's identity. + +- `storage.max_version` and `storage.min_version`: Define the acceptable range of TLS versions, enhancing security by restricting connections to secure TLS protocols (1.2 or 1.3). + +**Setting up an Insecure TLS Connection** +- **Enable TLS**: By setting `"use_ssl": true`, you encrypt the connection. +- **Skip Certificate Verification**: Setting `"ssl_insecure_skip_verify": true` bypasses the server's certificate verification, suitable only for non-production environments. + +**Setting up a Secure TLS Connection** +- Ensure `use_ssl` is set to `true`. +- Set `ssl_insecure_skip_verify` to `false` to enforce certificate verification against the CA specified in `ca_file`. +- Specify the path to the CA file in `ca_file` for server certificate verification. +- Adjust `min_version` and `max_version` to secure TLS versions, ideally 1.2 and 1.3. + +**Setting up a Mutual TLS (mTLS) Connection** +- Follow the steps for a secure TLS connection. +- Provide paths for `cert_file` and `key_file` for your application's TLS certificate and private key, enabling Redis server to verify your application's identity. + +**Example Gateway Configuration** +```json expandable +"storage": { + "type": "redis", + "addrs": [ + "server1:6379", + "server2:6380", + "server3:6381" + ], + "use_ssl": true, + "ssl_insecure_skip_verify": false, + "ca_file": "/path/to/ca.crt", + "cert_file": "/path/to/client.crt", + "key_file": "/path/to/client.key", + "max_version": "1.3", + "min_version": "1.2", + "enable_cluster": true, + "optimisation_max_idle": 2000, + "optimisation_max_active": 4000 +} +``` + +#### Troubleshooting Redis Cluster + +If you find that Tyk components fail to initialise when using Redis clustering, for example the application does not start and the last log file entry shows a message such as `Using clustered mode`, try setting the environment variable `REDIGOCLUSTER_SHARDCOUNT` to `128` on all hosts which connect to the Redis Cluster i.e. Gateway, Dashboard, Pump, MDCB. E.g. + +`REDIGOCLUSTER_SHARDCOUNT=128` + +If setting to `128` does not resolve the issue, try `256` instead. + + +## Configure Redis Sentinel + +From v2.9.3 Redis Sentinel is supported. + +Similar to Redis Cluster, our Gateway, Dashboard and Pump all support integration with Redis Sentinel. + +To configure Tyk to work with Redis Sentinel, list your servers under `addrs` and set the master name in your Gateway, Dashboard, Pump and MDCB config. Unlike Redis Cluster, `enable_cluster` should **not** be set. Indicative config snippets as follows: + +### Supported Versions +- Tyk 5.3 supports Redis 6.2.x, 7.0.x, and 7.2.x +- Tyk 5.2.x and earlier supports Redis 6.0.x and Redis 6.2.x only. + + +### Redis Sentinel and Gateway + +```json expandable +"storage": { + "type": "redis", + "addrs": [ + "server1:26379", + "server2:26379", + "server3:26379" + ], + "master_name": "mymaster", + "username": "", + "password": "", + "database": 0, + "optimisation_max_idle": 2000, + "optimisation_max_active": 4000, + "use_ssl": false +}, +``` + +### Redis Sentinel and Dashboard + +```json expandable +"redis_addrs": [ + "server1:26379", + "server2:26379", + "server3:26379" +], +"redis_master_name": "mymaster" +``` + +### Redis Sentinel and Pump + +```json expandable +"analytics_storage_config": { + "type": "redis", + "addrs": [ + "server1:26379", + "server2:26379", + "server3:26379" + ], + "master_name": "mymaster", + "username": "", + "password": "", + "database": 0, + "optimisation_max_idle": 100, + "use_ssl": false +}, +``` + + +<Warning> +When using Bitnami charts to install Redis Sentinel in k8s, a Redis service is exposed, which means that standard Redis config is required instead of the above setup, i.e. a single server in `addrs` and `master_name` is not required. +</Warning> + + +### Support for Redis Sentinel AUTH + +To support the use of Redis Sentinel AUTH (introduced in Redis 5.0.1) we have added the following global config settings in Tyk v3.0.2: + +* In the Tyk Gateway config file - `sentinel_password` +* In the Tyk Dashboard config file - `redis_sentinel_password` +* In the Tyk Pump config file - `sentinel_password` +* In the Tyk Identity Broker config file - `SentinelPassword` +* In the Tyk Synk config file - `sentinel_password` + +These settings allow you to support Sentinel password-only authentication in Redis version 5.0.1 and above. + +See the Redis and Sentinel authentication section of the [Redis Sentinel docs](https://redis.io/topics/sentinel) for more details. + +### Configure Redis TLS Encryption +Redis supports [SSL/TLS encryption](https://redis.io/topics/encryption) from version 6 as an optional feature, enhancing the security of data in transit. To configure TLS or mTLS connections between an application and Redis, consider the following settings in Tyk's configuration files: + +- `storage.use_ssl`: Set this to true to enable TLS encryption for the connection. + +- `storage.ssl_insecure_skip_verify`: A flag that, when set to true, instructs the application not to verify the Redis server's TLS certificate. This is not recommended for production due to the risk of `man-in-the-middle` attacks. + +From **Tyk 5.3**, additional options are available for more granular control: + +- `storage.ca_file`: Path to the Certificate Authority (CA) file for verifying the Redis server's certificate. + +- `storage.cert_file` and `storage.key_file`: Paths to your application's certificate and private key files, necessary for mTLS where both parties verify each other's identity. + +- `storage.max_version` and `storage.min_version`: Define the acceptable range of TLS versions, enhancing security by restricting connections to secure TLS protocols (1.2 or 1.3). + +**Setting up an Insecure TLS Connection** +- **Enable TLS**: By setting `"use_ssl": true`, you encrypt the connection. +- **Skip Certificate Verification**: Setting `"ssl_insecure_skip_verify": true` bypasses the server's certificate verification, suitable only for non-production environments. + +**Setting up a Secure TLS Connection** +- Ensure `use_ssl` is set to `true`. +- Set `ssl_insecure_skip_verify` to `false` to enforce certificate verification against the CA specified in `ca_file`. +- Specify the path to the CA file in `ca_file` for server certificate verification. +- Adjust `min_version` and `max_version` to secure TLS versions, ideally 1.2 and 1.3. + +**Setting up a Mutual TLS (mTLS) Connection** +- Follow the steps for a secure TLS connection. +- Provide paths for `cert_file` and `key_file` for your application's TLS certificate and private key, enabling Redis server to verify your application's identity. + +**Example Gateway Configuration** +```json expandable +"storage": { + "type": "redis", + "addrs": [ + "server1:6379", + "server2:6380", + "server3:6381" + ], + "use_ssl": true, + "ssl_insecure_skip_verify": false, + "ca_file": "/path/to/ca.crt", + "cert_file": "/path/to/client.crt", + "key_file": "/path/to/client.key", + "max_version": "1.3", + "min_version": "1.2", + "optimisation_max_idle": 2000, + "optimisation_max_active": 4000 +} +``` diff --git a/tyk-configuration-reference/tyk-identity-broker-configuration.mdx b/tyk-configuration-reference/tyk-identity-broker-configuration.mdx new file mode 100644 index 00000000..f0055aea --- /dev/null +++ b/tyk-configuration-reference/tyk-identity-broker-configuration.mdx @@ -0,0 +1,220 @@ +--- +title: "Tyk Identity Broker Configuration Options" +order: 3 +sidebarTitle: "Tyk Identity Broker" +--- + +The Tyk Identity Broker (TIB) is configured through two files: The configuration file `tib.conf` and the profiles file `profiles.json`. TIB can also be managed via the [TIB REST API](/tyk-identity-broker/tib-rest-api) for automated configurations. + +#### The `tib.conf` file + +```{.copyWrapper} expandable +{ + "Secret": "test-secret", + "ProfileDir": "path-to-backup-directory", + "HttpServerOptions": { + "UseSSL": true, + "CertFile": "./certs/server.pem", + "KeyFile": "./certs/server.key" + }, + "BackEnd": { + "Name": "in_memory", + "IdentityBackendSettings": { + "Hosts" : { + "localhost": "6379" + }, + "Username": "", + "Password": "", + "Database": 0, + "EnableCluster": false, + "MaxIdle": 1000, + "MaxActive": 2000, + "UseSSL": false, + "SSLInsecureSkipVerify": false + } + }, + "TykAPISettings": { + "GatewayConfig": { + "Endpoint": "http://{GATEWAY-DOMAIN}", + "Port": "8080", + "AdminSecret": "352d20ee67be67f6340b4c0605b044b7" + }, + "DashboardConfig": { + "Endpoint": "http://{DASHBOARD-DOMAIN}", + "Port": "3000", + "AdminSecret": "12345" + } + } +} +``` + +### Omitting the configuration file + +From TIB v1.3.1, the environment variable `TYK_IB_OMITCONFIGFILE` is provided to allow the configuration file to be omitted (ignored) when configuring TIB. + +If set to TRUE, then TIB will ignore any provided configuration file and set its parameters according to environment variables. TIB will fall back to the default value for any parameters not set in an environment variable. +This is particularly useful when using Docker, as this option will ensure that TIB will load the configuration via env vars and not expect a configuration file. + +The various options for `tib.conf` file are: + +### secret + +The REST API secret to configure the Tyk Identity Broker remotely. + +(env var:**TYK_IB_SECRET**) + +### ProfileDir + +Directory where the backup files will be stored. Backups files are created each time that a create, update or delete action is performed over any profile (and profiles are being read from a file not from mongo, in which case it will create a new document in the `profiles_backup` collection). + +(env var:**TYK_IB_PROFILEDIR**) + +### HttpServerOptions.UseSSL + +Set this to `true` to turn on SSL for the server, this is **highly recommended**. + +(env var:**TYK_IB_HTTPSERVEROPTIONS_USESSL**) + +### HttpServerOptions.KeyFile + +The path to the key file for this server, required for SSL. + +(env var:**TYK_IB_HTTPSERVEROPTIONS_KEYFILE**) + +### HttpServerOptions.CertFile + +The path to the certificate file for this server, required for SSL. + +(env var:**TYK_IB_HTTPSERVEROPTIONS_CERTFILE**) + +### BackEnd + +TIB is quite modular and different back-ends can be generated quite easily. By default, TIB will store profile configurations in memory, which does not require any new configuration. + +For Identity Handlers that provide token-based access, it is possible to enforce a "One token per provider, per user" policy, which keeps a cache of tokens assigned to identities in Redis, this is so that the broker can be scaled and share the cache across instances. + +Since profiles are unlikely to change often, profiles are kept in-memory, but can be added, removed and modified using an API for automated setups if required. + +### BackEnd.IdentityBackendSettings.Database + +If you are using multiple databases (not supported in Redis cluster), let TIB know which DB to use for Identity caching. + +(env var:**TYK_IB_BACKEND_IDENTITYBACKENDSETTINGS_DATABASE**) + +### BackEnd.IdentityBackendSettings.Username + +The username for Redis AUTH, if used (recommended). + +(env var:**TYK_IB_BACKEND_IDENTITYBACKENDSETTINGS_USERNAME**) + +### BackEnd.IdentityBackendSettings.Password + +The password for your Redis AUTH Username. + +(env var:**TYK_IB_BACKEND_IDENTITYBACKENDSETTINGS_PASSWORD**) + +### BackEnd.IdentityBackendSettings.Hosts + +Add your Redis hosts here as a map of hostname:port. Since TIB uses the same cluster driver as Tyk, it is possible to have TIB interact with your existing Redis cluster if you enable it. + +(env var:**TYK_IB_BACKEND_IDENTITYBACKENDSETTINGS_HOSTS**) + + +<Note> +To set this value via env var you must follow the declaration syntax `export TYK_IB_BACKEND_IDENTITYBACKENDSETTINGS_HOSTS="host1:port,host2:port"` +</Note> + + +### BackEnd.IdentityBackendSettings.MaxIdle + +Max idle connections to Redis. + +(env var:**TYK_IB_BACKEND_IDENTITYBACKENDSETTINGS_MAXIDLE**) + +### BackEnd.IdentityBackendSettings.MaxActive + +Max active Redis connections. + +(env var:**TYK_IB_BACKEND_IDENTITYBACKENDSETTINGS_MAXACTIVE**) + +### BackEnd.IdentityBackendSettings.EnableCluster + +If you are using Redis cluster, enable it here to enable the slots mode. + +(env var:**BackEnd.IdentityBackendSettings.EnableCluster**) + +### BackEnd.IdentityBackendSettings.UseSSL + +If you are using a TLS protected Redis enable to connect. + +(env var:**TYK_IB_BACKEND_IDENTITYBACKENDSETTINGS_USESSL**) + + +<Note> +This option is available from TIB v0.4.0 +</Note> + + +### BackEnd.IdentityBackendSettings.SSLInsecureSkipVerify + +Allows usage of self-signed certificates when connecting to an encrypted Redis database. + +(env var:**TYK_IB_BACKEND_IDENTITYBACKENDSETTINGS_SSLINSECURESKIPVERIFY**) + + +<Note> +This option is available from TIB v0.4.0 +</Note> + + + +### TykAPISettings + +This section enables you to configure the API credentials for the various Tyk Components TIB is interacting with. + +(env var:**TYK_IB_TYKAPISETTINGS**) + +### TykAPISettings.GatewayConfig.Endpoint + +The hostname of the Tyk Gateway (this is for token generation purposes). + +(env var:**TYK_IB_TYKAPISETTINGS_GATEWAYCONFIG_ENDPOINT**) + +### TykAPISettings.GatewayConfig.Port + +The port to use on the Tyk Gateway host. + +(env var:**TYK_IB_TYKAPISETTINGS_GATEWAYCONFIG_PORT**) + + +<Note> +For HTTP or HTTPS endpoints, you do need need to specify the default ports (80 and 443) for this setting. These two ports are handled automatically. +</Note> + + + +### TykAPISettings.GatewayConfig.AdminSecret + +The API secret for the Tyk Gateway REST API. + +(env var:**TYK_IB_TYKAPISETTINGS_GATEWAYCONFIG_ADMINSECRET**) + +### TykAPISettings.DashboardConfig.Endpoint + +The hostname of your Dashboard (Advanced API). + +(env var:**TYK_IB_TYKAPISETTINGS_DASHBOARDCONFIG_ENDPOINT**) + +### TykAPISettings.DashboardConfig.Port + +The port of your Advanced API. + +(env var:**TYK_IB_TYKAPISETTINGS_DASHBOARDCONFIG_PORT**) + +### TykAPISettings.DashboardConfig.AdminSecret + +The high-level secret for the Advanced API. This is required because of the SSO-nature of some of the actions provided by TIB, it requires the capability to access a special SSO endpoint in the Advanced API to create one-time tokens for access. + +(env var:**TYK_IB_TYKAPISETTINGS_DASHBOARDCONFIG_ADMINSECRET**) + + diff --git a/tyk-dashboard-api.mdx b/tyk-dashboard-api.mdx new file mode 100644 index 00000000..52e3ef24 --- /dev/null +++ b/tyk-dashboard-api.mdx @@ -0,0 +1,6 @@ +--- +title: "Tyk Dashboard API" +order: 3 +sidebarTitle: "Dashboard" +--- + diff --git a/tyk-dashboard/configuration.mdx b/tyk-dashboard/configuration.mdx new file mode 100644 index 00000000..90edce3e --- /dev/null +++ b/tyk-dashboard/configuration.mdx @@ -0,0 +1,210 @@ +--- +title: "Tyk Dashboard Configuration Options" +order: 2 +sidebarTitle: "Dashboard" +--- + +import DashboardConfig from '/snippets/dashboard-config.mdx'; + +You can use environment variables to override the config file for the Tyk Dashboard. The Dashboard configuration file can be found in the `tyk-dashboard` folder and by default is called `tyk_analytics.conf`, though it can be renamed and specified using the `--conf` flag. Environment variables are created from the dot notation versions of the JSON objects contained with the config files. +To understand how the environment variables notation works, see [Environment Variables](/tyk-oss-gateway/configuration). + +The Tyk Dashboard has a separate configuration file, it is small and comes packaged with the tarball. It uses a separate configuration file as it may be installed on a different host to your Tyk Gateway nodes. + +The Dashboard configuration file can be found in the `tyk-dashboard` folder and by default is called `tyk_analytics.conf`, though it can be renamed and specified using the `--conf` flag. + +Please consult the [data storage configuration](/api-management/dashboard-configuration#data-storage-solutions) guide for further information relating to how to configure Tyk's data storage across different database engines. + +### Environment Variables + +All the Dashboard environment variables have the prefix `TYK_DB_`. The environment variables will take precedence over the values in the configuration file. + +Environment variables (env var) can be used to override the settings defined in the configuration file. Where an environment variable is specified, its value will take precedence over the value in the configuration file. + +The file will look like the sample below, the various fields are explained in the following sections: + +```json expandable +{ + "listen_port": 3000, + "tyk_api_config": { + "Host": "http://tyk-gateway", + "Port": "8080", + "Secret": "352d20ee67be67f6340b4c0605b044b7" + }, + "enable_aggregate_lookups": true, + "storage": { + "main": { + "type": "postgres", + "connection_string": "user=laurentiughiur password=test123 database='tyk-test' host=127.0.0.1 port=5432", + "table_sharding": true + }, + "analytics": { + "type": "postgres", + "connection_string": "user=laurentiughiur password=test123 database='tyk-test' host=127.0.0.1 port=5432", + "table_sharding": true + }, + "logs": { + "type": "postgres", + "connection_string": "user=laurentiughiur password=test123 database='tyk-test' host=127.0.0.1 port=5432", + "table_sharding": true + }, + "uptime": { + "type": "postgres", + "connection_string": "user=laurentiughiur password=test123 database='tyk-test' host=127.0.0.1 port=5432", + "table_sharding": true + } + }, + "enable_ownership": false, + "mongo_url": "mongodb://tyk-mongo:27017/tyk_analytics", + "mongo_use_ssl": false, + "mongo_ssl_insecure_skip_verify": false, + "mongo_session_consistency": "", + "mongo_batch_size": 2000, + "page_size": 10, + "admin_secret": "12345", + "shared_node_secret": "352d20ee67be67f6340b4c0605b044b7", + "redis_port": 6379, + "redis_host": "tyk-redis", + "redis_username": "", + "redis_password": "", + "redis_master_name": "", + "redis_timeout": 0, + "redis_database": 0, + "enable_cluster": false, + "redis_use_ssl": false, + "redis_ssl_insecure_skip_verify": false, + "force_api_defaults": false, + "notify_on_change": true, + "license_key": "", + "redis_hosts": null, + "redis_addrs": null, + "hash_keys": true, + "enable_hashed_keys_listing": false, + "email_backend": { + "enable_email_notifications": false, + "code": "sendgrid", + "settings": { + "ClientKey": "" + }, + "default_from_email": "you@somewhere.com", + "default_from_name": "Some Person", + "dashboard_hostname": "" + }, + "hide_listen_path": false, + "sentry_code": "", + "sentry_js_code": "", + "use_sentry": false, + "enable_master_keys": false, + "enable_duplicate_slugs": true, + "show_org_id": true, + "host_config": { + "enable_host_names": true, + "disable_org_slug_prefix": true, + "hostname": "www.tyk-test.com", + "override_hostname": "www.tyk-test.com:8080", + "portal_domains": {}, + "portal_root_path": "/portal", + "generate_secure_paths": false, + "secure_cookies": false, + "use_strict_hostmatch": false + }, + "http_server_options": { + "use_ssl": false, + "certificates": [], + "min_version": 0, + "ssl_ciphers": null, + "ssl_insecure_skip_verify": false + }, + "basic-config-and-security/security": { + "allow_admin_reset_password": false, + "login_failure_username_limit": 0, + "login_failure_ip_limit": 0, + "login_failure_expiration": 0, + "login_disallow_forward_proxy": false, + "audit_log_path": "", + "user_password_max_days": 0, + "enforce_password_history": 0, + "force_first_login_pw_reset": false, + "enable_content_security_policy": false, + "allowed_content_sources": "", + "private_certificate_encoding_secret": "some-secret", + "open_policy":{ + "enabled": true, + "debug": true, + "enable_api": true + }, + "additional_permissions": { + "api_manager": "API Manager" + } + }, + "ui": { + "languages": { + "Chinese": "cn", + "English": "en", + "Korean": "ko" + }, + "hide_help": true, + "default_lang": "en", + "login_page": {}, + "nav": { + "dont_show_admin_sockets": false, + "hide_activity_by_api_section": false, + "hide_geo": false, + "hide_licenses_section": false, + "hide_logs": false, + "hide_tib_section": false + }, + "uptime": {}, + "portal_section": null, + "designer": {}, + "dont_show_admin_sockets": false, + "dont_allow_license_management": false, + "dont_allow_license_management_view": false, + "cloud": false + }, + "home_dir": "/opt/tyk-dashboard", + "identity_broker": { + "enabled": false, + "host": { + "connection_string": "", + "secret": "" + } + }, + "tagging_options": { + "tag_all_apis_by_org": false + }, + "use_sharded_analytics": true, + "enable_aggregate_lookups": true, + "aggregate_lookup_cutoff": "26/05/2016", + "maintenance_mode": false, + "allow_explicit_policy_id": true, + "private_key_path": "", + "node_schema_path": "", + "oauth_redirect_uri_separator": ";", + "statsd_connection_string": "", + "statsd_prefix": "", + "disable_parallel_sessions": false, + "dashboard_session_lifetime": 0, + "alternative_dashboard_url": "", + "sso_permission_defaults": null, + "sso_default_group_id": "", + "sso_custom_login_url": "", + "sso_custom_portal_login_url": "", + "sso_enable_user_lookup": false, + "notifications_listen_port": 5000, + "portal_session_lifetime": 0, + "enable_delete_key_by_hash": false, + "enable_update_key_by_hash": false, + "audit": { + "enabled": false, + "format": "", + "path": "", + "detailed_recording": false + }, + "enable_multi_org_users": false, + "version_check_url": "", + "health_check_endpoint_name": "" +} +``` + +<DashboardConfig/> diff --git a/tyk-developer-portal/customise.mdx b/tyk-developer-portal/customise.mdx new file mode 100644 index 00000000..e16fe64c --- /dev/null +++ b/tyk-developer-portal/customise.mdx @@ -0,0 +1,25 @@ +--- +title: "Portal Customization Overview" +order: 12 +noindex: True +sidebarTitle: "Overview" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +### Customize look and feel + +There are two primary ways to customize the developer portal. The first is to use the built-in CSS editor to inject new CSS into the existing portal templates. This is primarily useful for Cloud and Multi-Cloud users. + +The second method is to edit the Portal templates directly. The Tyk Dashboard ships with all the templates that make up the portal, these templates are based on Twitter Bootstrap and can be easily changed to suit your branding. + +All templates in Tyk Portal use the Golang Templating engine and have the capability to display Markdown. + +### Customize content + +The Tyk Portal comes with a basic CMS which can push content into Tyk Templates as markdown. That means a certain amount of customization around content can be done from within a live installation without needing to modify CSS or templates. + +The CMS and Menu Editor allows you to generate pages and menu structures that can be displayed in your Portal so that you can embellish the portal with additional content around your company, brand and bio. + diff --git a/tyk-developer-portal/customise/customize-api-visibility.mdx b/tyk-developer-portal/customise/customize-api-visibility.mdx new file mode 100644 index 00000000..ef912aad --- /dev/null +++ b/tyk-developer-portal/customise/customize-api-visibility.mdx @@ -0,0 +1,416 @@ +--- +title: "Customizing API Visibility" +'og:description': "A walk through how you can use custom Page Templates to control the visibility of your APIs so it can only be seen by specific group of developers." +tags: ['customizing EDP', 'EDP', 'customizing APIs EDP'] +noindex: True +sidebarTitle: "Customising API Visibility" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +By default, any user who accesses your developer Portal will be able to view all of the published APIs in the catalog. This behavior may not be desired and you may want to have more control of what APIs developers see in the catalog when accessing the portal. A common use case for this is if you have internal APIs that you want to publish only to your internal developers, and restrict view to others. + +We'll walk through how you can use custom Page Templates to control the visibility of your APIs so it can only be seen by specific group of developers. +In a nutshell, we are going to assign a group field to an API catalog profile, to a developer profile, and check if their group matched. + +*Please note that this does not support multiple groups for a single API catalog entry, nor for a single developer profile.* + +## Prerequisites +1. You have an API created in your Dashboard. See [Create an API](/api-management/gateway-config-managing-classic#create-an-api) for more details. +2. You have a Policy created in your Dashboard that has access rights to this API +3. You have a Portal Catalog entry for this API. Here we will call it "Internal API" +4. You have a developer account that can access your Developer Portal. + +## Add a group field to the API Catalog profile + +For this example, we'll add a custom field to the Portal catalog "Group". This group is set to "internal" which indicates that only developers in `internal` group shoud have access to the Catalog. + +Go to Portal Management > Catalog -> Your API screen + +<img src="/img/dashboard/portal-management/portal_catalogue_field_group.png" alt="portal_catalogue_fied_group" /> + + +## Add a custom field to the developer profile + +For this example, we'll add a custom field to the developer profile also called "Group". This group is set set to "internal" it means that developer should have access to the catalogs with the same Group restriction. + +Go to Portal Management > Developers screen +<img src="/img/dashboard/portal-management/deveoper_field_group.png" alt="developer_field_group.png" /> + + +This flag can also be [set programatically](/tyk-developer-portal/tyk-portal-classic/customise/custom-developer-portal#updating-a-developer-example-adding-custom-fields). + + +## Modify the Portal Catalog Template to add Show/Hide Logic + +The developer portal is fully customizable via templates. We'll add custom logic to the portal catalog template (catalogue.html) to show/hide the "Internal API" catalog based on the value of the "Group" field for the developer. + +The main difference from the default template is two changes: +1. Get user data state at the start of template: `{{$profile := .UserData }}` +2. Before rendering api catalog element, which renders list of APIs, we insert the following section: + +```go-html-template expandable +{{ $show := true }} + +{{ range $field, $value := $apiDetail.Fields }} + {{ $group_match := true }} + {{ if (eq $field "Group") }} + {{ $group_match = false }} + {{ range $dfield, $dvalue := $profile.Fields }} + {{ if eq $dfield "Group" }} + {{ if eq $dvalue $value }} + {{ $group_match = true }} + {{ end }} + {{ end }} + {{ end }} + {{ end }} + + {{ if not $group_match }} + {{ $show = false }} + {{ end }} +{{ end }} + +{{if $show}} +{/* Render catalog */} +{{end}} +``` + +<Expandable title={'Click to expand and see the customized catalog template'}> +```go-html-template expandable +{{ define "cataloguePage" }} {{ $org_id := .OrgId}} {{ template "header" .}} +{{ $page := .}} +{{$profile := .UserData }} +<body> + {{ template "navigation" . }} + <div> + {/* Main content here */} + <div class="container" style="margin-top:80px;"> + <div class="row"> + <h1>API Catalog</h1> + </div> + {{ if .Error }} + <div class="row"> + <div class="col-md-7 center"> + <div class="alert alert-danger text-center col-lg-8 col-lg-offset-2 success-page error">Error while loading data</div> + </div> + </div> + {{ end }} + <div class="row catalogues-list"> + {{$internal := index .UserData.Fields "internal"}} + + {{ if .Data.APIS }} {{ range $index, $apiDetail := .Data.APIS}} {{ if $apiDetail.Show }} + + {{ $show := true }} + + {{ range $field, $value := $apiDetail.Fields }} + {{ $group_match := true }} + {{ if (eq $field "Group") }} + {{ $group_match = false }} + {{ range $dfield, $dvalue := $profile.Fields }} + {{ if eq $dfield "Group" }} + {{ if eq $dvalue $value }} + {{ $group_match = true }} + {{ end }} + {{ end }} + {{ end }} + {{ end }} + + {{ if not $group_match }} + {{ $show = false }} + {{ end }} + {{ end }} + + {{if $show}} + <div class="col-md-4"> + <h2> + <span>{{$apiDetail.Name}}</span> + <span class="badge badge-primary">{{$apiDetail.AuthType}}</span> + </h2> + <p>{{$apiDetail.LongDescription | markDown}}</p> + {{ if $apiDetail.Documentation }} + <a href="{{ $page.PortalRoot }}apis/{{$apiDetail.Documentation}}/documentation/" class="btn btn-info catalogue"> + <span class="glyphicon glyphicon-book" aria-hidden></span> View documentation + </a> + <br/> + {{ end }} + {{if eq $apiDetail.Version "" }} + {{if eq $apiDetail.IsKeyless false}} + <a href="{{if $page.PortalConfig.EnableMultiSelection}}{{ $page.PortalRoot }}member/apis/request?policy_id={{$apiDetail.APIID}}{{else}}{{ $page.PortalRoot }}member/apis/{{$apiDetail.APIID}}/request{{end}}" class="btn btn-success catalogue"> + <span class="glyphicon glyphicon-ok-sign" aria-hidden></span> Request an API key + </a> + {{ end }} + {{ else }} + {{if eq $apiDetail.IsKeyless false}} + <a href="{{if $page.PortalConfig.EnableMultiSelection}}{{ $page.PortalRoot }}member/policies/request?policy_id={{$apiDetail.PolicyID}}{{else}}{{ $page.PortalRoot }}member/policies/{{$apiDetail.PolicyID}}/request{{end}}" class="btn btn-success catalogue"> + <span class="glyphicon glyphicon-ok-sign" aria-hidden></span> Request an API key + </a> + {{ end }} + {{ end }} + </div> + {{ end }} + {{ end }} {{ end }} + </div> + {{ else }} + <div class="row"> + <p> + <em>It looks like there are no APIs in the Catalog.</em> + </p> + </div> + {{ end }} + </div> + </div> + {{ template "footer" .}} + </div> + {/* /container */} + {{ template "scripts" .}} +</body> +</html> +{{ end }} + + +``` +</Expandable> + +We're now going to overwrite the default catalogue.html template in the 'portal/templates' directory on the Tyk Dashboard instance with the custom one above. + +**NOTE**: After replacing or updating a template, the Dashboard must be restarted to apply the changes. + +Now the visibility of the "Internal API" is driven by the value of the "Group" field on the developer profile. + +### Multiple API subscriptions +If you have enabled "Enable multiple API subscriptions" option in the portal settings, you also need to modify `request_multi_key.html` template. +The main difference from the default template is two changes: +1. Get user data state at the start of template: `{{$profile := .UserData }}` +2. Before rendering `<li>` element, which renders list of APIs, we insert the following section: +```go-html-template expandable +{{ range $field, $value := $apiDetail.Fields }} + {{ $group_match := true }} + {{ if (eq $field "Group") }} + {{ $group_match = false }} + {{ range $dfield, $dvalue := $profile.Fields }} + {{ if eq $dfield "Group" }} + {{ if eq $dvalue $value }} + {{ $group_match = true }} + {{ end }} + {{ end }} + {{ end }} + {{ end }} + + {{ if not $group_match }} + {{ $match = true }} + {{ end }} +{{ end }} +``` + +<Expandable title={'Click to expand and see the full template'}> +```go-html-template expandable +{{ define "requestMultiKey" }} {{ template "header" .}} +{{$catalogue := .Catalogue}} +{{$catalogues := .Catalogues}} +{{$key := .Key}} +{{$modifyKey := false}} +{{$addKey := true}} +{{if .Key}}{{$modifyKey = true}}{{$addKey = false}}{{end}} +{{$profile := .UserData }} +<body> + <div> + <div class="page-header"> + <div class="page-header-container"> + <div class="title text-center"> + {{ if .Key }} + <h1>Modify API Key</h1> + {{ else }} + <h1>Request API Key</h1> + {{ end }} + </div> + </div> + </div> + <div + class="container content-wrapper key-request-flow-wrapper" + data-fixed-api="{{$catalogue}}" + data-key-req-fields-length="{{len .PortalConfig.KeyRequestFields}}" + > + <div class="row text-center"> + <div class="col-lg-12 text-center"> + {{ if .Error }} + <div class="alert alert-danger"> + {{.Error}} + </div> + {{ end }} + {{ if not .DenyRequest }} + <ol class="breadcrumb"> + <li class="cogs active "><a href="#choose-api" data-toggle="tab" aria-controls="choose-api" role="tab" title="Select API">Select API</a></li> + <li class="info disabled"><a href="#details" data-toggle="tab" aria-controls="details" role="tab" title="Enter details">Enter details</a></li> + <li class="check "><a href="#complete" data-toggle="tab" aria-controls="complete" role="tab" title="Complete">Final step</a></li> + </ol> + {{ end }} + </div> + </div> + {{ if not .DenyRequest }} + <form action="" method="POST" enctype="multipart/form-data"> + <div class="alert alert-danger no-items-error" style="display: none"> + You need to select at least an API for a key. + </div> + <div class="choose-api-wrapper auth-apis text-center"> + <div class="selectable-list-component"> + <h3 class="selected-items-title text-left">Selected APIs</h3> + {{if .Key}} + <p class="text-left"> List of APIs the key access</p> + {{else}} + <p class="text-left"> List of APIs the key will be generated for </p> + {{end}} + <div class="alert alert-info no-selected-api-msg">No selected APIs</div> + <ol class="selected-items items-list list-group"> + {{range $index, $cat := $catalogues}} + <li + class="list-group-item item active clickable-item" + data-auth-type="{{$cat.AuthType}}" + data-use-certificate="{{$cat.UseCertificate}}" + > + <div class="details-container"> + <input type="checkbox" name="apply_policies[]" checked value="{{$cat.PolicyID}}" /> + <button type="button" class="btn btn-success add-item-btn"><span class="fa fa-check"></span> + <br />Select API</button> + <button type="button" class="btn btn-danger remove-item-btn"><span class="fa fa-times"></span> + <br />Remove API</button> + <span class="item-title">{{$cat.Name}}</span> + <span class="badge badge-primary">{{$cat.AuthType}}</span> + </div> + </li> + {{end}} + </ol> + <h3 class="text-left">Available APIs to connect</h3> + <p class="text-left">List of APIs availble for key request. Once an API is selected the entire list is filtered by the selected APIs authentication type.</p> + <div class="alert alert-info no-available-apis-msg">No APIs available for selection</div> + <ol class="selectable-list items-list list-group"> + {{$authType := $catalogue.AuthType}} + {{ range $index, $apiDetail := .APIS}} + {{ if $apiDetail.Show }} + {{ if ne $apiDetail.AuthType "oauth"}} + + {{ $match := false }} + {{ range $cid, $cat := $catalogues }} + {{if eq $apiDetail.PolicyID $cat.PolicyID}} + {{ $match = true }} + {{end}} + {{end}} + + {{ range $field, $value := $apiDetail.Fields }} + {{ $group_match := true }} + {{ if (eq $field "Group") }} + {{ $group_match = false }} + {{ range $dfield, $dvalue := $profile.Fields }} + {{ if eq $dfield "Group" }} + {{ if eq $dvalue $value }} + {{ $group_match = true }} + {{ end }} + {{ end }} + {{ end }} + {{ end }} + + {{ if not $group_match }} + {{ $match = true }} + {{ end }} + {{ end }} + + {{ if and (ne $match true) (or $addKey (eq $apiDetail.AuthType $authType)) }} + <li + class="list-group-item item clickable-item" + data-id="{{$apiDetail.PolicyID}}" + data-auth-type="{{$apiDetail.AuthType}}" + data-use-certificate="{{$apiDetail.UseCertificate}}" + style="{{if ne $apiDetail.AuthType $authType }}display:none{{end}}" + > + <div class="details-container"> + <input type="checkbox" name="apply_policies[]" value="{{$apiDetail.PolicyID}}" /> + <button type="button" class="btn btn-success add-item-btn"><span class="fa fa-check"></span> + <br />Select API</button> + <button type="button" class="btn btn-danger remove-item-btn"><span class="fa fa-times"></span> + <br />Remove API</button> + <span class="item-title">{{$apiDetail.Name}}</span> + <span class="badge badge-primary">{{$apiDetail.AuthType}}</span> + </div> + </li> + {{end}} + {{end}} + {{end}} + {{end}} + </ol> + </div> + <ul class="list-inline"> + <li class="pull-left"> + <a href="{{ .PortalRoot }}apis/" class="btn btn-success outline">Back to Api Catalogue</a> + </li> + <li class="pull-right"> + <button type="button" class="btn btn-success next-auth-step" style="display: none">Save and continue</button> + <button type="submit" class="btn btn-success request-key-btn req-key-btn-first-step" style="display: none">Request key</button> + </li> + </ul> + </div> + <div class="request-key-form"> + <input type="hidden" name="csrf_token" value="{{ .Token }}"> + {{if gt (len .PortalConfig.KeyRequestFields) 0 }} + <h3 class="text-left">Key request form</h3> + {{ range $fieldname := .PortalConfig.KeyRequestFields }} + <div class="form-group"> + <label for="{{$fieldname}}">{{$fieldname}}</label> + <input type="text" class="form-control" id="{{$fieldname}}" name="{{$fieldname}}" placeholder=""> + </div> + {{ end }} + {{ end }} + {{ if $catalogue }} + <div class="jwt-form" style="display: none"> + <h3 class="text-left">JWT secret</h3> + <div class="form-group"> + <p> + This API is configured to validate against JSON Web Tokens, in order for this to work, we will need to know your HMAC Secret OR a valid RSA public key, please enter this below as part of your key request. + </p> + <label>Signature validation key:</label> + <textarea rows="10" class="form-control" id="jwt_secret" name="jwt_secret" placeholder="" value="" style="font-family: monospace;"></textarea> + </div> + </div> + <div class="use-certificate-form" style="display: none"> + <h3 class="text-left">Certificate</h3> + <div class="form-group"> + <label>Upload your public client certificate in PEM format:</label> + <textarea rows="10" class="form-control" id="certificate_upload" name="certificate" placeholder="" value="" style="font-family: monospace;"></textarea> + </div> + </div> + {{ end }} + <ul class="list-inline"> + <li class="pull-left"> + <button type="button" class="btn btn-success prev-auth-step outline">Back to Apis list</button> + </li> + <li class="pull-right"> + {{if $addKey}} + <button type="submit" class="btn btn-success request-key-btn">Request key</button> + {{else}} + <button type="submit" class="btn btn-success request-key-btn">Request key changes</button> + {{end}} + </li> + </ul> + </div> + </form> + {{ end }} + </div> + </div> + {{ template "navigation" . }} + {{ template "footer" .}} + {/* /container */} + {{ template "scripts" .}} +</body> +</html> +{{ end }} +``` +</Expandable> + +#### Developer Logged In, Group field set to internal (Internal API is visible) +<img src="/img/dashboard/portal-management/dev_logged_in_internal.jpg" alt="dev_logged_in_internal" width="700" /> + +#### Developer Logged In, Group field not set or set so group other than internal (Internal API not visible) +<img src="/img/dashboard/portal-management/dev_logged_in_external.jpg" alt="dev_logged_in_external" width="700" /> + +#### No User Logged In (Internal API not visible) +<img src="/img/dashboard/portal-management/no_user_logged_in.jpg" alt="no_user_logged_in" width="700" /> diff --git a/tyk-developer-portal/tyk-enterprise-developer-portal/api-consumer-portal/access-api-product.mdx b/tyk-developer-portal/tyk-enterprise-developer-portal/api-consumer-portal/access-api-product.mdx new file mode 100644 index 00000000..f2d9af2e --- /dev/null +++ b/tyk-developer-portal/tyk-enterprise-developer-portal/api-consumer-portal/access-api-product.mdx @@ -0,0 +1,50 @@ +--- +title: "Access an API Product" +'og:description': "How to access an api product in Tyk developer portal" +tags: ['Developer Portal', 'Tyk', 'API Consumer'] +sidebarTitle: "Access an API Product" +--- + +## Introduction + +This section explains how to access API products as a registered portal developer + +**Prerequisites** + +You need to have successfully registered for a portal. + +**Step by step instructions** + +1. Login to the external developer portal +2. Go to **Catalogs** to view the available API products + + + + <Note> + Using the filter at the top, you can filter on the different created catalogs including different API Products, e.g. a custom catalog that only you and a specific team can access. + </Note> + + +3. When selecting on a product, you can click **More info** to access the product overview page. +4. On the product overview page, you can view documentation for each API included in the API product. You can also view information about which catalog the API product is part of. Each catalog may have different plans available so you need to select a catalog based on which plan you want to access. +5. Click **Add to cart**. + + + + <Note> + You can add multiple products to the cart. You can receive one set of access credentials for multiple products as long as they are part of the same cart. If you are adding two products to the cart, and they are part of different catalogs, e.g. Private and Public, you will need to go through two request flows, and you will get two different sets of credentials for each API Product. + </Note> + + +6. Add the details needed and select a subscription plan for your API Product(s) chosen. +7. Create a new app or add to an existing one. If you already have an existing app created you can access it via the drop down or select **Create a new app** to add the credentials to an existing app. +8. Click **Request access**. +9. Navigate to My apps and view the app you created. Depending if the plan requires manual approval by an admin or not, you will either see that the request is pending or you can see the approved access credentials immediately you can start using them. + + + + <Note> + When sending a query, make sure to use the Base URL outlined in the overview of the API Product. + </Note> + + diff --git a/tyk-developer-portal/tyk-enterprise-developer-portal/api-consumer-portal/reset-password.mdx b/tyk-developer-portal/tyk-enterprise-developer-portal/api-consumer-portal/reset-password.mdx new file mode 100644 index 00000000..176cb4f5 --- /dev/null +++ b/tyk-developer-portal/tyk-enterprise-developer-portal/api-consumer-portal/reset-password.mdx @@ -0,0 +1,33 @@ +--- +title: "Reset Password" +'og:description': "How to reset password in Tyk developer portal" +tags: ['Developer Portal', 'Tyk', 'API Consumer'] +sidebarTitle: "Reset Password" +--- + +This page goes through the reset password routine. If you have forgotten your password you can easily reset it via the portal. + +1. Navigate to the live portal and click **Login** and you’ll get to the login screen. + + <img src="/img/dashboard/portal-management/enterprise-portal/login-portal-forgot.png" alt="Portal login screen" /> + +2. At the login screen click **Forgot Password?** link and you’ll be redirected to the reset password form. + + <img src="/img/dashboard/portal-management/enterprise-portal/forgot-password.png" alt="Forgot Password email address" /> + +3. Enter your email address, and click **Reset** and you’ll see this message. + + <img src="/img/dashboard/portal-management/enterprise-portal/forgot-password-email.png" alt="Forgot Password email sent" /> + +4. Check your email and you should have received an email that contains a link the following format: +`https://the-portal-domain.com/auth/reset/code?token=<token-id>` + +5. Click on the link and you will be taken to the reset password screen. +6. Enter your new password and click **Reset**. + + <img src="/img/dashboard/portal-management/enterprise-portal/reset-password-request.png" alt="Enter and confirm new password" /> + +7. Click **Login again** to go to the Login screen. + + <img src="/img/dashboard/portal-management/enterprise-portal/reset-done.png" alt="Enter and confirm new password" /> + diff --git a/tyk-developer-portal/tyk-portal-classic.mdx b/tyk-developer-portal/tyk-portal-classic.mdx new file mode 100644 index 00000000..1a64ff14 --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic.mdx @@ -0,0 +1,38 @@ +--- +title: "Tyk Classic Developer Portal" +tags: [''] +order: 1 +noindex: True +sidebarTitle: "Overview" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +The Tyk Classic Developer Portal is a small CMS-like system that enables you to expose a facade of your APIs and then allow third-party developers to register and use your APIs. + +The Tyk Classic Developer Portal enables you to: + +* Host pages written in Markdown to describe your API and product. +* Use Bootstrap based templates to completely customize the look and feel. +* Host your API documentation in Swagger/OpenAPI or API Blueprint for developers to use. In the case of Swagger/OpenAPI, you can either paste your Swagger content (JSON or YAML) into the code editor, or via a link to a public Swagger hosted URL, which can then be rendered by using [Swagger UI](https://swagger.io/tools/swagger-ui/). +* Unlike other platforms, Tyk will not auto-publish your APIs to the Portal, instead they are presented as a facade, you choose what APIs and what Policies to expose to the Portal. +* Fully control the flow of the developer sign-up and enrollment. + +The Tyk Classic Developer Portal is suitable for primary use cases, however if you want a more feature-rich experience then you should consider the [Tyk Enterprise Developer Portal](/portal/overview/intro). This provides audience management, full UI customization including custom JS insertion and custom page templates, forums and blog features, grouping APIs into API Products, and other enterprise-grade features. + + +<Note> +Support for API Blueprint is being deprecated. See [Importing APIs](/api-management/gateway-config-managing-classic#api-blueprint-is-being-deprecated) for more details. +</Note> + + +The Tyk Classic Developer Portal is available to Tyk Self-Managed and Tyk Cloud users. + +For examples of how clients have used our portal, visit: + +- [https://developer.hotelbeds.com/](https://developer.hotelbeds.com/) +- [https://developer.ft.com/portal](https://developer.ft.com/portal) +- [https://developer.geops.io/](https://developer.geops.io/) +- [https://opentransportdata.swiss/en/](https://opentransportdata.swiss/en/) diff --git a/tyk-developer-portal/tyk-portal-classic/curity-dcr.mdx b/tyk-developer-portal/tyk-portal-classic/curity-dcr.mdx new file mode 100644 index 00000000..c66484da --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/curity-dcr.mdx @@ -0,0 +1,329 @@ +--- +title: "Step by step guide using Curity" +order: 4 +noindex: True +sidebarTitle: "Step by step guide using Curity" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +This guide describes how to integrate [The Curity Identity Server](https://curity.io/) with the Tyk Developer Portal using [OpenID Connect Dynamic Client Registration protocol](https://tools.ietf.org/html/rfc7591). + +Use case outline: + +1. A developer registers an account with the Tyk Developer Portal and uses the portal to create a new client. + +2. Tyk sends a <Tooltip tip="Dynamic Client Registration">DCR</Tooltip> request to the <Tooltip tip="Identity Provider">IDP</Tooltip>, in this case The Curity Identity Server. The IDP replies with the Client ID and Secret. + +3. Using the provided Client ID and Secret, the developer (or an application) can initiate an Authorization flow to obtain an Access Token from The Curity Identity Server. + +4. The developer (or the application) can then use the Access Token when calling an API exposed by Tyk. In the case when a JWT is used to protect the API, Tyk validates the token using the <Tooltip tip="JSON Web Key Sets">JWKS</Tooltip> provided by The Curity Identity Server. The API can also be protected using the [Split Token Approach](https://github.com/sedkis/split-token-tyk). + +### Requirements + +- An installation of The Curity Identity Server. Follow the [Getting Started Guide](https://curity.io/resources/getting-started/) if an installation is not already available. +- A [Tyk Self-Managed installation](/tyk-self-managed/install) (Gateway + Dashboard). + +### Enable DCR in The Curity Identity Server + +By default, DCR is not enabled in The Curity Identity Server. In the Admin UI, go to **Profiles** → **Token Service** → **General** → **Dynamic Registration**. Set both **Enable DCR** and **Non-templatized** to enabled and set the **Authentication Method** to `no-authentication`. + +<img src="/img/dcr/curity/1-curity-admin-ui-dcr.png" alt="Enable DCR" /> + +Navigate to **Profiles** → **Token Service** → **Endpoints** and locate the path of the DCR endpoint. The path is needed later when configuring DCR in Tyk. + +<img src="/img/dcr/curity/2-curity-dcr-endpoint.png" alt="DCR Endpoint" /> + + +<Note> +**Commit the changes** + +Remember to **Commit** the changes by going to **Changes** → **Commit**. +</Note> + + +### Setting up Tyk + +First check `tyk_analytics.conf` and make sure that a proper `oauth_redirect_uri_separator` parameter is set. This sets the character that separates multiple redirect uri's to `;`. + +```json + "oauth_redirect_uri_separator": ";", +``` + +If a change is needed in `tyk_analytics.conf`, remember to restart the service to apply the change. + +#### Create DCR proxy API + +The Curity Identity Server requires a token with a `dcr` scope in order to authenticate the DCR endpoint. A workaround is to configure the DCR endpoint to use no-authentication. A proxy API can be configured in such a way that Tyk will proxy the DCR request to the Curity Identity Server and a static token used to authenticate the DCR proxy API. + + +<Warning> +**Use in secure environments only** + +Make sure that the communication between Tyk and the Curity Identity Server is appropriately secured. The DCR endpoint on the Curity Identity Server if set to `no-authentication` should only be accessible by Tyk. +</Warning> + + +In the Tyk Dashboard, navigate to **System Management** → **APIs**. Create a new API and give it the name, `dcr`. Make sure the **API Listen Path** is set to `/dcr/`. +Set the Target URL to the DCR endpoint of the Curity Identity Server (Ex. https://idsvr.example.com/token-service/oauth-registration). + +In the **Authentication** section, set **Authentication Mode** to `Authentication Token`. + +#### Create an example API + +In the Tyk Dashboard, navigate to **System Management** → **APIs**. Create a new API and give it the name, ex. `curity-api`: + +<img src="/img/dcr/curity/3-curity-create-api.png" alt="Create API" /> + +Click **Configure API** and scroll down to the **Authentication** section. Two options to protect an API are outlined below. Choose the options that best fits your needs. + +1. **Split Token Approach** - This would be the preferred option and is fully detailed in the [split-token-tyk](https://github.com/sedkis/split-token-tyk) GitHub repository with examples. The basics of this approach is that Tyk proxies the IDP's token endpoint and splits the token to only return the signature of the JWT instead of the complete JWT. The client calling the API will use the signature as an opaque token in the Authorize header. Tyk will then look up the complete JWT using the signature as the key and then add the complete JWT to the Authorization header in the request to the upstream API. + +2. **JWT Approach** - This approach means that the IDP issues a JWT to the client (using the dynamically registered client) and the complete JWT is sent in the Authorization header in the API request to Tyk. Although this is absolutely a viable approach there are some potential risks with issuing JWTs to public clients since they could contain Personal Identifiable Information (PII). + +<Tabs> +<Tab title="Split Token Approach"> + +The configuration of the Split Token Approach is outlined in the readme in the [GitHub repository](https://github.com/sedkis/split-token-tyk). Make sure to follow the instructions to configure Tyk to handle the Split Token Approach before continuing. + +For the Split Token Approach, configure the Authentication as outlined in the below screenshot for an example API. + +<img src="/img/dcr/curity/4-split-curity-configure-api.png" alt="Configure Split Token API" /> + +#### Create a facade API + +The Tyk Gateway needs to make use of a facade API in order for the API to be published to the Developer Portal when protected with the Split Token Approach. + +In the Tyk Dashboard, navigate to **System Management** → **APIs**. Create a new API and give it the name `facade-oauth-registration`. +Set the Target URL to `http://httpbin.org`. + + +<Warning> +**NOTE** + +The `path` and `Target URL` for this API doesn't matter and will never be used. +</Warning> + + +In the **Authentication** section, set **Authentication Mode** to `JSON Web Token (JWT)`. +<img src="/img/dcr/curity/5-split-facade-curity-configure-api.png.png" alt="Configure Facade API" /> + + +<Note> +**Obtaining the JWKS URI** + +The JWKS URI can be obtained via the `.well-known/openid-configuration` endpoint as it's a required value. The below cURL command can get the `"jwks_uri"` value directly. + +```shell +curl -sS https://idsvr.example.com/oauth/v2/oauth-anonymous/.well-known/openid-configuration | grep -o '"jwks_uri":"[^"]*' | cut -d'"' -f4 +``` +</Note> + + +#### Create and assign a policy + +The Facade API needs a policy in order to be published to the Developer Portal. + +Switch to **System Management** → **Policies**. Click **Add Policy** and select the Facade API created previously (`facade-oauth-registration`) from the list. Then switch to the **Configurations** tab. Name the policy `facade-policy`. Select an expiry and click `Create Policy`. + +Navigate back to **System Management** → **APIs**, click `facade-oauth-registration`, scroll down to the **Authentication** section and select the newly created policy in the **Default Policy** setting. Click `Update` to save the changes. + +#### Create a Key for the DCR proxy + +Navigate to **System Management** → **Keys**, click `Add Key`. Switch to the `Choose API` tab. Select the previously created `DCR` API. Under `2. Configurations` give the key an alias and set an expiry. Then click `Create Key`. + +<img src="/img/dcr/curity/6-split-dcr-key-curity.png" alt="DCR API Key" /> + + +<Warning> +**Important** + +Take note of the `Key Hash` and `Key ID` as they will be needed later. +</Warning> + + +#### Publish the API to the Developer Portal + +The API and the Facade API are now configured and can used to publish the API to the Tyk Developer Portal. Navigate to **Portal Management** → **Catalog**, then click **Add New API**. Give it a public name, ex. `OAuth Facade API` and select the `facade-policy`. + +<img src="/img/dcr/curity/7-split-curity-publish-api.png" alt="Publish API" /> + +Navigate to the **Settings** tab and check the box `Override global settings`. Then scroll down to the **Dynamic Client Registration for portal APIs** section and toggle the switch to enable. Configure as pictured below: + +<img src="/img/dcr/curity/8-split-curity-configure-dcr.png" alt="Configure DCR" /> + +Config parameter | Description | Value +----------------------------------|-------------------------------------|----- +Providers | The IDP vendor | `Other` +Grant Types | What grant types the DCR client will support | `Client Credentials` and/or `Authorization Code` +Token Endpoint Auth Method | How the client authenticates against the IDPs token endpoint | `Client Secret - Post` +Response Types | OAuth 2.0 response types that will be used by the client. | `Token` and/or `Authorization Code` +Identity Provider Host | The Base URL of the IDP | Ex. `https://idsvr.example.com` +Client Registration Endpoint | The proxy DCR endpoint created previously | Ex. `https://tyk-gateway/dcr/` +Initial Registration Access Token | Token to authenticate the DCR endpoint | Add the DCR `Key ID` created in previous step + +### Testing the flow + +Tyk and The Curity Identity Server should now be configured and the flow to register an OAuth client using the Tyk Developer Portal can be tested. + +#### Create an OAuth Client +Start by registering a developer by navigating to **Portal Management** → **Developers** and add a developer. Then open the Tyk Developer Portal (ex. `http://<host>:3000/portal`) and open the **OAuth clients** page. Start the wizard by clicking **Create first Oauth Client**. + +<img src="/img/dcr/curity/9-split-curity-create-oauth-client-wizard.png" alt="Create OAuth client wizard" /> + +Select the API previously published named **OAuth Facade API** and then click **Save and continue**. + +Enter a Client name and add at least one redirect URL(separate with `;`), then click **Create**. + +<img src="/img/dcr/curity/10-split-curity-application-details.png" alt="Application details" /> + + +<Note> +**OAuth.Tools** + +If using [OAuth.tools](https://oauth.tools/) to obtain a token, make sure to add the appropriate redirect URL. Ex: + +``` +http://localhost:7777;https://oauth.tools/callback/code;app://oauth.tools/callback/code +``` + +The web-based version of [OAuth.tools](https://oauth.tools/) using the Code Flow would require `https://oauth.tools/callback/code` and the [App version of OAuth.tools](https://developer.curity.io/oauth-tools) requires `app://oauth.tools/callback/code`. +</Note> + + +Tyk will make a call to the DCR proxy endpoint that will in turn call The Curity Identity Server and its DCR endpoint to register a dynamic client. The details of the dynamically registered client will be displayed. + +<img src="/img/dcr/curity/11-split-curity-dcr-client-details.png" alt="DCR client details" /> + +#### Obtain a token using DCR client + +[OAuth.tools](https://oauth.tools/) can be used to obtain an access token from The Curity Identity Server using the DCR information. This call needs to be made to the token endpoint configured on Tyk to handle the [Split Token Approach](https://github.com/sedkis/split-token-tyk). + +Start an External API Flow. Copy the **Client ID** and the **Client Secret** to the appropriate fields in [OAuth.tools](https://oauth.tools/). Run the flow to obtain a token. + +<img src="/img/dcr/curity/12-split-curity-oauth-tools.png" alt="OAuth.tools" /> + + +<Note> +**Split Token** + +Note that the token returned is the signature of a JWT as this is executing the Split Token Approach. +</Note> + + +#### Use token in request to API +The token can now be used in an **External API** flow in [OAuth.tools](https://oauth.tools/) to call the API that Tyk is proxying. Tyk will use the signature passed in the Authorization header and lookup the complete JWT in its cache. The complete JWT is then added to the upstream Authorization header as shown in the echo response from Httpbin.org in the below example. + +<img src="/img/dcr/curity/13-split-curity-external-api-flow.png" alt="Call API" /> + +</Tab> +<Tab title="JWT Approach"> + +For the JWT Approach, configure the Authentication as outlined in the below screenshot. +<img src="/img/dcr/curity/4-jwt-curity-configure-api.png" alt="Configure API" /> + + +<Note> +**Obtaining the JWKS URI** + +The JWKS URI can be obtained via the `.well-known/openid-configuration` endpoint as it's a required value. The below cURL command can get the `"jwks_uri"` value directly. + +```shell +curl -sS https://idsvr.example.com/oauth/v2/oauth-anonymous/.well-known/openid-configuration | grep -o '"jwks_uri":"[^"]*' | cut -d'"' -f4 +``` +</Note> + + +Click **Save** to save the API. + +#### Create and assign a policy + +Switch to **System Management** → **Policies**. Click **Add Policy** and select the API created previously (`curity-api`) from the list. Then switch to the **Configurations** tab. Name the policy `Curity Policy`. Select an expiry and click `Create Policy`. + +Navigate to **System Management** → **APIs**, click `curity-api`, scroll down to the **Authentication** section and select the newly created policy in the **Default Policy** setting. + +<img src="/img/dcr/curity/5-jwt-curity-configure-policy.png" alt="Configure policy" /> + +#### Create a Key for the DCR proxy +Navigate to **System Management** → **Keys**, click `Add Key`. Switch to the `Choose API` tab. Select the previously created `DCR` API. Under `2. Configurations` give the key an alias and set an expiry. Then click `Create Key`. + +<img src="/img/dcr/curity/6-jwt-dcr-key-curity.png" alt="DCR API Key" /> + + +<Warning> +**Important** + +Take note of the `Key Hash` and `Key ID` as they will be needed later. +</Warning> + + +#### Publish the API to the Developer Portal + +The API is now configured and can be published to the Tyk Developer Portal. Navigate to **Portal Management** → **Catalog**, then click **Add New API**. Give it a public name, ex. `Curity Demo API` and select the `Curity Policy`. + +<img src="/img/dcr/curity/7-jwt-curity-publish-api.png" alt="Publish API" /> + +Navigate to the **Settings** tab and check the box `Override global settings`. Then scroll down to the **Dynamic Client Registration for portal APIs** section and toggle the switch to enable. Configure as pictured below: + +<img src="/img/dcr/curity/8-jwt-curity-configure-dcr.png" alt="Configure DCR" /> + +Config parameter | Description | Value +----------------------------------|-------------------------------------|----- +Providers | The IDP vendor | `Other` +Grant Types | What grant types the DCR client will support | `Client Credentials` and/or `Authorization Code` +Token Endpoint Auth Method | How the client authenticates against the IDPs token endpoint | `Client Secret - Post` +Response Types | OAuth 2.0 response types that will be used by the client. | `Token` and/or `Authorization Code` +Identity Provider Host | The Base URL of the IDP | Ex. `https://idsvr.example.com` +Client Registration Endpoint | The DCR endpoint created previously | Ex. `https://tyk-gateway/dcr/` +Initial Registration Access Token | Token to authenticate the DCR endpoint | Add the key created in previous step + +### Testing the flow + +Tyk and The Curity Identity Server should now be configured and the flow to register an OAuth client using the Tyk Developer Portal can be tested. + +#### Create an OAuth Client +Start by registering a developer by navigating to **Portal Management** → **Developers** and add a developer. Then open the Tyk Developer Portal (ex. `http://<host>:3000/portal`) and open the **OAuth clients** page. Start the wizard by clicking **Create first Oauth Client**. + +<img src="/img/dcr/curity/9-jwt-curity-create-oauth-client-wizard.png" alt="Create OAuth client wizard" /> + +Select the API previously published named **Curity API** and then click **Save and continue**. + +Enter a Client name and add at least one redirect URL, then click **Create**. + +<img src="/img/dcr/curity/10-jwt-curity-application-details.png" alt="Application details" /> + + +<Note> +**OAuth.Tools** + +If using [OAuth.tools](https://oauth.tools/) to obtain a token, make sure to add the appropriate redirect URL. Ex: + +``` +http://localhost:7777;https://oauth.tools/callback/code;app://oauth.tools/callback/code +``` + +The web-based version of [OAuth.tools](https://oauth.tools/) using the Code Flow would require `https://oauth.tools/callback/code` and the [App version of OAuth.tools](https://developer.curity.io/oauth-tools) requires `app://oauth.tools/callback/code`. +</Note> + + +Tyk will make a call to The Curity Identity Server and the DCR endpoint to register a dynamic client. The details of the dynamically registered client will be displayed. + +<img src="/img/dcr/curity/11-jwt-curity-dcr-client-details.png" alt="DCR client details" /> + +#### Obtain a token using DCR client + +[OAuth.tools](https://oauth.tools/) can be used to obtain an access token from The Curity Identity Server using the DCR information. + +Start a Code or Client Credentials Flow. Copy the **Client ID** and the **Client Secret** to the appropriate fields in [OAuth.tools](https://oauth.tools/). Run the flow to obtain a token (JWT). + +<img src="/img/dcr/curity/12-jwt-curity-oauth-tools.png" alt="OAuth.tools" /> + +#### Use access token in request to API +The token can now be used in an **External API** flow in [OAuth.tools](https://oauth.tools/) to call the API that Tyk is proxying. Tyk will validate the JWT and allow the call to the upstream API (httpbin.org in this example). The response from the API is displayed in the right panel in [OAuth.tools](https://oauth.tools/). Httpbin.org echoes back what it receives in the request from Tyk. Note that the complete JWT is forwarded. + +<img src="/img/dcr/curity/13-jwt-curity-external-api-flow.png" alt="Call API" /> + +</Tab> +</Tabs> diff --git a/tyk-developer-portal/tyk-portal-classic/customise/changing-the-navigation.mdx b/tyk-developer-portal/tyk-portal-classic/customise/changing-the-navigation.mdx new file mode 100644 index 00000000..311d8a50 --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/customise/changing-the-navigation.mdx @@ -0,0 +1,55 @@ +--- +title: "Customize the Portal Menus" +order: 1 +noindex: True +sidebarTitle: "Customise the Portal Menus" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +The Portal supports a data structure to hold rudimentary menus, all pages have access to all menus, and can be accessed using the `.Menus.MenuName` field tag. They are arrays that consist of slugs and names, an implementation example would be: + +```{.copyWrapper} expandable +<ul class="nav navbar-nav"> + {{ range $index, $menuItem := .Menus.Main}} + <li><a href="/portal/{{$menuItem.URL}}">{{$menuItem.Title}}</a></li> + {{ end }} + <li><a href="/portal/apis/">API Catalog</a></li> + {{ if not .PortalConfig.DisableSignup }} + {{ if not .UserData }} + <li><a href="/portal/register/">Register</a></li> + {{ end }} + {{ end }} + {{ if not .PortalConfig.DisableLogin }} + {{ if not .UserData }} + <li><a href="/portal/login/">Log in</a></li> + {{ end }} + {{ end }} +</ul> +``` + +In the snippet above we can also see a set of settings fields, in order to react to the configuration of the Portal, the core Portal config object is exposed to the template and can be used to change how the template is rendered. + +### Customizing the menu with the Dashboard + +The Dashboard has a simple menu editor, you can create the above data structures from the **Portal Management > Menus** option + +<img src="/img/dashboard/portal-management/portal_nav_menus_2.5.png" alt="Menus nav" /> + +The Portal will come with two menus built in, "Main" and "Secondary", the "Main" menu will appear in the primary navigation (top nav) of the templates supplied with the Portal, while the secondary will show on the right hand side of the Default Page Templates. + +To add a new menu item to the Dashboard main navigation, select the "Main" Menu from the drop down: + +<img src="/img/dashboard/portal-management/portal_menus_2.5.png" alt="Manage portal menu" /> + +Click **ADD**: + +<img src="/img/dashboard/portal-management/add_portal_menu_2.5.png" alt="Manage portal menu" /> + +Once added, you can select it from the drop down and add the title and URL fields that will control where the menu item will direct the user. + +<img src="/img/dashboard/portal-management/portal_menu_dropdown_2.5.png" alt="Edit portal menu" /> + +The menu item, once saved, will appear in your Portal navigation instantly. diff --git a/tyk-developer-portal/tyk-portal-classic/customise/custom-developer-portal.mdx b/tyk-developer-portal/tyk-portal-classic/customise/custom-developer-portal.mdx new file mode 100644 index 00000000..61110de0 --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/customise/custom-developer-portal.mdx @@ -0,0 +1,254 @@ +--- +title: "Creating a Custom Developer Portal" +order: 5 +noindex: True +sidebarTitle: "Creating a Custom Developer Portal" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; +import PortalDeveloperAnalytics from '/snippets/portal-developer-analytics.mdx'; + +<LegacyClassicPortalApi/> + + +<Note> +This functionality is available from v2.3.8 +</Note> + + + +## Why Build a Custom Developer Portal? + +The Tyk Dashboard includes portal functionality by default, but in some cases it is required to use custom logic or, for example, embed the portal into an existing platform. Thankfully Tyk is flexible enough to provide an easy way of integrating the portal to any platform and language using a few API calls. + +The source code is available from the following GitHub repo - [https://github.com/TykTechnologies/tyk-custom-portal](https://github.com/TykTechnologies/tyk-custom-portal) + +A video covering the process of building a custom portal is available to view here: + +<iframe width="560" height="315" src="https://www.youtube.com/embed/elrAEp1EZ_s" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + +## Building Blocks + +Before starting work on implementing a custom developer portal, let's understand the basic building blocks. + +### Obtaining a Dashboard API Key + +To run queries against the Tyk Dashboard API you need to obtain credentials from the **Users** screen. + +1. Select **Users** from the **System Management** section. +2. In the **Users** list, click **Edit** for your user. +3. The API key is the **Tyk Dashboard API Access Credentials**, copy this somewhere you can refer to it. + +### API Key Location +Let's save it to the environment variable to simplify code examples in this guide. All the commands should be run in your terminal. + + +<Note> +Don't forget to replace TYK_API_KEY=1efdefd6c93046bc4102f7bf77f17f4e with your own value +</Note> + + + +### Creating a Developer + +#### Request + +```{.copyWrapper} expandable +curl https://admin.cloud.tyk.io/api/portal/developers \ + -X POST \ + -H "authorization: $TYK_API_KEY" \ + -d \ +'{ + "email": "apidev@example.com", + "password": "supersecret", + "fields": { + "Name": "John Snow" + } +}' +``` + +#### Response + +``` +{"Status":"OK","Message":"598d4a33ac42130001c1257c","Meta":null} +``` + +Where `Message` contains the developer internal ID., You do not have to remember it, since you can find a developer by his email, using the API. + +### Developer by Email + +#### Request + +```{.copyWrapper} +curl https://admin.cloud.tyk.io/api/portal/developers/email/apidev%40example.com \ + -X GET \ + -H "authorization: $TYK_API_KEY" +``` + +#### Response + +``` +{"id":"598d4a33ac42130001c1257c","email":"apidev@example.com","date_created":"2017-08-11T06:09:55.654Z","inactive":false,"org_id":"59368a6b5eeba30001786baf","api_keys":{},"subscriptions":{},"fields":{"Name":"John Snow","source":"google search"},"nonce":"","sso_key":""} +``` + +### Developer Validation + +By default, the Tyk Developer portal automatically accepts all developer registrations, if they are not completely disabled in the portal configuration. + + +<Note> +Do not confuse developer registration with key access, if they are registered to the portal, it does not mean they automatically have access to your APIs. +</Note> + + +If you want to allow developer registration but add an additional layer of verification, you can use the developer `inactive` attribute to handle it. By default, it is `false`, and you can set it to `true` if additional verification is needed. To make this work, you need to add additional logic to your custom portal code. + +### Updating a Developer. Example: Adding Custom Fields + +Let's say we want to add new custom "field" to the developer, to store some internal meta information. Tyk so far does not support PATCH semantics, e.g. you cannot update only single field, you need to provide the full modified record. + +Lets created an updated developer record, based on the example response provided in Developer by Email section. Let's add new "traffic_source" custom field. + +#### Request + +```{.copyWrapper} expandable +curl https://admin.cloud.tyk.io/api/portal/developers/598d4a33ac42130001c1257c \ + -X PUT \ + -H "authorization: $TYK_API_KEY" \ + -d \ + '{ + "id":"598d4a33ac42130001c1257c", + "email":"apidev@example.com", + "date_created":"2017-08-11T06:09:55.654Z", + "inactive":false, + "org_id":"59368a6b5eeba30001786baf", + "api_keys":{}, + "subscriptions":{}, + "fields":{ + "Name":"John Snow", + "traffic_source":"google search" + } + }' +``` + +#### Response + +``` +{"Status":"OK","Message":"Data updated","Meta":null} +``` + +Note that all non-empty custom fields are shown in the Tyk Dashboard Developer view. All the keys created for this developer inherit his custom fields, if they are specified in the Portal settings **Signup fields** list. + + +### Login Workflow: Checking User Credentials + +If you need to implement own login workflow, you need be able to validate user password. + +#### Request + +```{.copyWrapper} expandable + curl https://admin.cloud.tyk.io/api/portal/developers/verify_credentials \ + -X POST \ + -H "authorization: $TYK_API_KEY" \ + -d \ + '{ + "username": "<developer-email>", + "password": "<developer-password>" + }' +``` + +If the user credentials are verified, the HTTP response code will be 200, otherwise credentials do match and a 401 error will be returned. + +### Listing Available APIs + +Inside the admin dashboard in the portal menu, you can define **catalogs** with the list of APIs available to developers. Each defined API is identified by its policy id field. + +#### Request + +```{.copyWrapper} +curl https://admin.cloud.tyk.io/api/portal/catalogue \ +-X GET \ +-H "authorization: $TYK_API_KEY" +``` + +#### Response + +``` expandable +{ + "id":"5940e3d29ba5330001647b21", + "org_id":"59368a6b5eeba30001786baf", + "apis":[ + { + "name":"asdasd", + "short_description":"", + "long_description":"", + "show":true, + "api_id":"", + "policy_id":"59527b8c375f1e000146556b", + "documentation":"", + "version":"v2" + }, + { + "name":"asdaszczxczx", + "short_description":"", + "long_description":"", + "show":true, + "api_id":"", + "policy_id":"5944267f8b8e5500013082a5", + "documentation":"", + "version":"v2" + } + ] +} +``` + +### Issuing Keys + +To generate a key for the developer, first he should send a request to the administrator of the API, and if needed provide details of the key usage. By default, all keys requests are approved automatically, and the user immediately receives his API key, but you can turn on the **Review all key requests before approving them** option in the portal settings, to add additional verification step, and approve all keys manually. Once a key is issued, the user receives an email with the key details. + +#### Request + +```{.copyWrapper} expandable +curl https://admin.cloud.tyk.io/api/portal/requests \ +-X POST \ +-H "authorization: $TYK_API_KEY" \ +-d \ +'{ + "by_user":"<developer-id>", + "for_plan": "<api-policy-id>" + "version": "v2", + "fields":{ + "custom_field":"value", + } +}' +``` + +#### Response + +``` + {"Status":"OK","Message":"Data updated","Meta":null} +``` + +### Checking User Subscriptions and Keys + +The Developer object contains the `subscriptions` field with information about user API keys, and associtated API's (policies). For Example: + +#### Request + +```{.copyWrapper} +"subscriptions":{"<policy-id-1>": "<api-key-1>", "<policy-id-2>": "<api-key-2>"}, +``` + +<PortalDeveloperAnalytics/> + + +<h2 id="building-portal"><a name="building-portal"></a> Building a Portal</h2> +This guide includes the implementation of a full featured developer portal written in Ruby in just 250 lines of code. This portal implementation does not utilize any database and uses our Tyk Dashboard API to store and fetch all the data. + +To run it, you need to have Ruby 2.3+ (latest version). Older versions may work but have not been tested. + +First, you need to install dependencies by running `gem install sinatra excon --no-ri` in your terminal. + +Then run it like this: `TYK_PORTAL_PORT=8080 TYK_API_KEY=<your-api-key-here> ruby portal.rb` + +You can also specify the `TYK_DASHBOARD_URL` if you are trying this portal with an Self-Managed installation. By default, it is configured to work with Cloud or Multi-Cloud. diff --git a/tyk-developer-portal/tyk-portal-classic/customise/customise-documentation.mdx b/tyk-developer-portal/tyk-portal-classic/customise/customise-documentation.mdx new file mode 100644 index 00000000..ff617fa0 --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/customise/customise-documentation.mdx @@ -0,0 +1,66 @@ +--- +title: "Swap out Swagger UI for ReDoc" +order: 6 +noindex: True +sidebarTitle: "Swap out Swagger UI for ReDoc" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +This short guide will show you how easy it is to swap out out the default https://swagger.io/tools/swagger-ui/ library for Portal Catalog API documentation for another tool like [ReDoc](http://rebilly.github.io/ReDoc/) + +* Open up the default `/opt/tyk-dashboard/portal/templates/swagger.html` + +``` expandable + {{ define "swaggerPage" }} + {{ template "header" .}} + <link href='/portal-assets/css/swagger.min.css' media='screen' rel='stylesheet' type='text/css'/> + {/* <link href='/portal-assets/css/swagger-ui.css' media='screen' rel='stylesheet' type='text/css'/> */} + <body> + {{ template "navigation" . }} + <div> + <div class="container" style="margin-top: 80px;"> + <svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" style="position:absolute;width:0;height:0"> +... + </svg> + <div id="swagger-ui"></div> + </div> + </div> + {{ template "footer" .}} + {{ template "scripts" .}} + <script src="/portal-assets/js/vendors.min.js"> </script> + <script src="/portal-assets/js/swagger.min.js"> </script> + <script type="text/javascript"> +... + </script> + </body> +</html> +{{ end }} +``` + +* Replace the content of `swagger.html` with the following: + +``` expandable + {{ define "swaggerPage" }} + {{ template "header" .}} + <body> + {{ template "navigation" . }} + <div> + <div class="container" style="margin-top: 80px;"> + <redoc spec-url="{{.SwaggerURL}}"></redoc> + </div> + </div> + {{ template "footer" .}} + <script src="https://cdn.jsdelivr.net/npm/redoc@next/bundles/redoc.standalone.js"></script> + </body> +</html> +{{ end }} +``` + +* Restart your dashboard service + +* Browse your portal documentation + +<img src="/img/dashboard/portal-management/redoc-petstore-tyk.png" alt="Tyk Portal Catalogue API Documentation with ReDoc" /> diff --git a/tyk-developer-portal/tyk-portal-classic/customise/customising-using-dashboard.mdx b/tyk-developer-portal/tyk-portal-classic/customise/customising-using-dashboard.mdx new file mode 100644 index 00000000..9f0c1606 --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/customise/customising-using-dashboard.mdx @@ -0,0 +1,123 @@ +--- +title: "Customize Pages with CSS and JavaScript" +order: 3 +noindex: True +sidebarTitle: "Customise Pages with CSS and JavaScript" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +The main customization that can be done with the Tyk Dashboard is via the CSS Editor. + +JS customization is also available in a programmatic way. + +#### Step 1: Open CSS Editor + +Click **CSS** from the **Portal Management** menu. + +<img src="/img/dashboard/portal-management/cssNav.png" alt="Portal management menu" /> + +#### Step 2: Make CSS Amendments + +In the CSS Editor, add the classes that you would like to override in the home page. For Tyk Cloud and Multi-Cloud users, this will already be filled in with some initial overrides for you: + +<img src="/img/dashboard/portal-management/cssEditor.png" alt="Portal CSS editor" /> + +#### Step 3: Make Email CSS Amendments + +<img src="/img/dashboard/portal-management/portal_email_css.png" alt="Email CSS editor" /> + +If you wish to customize how emails are displayed to end-users, then you can also add new classes to the Email CSS editor, these classes will be added in-line to the email that is sent out. + +Once you have finished making your changes, click **Update** and the new CSS will be available on your site. + +### Updating CSS via API +Alternatively, as always, you can perform the above actions with an API call instead of through the Dashboard UI. + +First, we'll need to get the block ID of the CSS component in order to update it. This is stored in Mongo by the Dashboard. +To get the block ID, we have to make a REST call to the Dashboard API. + +To do so, run this `curl` command: + +```{.copyWrapper} +curl www.tyk-test.com:3000/api/portal/css \ +-H "Authorization:{DASHBOARD_API_KEY}" +``` +Response: +```{.copyWrapper} expandable +{ + "email_css": "", + "id": "{CSS_BLOCK_ID}, + "org_id": "{ORG_ID}", + "page_css": ".btn-success {background-color: magenta1}" +} +``` +Now we can use the `id` and the `org_id` to update the CSS. +The below `curl` command will update the CSS for a specific organization. + +```{.copyWrapper} expandable +curl -X PUT http://tyk-dashboard.com/api/portal/css \ + -H "authorization:{DASHBOARD_API_KEY}" \ + -d '{ + "email_css": "", + "id": "{CSS_BLOCK_ID}, + "org_id": "{ORG_ID}", + "page_css": ".btn-success {background-color: magenta}" + }' +``` + + [1]: /img/dashboard/portal-management/portal_man_css.png + [2]: /img/dashboard/portal-management/portal_site_css.png + + ### Updating JavaScript via API + + In order to initialize the portal JS object in the database use the following request where `console.log(1)` should be replaced by your JS snippet: + + ```{.copyWrapper} +curl -X POST www.tyk-test.com:3000/api/portal/js \ +-H "Authorization:{DASHBOARD_API_KEY}" \ +-d '{"page_js": "console.log(1)"}' +``` + +Request: +```{.copyWrapper} +{ + "page_js": "console.log(1)" +} +``` + +Response: +```{.copyWrapper} +{ + "Status": "OK", + "Message": "609b71df21c9371dd5906ec1", + "Meta": null +} +``` + +The endpoint will return the ID of the portal JS object, this can be used to update it. + + ```{.copyWrapper} +curl www.tyk-test.com:3000/api/portal/js \ +-H "Authorization:{DASHBOARD_API_KEY}" \ +--data '{"page_js": "console.log(2)", "id": "609b71df21c9371dd5906ec1"}' +``` + +Request: +```{.copyWrapper} +{ + "page_js": "console.log(2)", + "id": "609b71df21c9371dd5906ec1" +} +``` + +Response: +```{.copyWrapper} +{ + "page_js": "console.log(1)" +} +``` + +The JavaScript snippet that's added through this endpoint is injected at the bottom of the portal page using a `<script>` tag. diff --git a/tyk-developer-portal/tyk-portal-classic/customise/customize-with-jquery.mdx b/tyk-developer-portal/tyk-portal-classic/customise/customize-with-jquery.mdx new file mode 100644 index 00000000..f7d6cba9 --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/customise/customize-with-jquery.mdx @@ -0,0 +1,113 @@ +--- +title: "Customizing using jQuery" +noindex: True +sidebarTitle: "Customising using jQuery" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +Tyk Portal comes prepackaged with jQuery. This opens up a whole world of customization, by extending our Portal using JavaScript and HTML to create dynamic content. + + +## Dynamic Content Rendering & Filtering + +Let's walk through an example where you use jQuery to fetch data from a REST endpoint, then display it in a table where we can filter our results. + +<iframe width="560" height="315" src="https://www.youtube.com/embed/njRgYUpL5vs" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + + +**First of all, create a custom page in the portal.** + + +<img src="/img/dashboard/portal-management/new_custom_page.png" alt="custom_page_setup" /> + +In the MainBody, you can paste the code below (click the text to display): + +<Expandable title={'Click to display the code'}> +```.html expandable + +<h2> Filterable Table </h2> + +<script> +window.onload = function() { + + $.ajax({ + type: "GET", + url: "https://www.mocky.io/v2/5eb1a7c53200005c8f28f8b5", + beforeSend: function() + { + $('html, body').animate({scrollTop: 0 + }, 'slow'); + $("#response").html('<img src="loading.gif" align="absmiddle" alt="Loading..."> Loading...<br /><br />'); + }, + success: function(response) + { + var htmlResponse = '<table id=results>\ + <thead>\ + <tr>\ + <th>Name</th>\ + <th>Location</th>\ + <th>Age</th>\ + </tr>\ + </thead>\ + <tbody id="myTable">' + + response.forEach( item => { + htmlResponse += ' <tr>\ + <td>' + item.name + '</td>\ + <td>' + item.location + '</td>\ + <td>' + item.Age + '</td>\ + </tr>' + }); + htmlResponse += "</tbody></table>" + + $('#results')[0].innerHTML = htmlResponse; + } + }); + + $("#myInput").on("keyup", function() { + var value = $(this).val().toLowerCase(); + $("#myTable tr").filter(function() { + $(this).toggle($(this).text().toLowerCase().indexOf(value) > -1) + }); + }); + } +</script> + + +<style> +table { + font-family: arial, sans-serif; + border-collapse: collapse; + width: 100%; +} + +td, th { + border: 1px solid #dddddd; + text-align: left; + padding: 8px; +} + +tr:nth-child(even) { + background-color: #dddddd; +} +</style> + +<p>Type something in the input field to search the table for first names, last names or emails:</p> +<input id="myInput" type="text" placeholder="Search.."> +<br /><br /> + +<div id=results> +</results> +``` +</Expandable> + +And save. + +Now visit the portal at "http://dashboard-host:3000/portal/custom" + +<img src="/img/dashboard/portal-management/custom_page_dynamic.png" alt="custom_page_display" /> + +You now have a searchable Input box that will dynamically filter the results of the table. diff --git a/tyk-developer-portal/tyk-portal-classic/customise/developer-meta-data.mdx b/tyk-developer-portal/tyk-portal-classic/customise/developer-meta-data.mdx new file mode 100644 index 00000000..94c51686 --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/customise/developer-meta-data.mdx @@ -0,0 +1,20 @@ +--- +title: "Customize the Developer Signup Form" +order: 4 +noindex: True +sidebarTitle: "Customise the Developer Signup Form" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +When a developer signs up to your developer Portal, you might wish to capture more information about the developer than is supplied by the default form. To enable new fields in this form (they are automatically added to the form as you add them), go to the **Portal Management > Settings** screen, and edit the **Sign up form customization** section: + +<img src="/img/dashboard/portal-management/dev_cusomise_2.5.png" alt="Tyk developer portal sign up form customization" /> + +### Developer metadata and keys + +All developer metadata is automatically added to the key metadata when a token is generated, this can be useful if you need to add more information to your upstream requests. + +A developer username will also automatically be made the alias for an API token so that it is easy to identify in the analytics. diff --git a/tyk-developer-portal/tyk-portal-classic/developer-profiles.mdx b/tyk-developer-portal/tyk-portal-classic/developer-profiles.mdx new file mode 100644 index 00000000..66526c3a --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/developer-profiles.mdx @@ -0,0 +1,160 @@ +--- +title: "Developer Profiles" +order: 2 +noindex: True +sidebarTitle: "Developer Profiles" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +Users that are signed up to your portal are called "Developers", these users have access to a Dashboard page which show them their API usage over the past 7 days as well as the policy and quota limits on their relevant keys. + +Developers can sign up to multiple APIs using the API catalog. + +Developer accounts belong to an organization ID, so accounts cannot be shared across organizations in a Tyk Dashboard setup. + +### Navigate to the Portal Developers Section + +<img src="/img/2.10/developers_menu.png" alt="Developer Menu" /> + +#### Select Add Developer + +<img src="/img/2.10/add_developer.png" alt="Developer Profile add" /> + +### Add Basic Details + +<img src="/img/2.10/add_developer_details.png" alt="Developer Profile Create Details" /> + +### Developer Profile Overview + +The first panel in a developer profile will show you an avatar (if they have a Gravatar-enabled email address), as well as the basic fields of their signup: + +<img src="/img/2.10/developer_details.png" alt="Developer profile detail" /> + +### Developer Usage + +The next panel will show you their apI usage as an aggregate for all the tokens that they have generated with their developer access: + +<img src="/img/2.10/developer_graph.png" alt="Developer usage graph" /> + +### Developer Keys + +In this panel, you will be able to see the various Keys the developer has access to, and the policies that are connected to the individual Key. + + +<Note> +From version 1.9, you can now apply multiple policies to an individual Key. +</Note> + + +To drill down into the specific usage patterns for each Key, click **ANALYTICS** for the Key. + +<img src="/img/2.10/developer_keys.png" alt="Developer Keys" /> + +### Add a New Key + +To subscribe a developer to a new Key, from the Edit Developer screen, click **New Key**. From the pop-up screen, select one or more policies from the drop-down list and click **Request Key**. + + <img src="/img/2.10/developer_new_key.png" alt="New Key Request" /> + +### Changing Developer Policy Keys + +#### Step 1: View the Developer Profile + +Browse to the developers list view and select the developer that you wish to manage. + +<img src="/img/2.10/developer_details.png" alt="Developer profile detail" /> + +#### Step 2: View Keys List + +This sections shows you the Keys and the policies connected to them. This view will always try to match the access level to a catalog entry, if the policy assigned to a developer is not in the catalog, the entry will read "(No Catalog Entry)". We recommend that all policy levels are in your catalog, even if they are not all live. + +#### Step 3: Click Options + +From the Options drop-down for the Key, select **Change Policy**. + +<img src="/img/2.10/developer_keys.png" alt="Keys Sections" /> + +#### Step 4: Select the New Policy + +Select a new policy to add to your Key from the **Policies** drop-down list. You can also remove existing policies connected to the Key. + +<img src="/img/2.10/developer_new_key.png" alt="Change policy drop down list" /> + +#### Step 5: Save the Change + +Click **CHANGE KEY POLICY** to save the changes. + +### Developer OAuth Clients + + +### Edit the Developer Profile + +All fields in the profile are editable. In this section you can select a field and modify that data for the developer. This will not affect any tokens they may have, but it will affect how it appears in their Developer Dashboard in your Portal. + +<img src="/img/2.10/edit_developer_details.png" alt="Developer edit form" /> + +Developers can edit this data themselves in their accounts section. + +### Search for a Developer + +You can search for a developer (by email address) by entering their address in the Search field. + +This option is only available from Dashboard v1.3.1.2 and onwards. + +<img src="/img/2.10/search_developers.png" alt="Developer Profile Search" /> + +### Developer Edit Profile + +Once logged in, a developer can edit their profile. Select **Edit profile** from the **Account** menu drop-down list. + +<img src="/img/dashboard/portal-management/developer_manage_profile.png" alt="Manage Profile" /> + +A developer can change the following: +* Email +* Change Password +* Name +* Telephone +* Country Location + +### Reset Developer Password + +If a developer has forgotten their password, they can request a password reset email from the Login screen. + +<img src="/img/dashboard/portal-management/login_screen.png" alt="Login Screen" /> + +1. Click **Request password reset** +2. Enter your email address and click **Send Password reset email** + +<img src="/img/dashboard/portal-management/email_password_request.png" alt="Email Reset" /> + +You will be sent an email with a link to reset your Developer password. Enter your new password and click **Update**. You can then login with your new details. + + +<Note> +Your password must be a minimum of 6 characters. +</Note> + + +<img src="/img/dashboard/portal-management/password_confirmation.png" alt="Confirm password" /> + + + + + + [1]: /img/dashboard/portal-management/developer_menu_2.5.png + [2]: /img/dashboard/portal-management/add_developer_2.5.png + [3]: /img/dashboard/portal-management/developer_details_2.5.png + [4]: /img/dashboard/portal-management/developer_overview_2.5.png + [5]: /img/dashboard/portal-management/developer_usage_2.5.png + [6]: /img/dashboard/portal-management/developer_subs_2.5.png + [7]: /img/dashboard/portal-management/developer_edit_2.5.png + [8]: /img/dashboard/portal-management/developer_search_2.5.png + [13]: /img/dashboard/portal-management/developer_edit_2.5.png + [14]: /img/dashboard/portal-management/keys_dev_profile.png + [15]: /img/dashboard/portal-management/change_key_policy.png + [16]: /img/dashboard/portal-management/new_key_request.png + + diff --git a/tyk-developer-portal/tyk-portal-classic/dynamic-client-registration.mdx b/tyk-developer-portal/tyk-portal-classic/dynamic-client-registration.mdx new file mode 100644 index 00000000..d2d96f98 --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/dynamic-client-registration.mdx @@ -0,0 +1,43 @@ +--- +title: "Classic Portal - Dynamic Client Registration" +order: 3 +noindex: True +sidebarTitle: "Overview" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +## OAuth 2.0 Dynamic Client Registration Protocol (DCR) + +Available from version 3.2.0 onwards. + +## What is Dynamic Client Registration? + +DCR is a protocol of the Internet Engineering Task Force put in place to set standards in the dynamic registration of clients with authorization servers. +We will go into the specifics of how it works in the context of Tyk, but if you are interested in reading the full RFC, go to: https://tools.ietf.org/html/rfc7591 + +## Why should I use it? + +DCR is a way for you to integrate your developer portal with an external identity provider such as Keycloak, Gluu, Auth0, Okta etc... +The portal developer won't notice a difference. However when they create the app via Tyk Developer portal, Tyk will dynamically register that client on your authorization server. This means that it is the Authorization Server who will issue issue the Client ID and Client Secret for the app. +Some of our users leverage external Identity Providers because they provide a variety of features to support organizations in managing identity in one place across all their stack. + +This feature is optional and you can still have a great level of security only using Tyk as your authorization server. + +## Enabling Dynamic Client Registration + +We provide guides for the following identity providers: + +- [Gluu](/tyk-developer-portal/tyk-portal-classic/gluu-dcr). Official docs are available [here](https://gluu.org/docs/gluu-server/4.0/admin-guide/openid-connect/#dynamic-client-registration). +- [Curity](/tyk-developer-portal/tyk-portal-classic/curity-dcr). Official docs are available [here](https://curity.io/docs/idsvr/latest/token-service-admin-guide/dcr.html). +- [Keycloak](/tyk-developer-portal/tyk-portal-classic/keycloak-dcr). Official docs are available [here](https://github.com/keycloak/keycloak-documentation/blob/master/securing_apps/topics/client-registration.adoc). +- [OKTA](/tyk-developer-portal/tyk-portal-classic/okta-dcr). Official docs are available [here](https://developer.okta.com/docs/reference/api/oauth-clients/). + + +In case your provider isn't on the list, use the "Other" provider option in the DCR settings. This mode would keep the interaction with your IDP as standard possible. Note that not all IDPs fully implement the standard. + +## Troubleshooting + +The DCR functionality abstracts most of the errors to the end user (in this case, the developer). In order to diagnose issues between Tyk and your IDP, please refer to the Tyk Dashboard logs. diff --git a/tyk-developer-portal/tyk-portal-classic/gluu-dcr.mdx b/tyk-developer-portal/tyk-portal-classic/gluu-dcr.mdx new file mode 100644 index 00000000..65907c45 --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/gluu-dcr.mdx @@ -0,0 +1,147 @@ +--- +title: "Step by step guide using Gluu" +order: 3 +noindex: True +sidebarTitle: "Step by step guide using Gluu" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +We are going walk you through a basic integration of Tyk with [Gluu](https://gluu.org/) using the [OpenID Connect Dynamic Client Registration protocol](https://tools.ietf.org/html/rfc7591). Our current implementation provides support for the client credentials flow with support for <Tooltip tip="JSON Web Tokens">JWT</Tooltip>. + +The user journey is as follow: + +1. A developer signs up and creates a Dynamic Client Registration provider using your Developer Portal. + +2. Tyk sends the Dynamic Client Registration call to your <Tooltip tip="Identity Provider">IDP</Tooltip>. The IDP replies with the client ID and secret. + +3. Using that information, the developer (or the application) triggers a call to the token endpoint of the IDP. + +4. Your developer (or the application) then triggers a call to Tyk, using the token that was generated by the IDP. Tyk validates this token using the <Tooltip tip="JSON Web Key Sets">JWKS</Tooltip> provided by the IDP. + +### Requirements + +- A Gluu installation, more details [here](https://gluu.org/get-started/). +- A [Tyk Self Managed installation](/tyk-self-managed/install) (Gateway + Dashboard). + +### Getting started with Gluu + +In order to get started with Dynamic Client Registration you’ll need to get the OpenID Connect registration endpoint. Open your Gluu dashboard and select the "Configuration" section. Select "JSON Configuration" and toggle the "OxAuth Configuration" tab. + +<img src="/img/dcr/gluu/step_1.png" alt="Step 1" /> + +In this view you will find the registration endpoint: + +<img src="/img/dcr/gluu/step_2.png" alt="Step 2" /> + +Another endpoint that will be relevant for your setup is the Well-Known configuration endpoint. Keep both URLs handy as you’ll use them for our next steps. This endpoint typically looks as follows: https://gluu-server/.well-known/openid-configuration + +Because of known issues with Tyk’s JWT driver, you’ll set specific algorithms for the JWKS endpoint. In the same "OxAuth Configuration" tab, scroll down to "jwksAlgorithmsSupported" and select the following options: + +<img src="/img/dcr/gluu/step_3.png" alt="Step 3" /> + +Click "Save OxAuth Configuration" afterwards. + +For more information on this particular issue please check [this thread](https://support.gluu.org/authentication/8780/wrong-size-of-ec-x-value-in-jwks_uri-while-using-openid/) in the Gluu forum. + +### Setting up Tyk + +Now you're ready to set up Tyk. For compatibility reasons, check your `tyk_analytics.conf` and make sure that a proper `oauth_redirect_uri_separator` parameter is set. You can use the following value: + +```json + "oauth_redirect_uri_separator": ";", +``` + +Remember to restart the service after applying the above change. + +Now open the Tyk Dashboard and click **APIs** under **System Management**. Create a new API called "Gluu API": + +<img src="/img/dcr/gluu/step_4.png" alt="Step 4" /> + +After the first part of the API creation form was filled, click on "Configure API" and set the authentication settings as follows: + +<img src="/img/dcr/gluu/step_5.png" alt="Step 5" /> + + +<Note> +Where do I get the proper JWKS URI for my Gluu environment? + +The JWKS URI is a required field in the `.well-known/openid-configuration` endpoint of your OpenID Connect Provider metadata. Typically found as `"jwks_uri"`. Please see the spec https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderConfigurationResponse for further information. +</Note> + + +For the **Identity Source** field use `"client_id"` and for **Policy Field Name** use `"pol"`. + +Click "Save" and switch to the "Policies" button under "System Management". Once in this section, click on "Create a Policy" and call it "Gluu Policy". Use the default values for this one. Remember to select the previously created "Gluu API" in the access rights section. You will also need to set an expiration setting for the keys. + +After the policy is ready, switch back to the API settings and make sure that the API is using the appropriate policy: + +<img src="/img/dcr/gluu/step_6.png" alt="Step 6" /> + +Now you’re ready to add this API to the developer portal. Switch to the "Catalog" section under "Portal Management" on the navigation menu. Click on "Add New API", set a name for it and select the newly created policy. For this example use "Gluu Policy": + +<img src="/img/dcr/gluu/step_7.png" alt="Step 7" /> + +Hit "Save" and click on the recently created item again, switch to the "Settings" tab that’s next to "API Details". In "API Details" toggle the "Override global settings" option. + + +<Note> +Tyk lets you set global portal settings that apply to **all portal-listed APIs**, in this guide we assume you’re enabling and setting up DCR for a single API. In case you want to enable DCR for all the APIs, you should go to the **Settings** section under **Portal Management**, and in the **API Access** tab you can enter your DCR settings there. +</Note> + + +Once the "Override global settings" option is toggled, scroll down to the DCR section in the bottom and enter the following settings: + +<img src="/img/dcr/gluu/step_8.png" alt="Step 8" /> + +**Providers:** Different providers might implement the standard in slightly different ways. Tyk provides a specific driver for each one. For IDPs that aren’t on the list use the "Other" option. For this guide, pick "Gluu". + +**Grant Types:** The [OAuth 2.0 grant types](/api-management/authentication/oauth-2) types that will be used by the client, see the [specification](https://openid.net/specs/openid-connect-registration-1_0.html#rfc.section.2) for more details. Set "Client Credentials". + +**Token Endpoint Auth Method:** defines the way the client will authenticate against the token endpoint. Use "Client Secret - Post". + +**Response Types:** OAuth 2.0 response types that will be used by the client. Set **Token**. + +**Identity Provider Host:** Base IDP URL, e.g. `https://gluu-server/` + +**Client Registration Endpoint:** OpenID Connect client registration endpoint. The value we use is `https://gluu-server/oxauth/restv1/register` + +This value is found in your well-known discovery document as `registration_endpoint`. The well-known location URL is typically `https://gluu-server/.well-known/openid-configuration` (replace "gluu-server" with your hostname). + +**Initial Registration Access Token:** the token that’s used to register new clients, this was generated in the early steps of the guide. + +### Testing the flow + +Now that both Tyk and Gluu are ready you can try the complete flow. Click "Developers" under "Portal Management", then click "Add developer" and enter some basic information here to create a developer user. + +After the developer is created, open the portal, click on the "OAuth Clients" navigation bar button and follow the wizard: + +<img src="/img/dcr/gluu/step_9.png" alt="Step 9" /> + +After clicking "Create first OAuth Client" you’ll see your previously created "Gluu API". Select it and click "Save and continue". The following screen will require you to enter a client name. It’s possible to set redirect URLs if you also plan to use this client for other flow types. This setting can be left blank for the purposes of this example. + +<img src="/img/dcr/gluu/step_10.png" alt="Step 10" /> + +Once you click "Create", Tyk will trigger a registration on your IDP and the details of your client will show up: + +<img src="/img/dcr/gluu/step_11.png" alt="Step 11" /> + +If you check the Gluu dashboard you will see new client (named "GluuClient"): + +<img src="/img/dcr/gluu/step_12.png" alt="Step 12" /> + +The next step is to generate a token and use it for accessing your "Gluu API". you can use Postman for this. You will need the token URL which it’s also present in the Well-Known URI of your organization. The field is named `"token_endpoint"`. +For this example use the following: https://gluu-server/oxauth/restv1/token + +Your Postman request should contain the following body, where `"client_id"` and `"client_secret"` are the values you got from the developer portal: + +<img src="/img/dcr/gluu/step_13.png" alt="Step 13" /> + +Note that you aren’t using any additional headers for this request, the client credentials are enough. + +Once you get a response from the IDP, you can copy the `"access_token"` and use it to access your "Gluu API", this request will be proxied by Tyk: + +<img src="/img/dcr/gluu/step_14.png" alt="Step 14" /> + diff --git a/tyk-developer-portal/tyk-portal-classic/graphql.mdx b/tyk-developer-portal/tyk-portal-classic/graphql.mdx new file mode 100644 index 00000000..c3664c4a --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/graphql.mdx @@ -0,0 +1,51 @@ +--- +title: "Developer Portal GraphQL" +'og:description': "How to publish GraphQL APIs to your Tyk Developer Portal" +tags: ['GraphQL', 'Playground', 'CORS', 'UDG'] +order: 7 +noindex: True +sidebarTitle: "GraphQL with Classic Portal" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +As of Tyk v3.0.0, you can now publish GraphQL APIs, including [Universal Data Graph](/api-management/data-graph#overview) APIs(UDG) to the Tyk Developer Portal. + +When you do that, your API consumers can navigate through a GraphQL Playground, with an IDE complete with Intellisense. + +<img src="/img/portal/portal-graphql.png" alt="Portal GraphQL Playground" /> + +## Video Walkthrough + +We have a YouTube walkthrough of how to publish a GraphQL API to your Developer Portal: + +<iframe width="560" height="315" src="https://www.youtube.com/embed/A2Hn9ub2mNg" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + +## How To Set Up + +Simply create a GraphQL or Universal Data Graph API, create a Policy which protects it, and then publish it to the Developer Portal Catalog. + +In the "Create a Catalog" section, at the bottom, make sure you enable the "Display Playground" + + +<img src="/img/portal/portal-graphql-setup.png" alt="Portal GraphQL Playground Setup" /> + +And then, when your API consumers are on the Developer Portal Catalog and click on View Documentation, they will be taken to the GraphQL Playground. + +<img src="/img/portal/portal-graphql-playground-viewdocs.png" alt="Portal GraphQL Playground View Docs" /> + + +## Protected GraphQL Catalog + +If you have a protected API, your users won't be able to inspect the GraphQL schema or make API calls until they add their API Key to the Headers section: + +<img src="/img/portal/portal-graphql-header-injection.png" alt="Portal GraphQL Playground Header Injection" /> + +## CORS + +You may have to enable the following CORS settings in the "Advanced Options" of the API Designer to allow your consumers to access the GraphQL Playground: + + +<img src="/img/portal/portal-graphql-cors.png" alt="Portal GraphQL Playground CORS" /> diff --git a/tyk-developer-portal/tyk-portal-classic/key-requests.mdx b/tyk-developer-portal/tyk-portal-classic/key-requests.mdx new file mode 100644 index 00000000..b88cedd8 --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/key-requests.mdx @@ -0,0 +1,47 @@ +--- +title: "Key Requests" +noindex: True +sidebarTitle: "Key Requests" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +## Key Requests + +A key request is a record that is generated when a developer requests an access token for an API published in the API Catalog. The Key request encompasses the following information: + +- The policy of which access is being requested +- The developer doing the requesting +- The catalog entry in question +- The reasoning of why the developer should have access (these are dynamic fields and can be configured) + +When a developer requests access to an API Catalog entry, this key request represents that request for access. The key request can then be acted on, either by the portal itself, or by an administrator. The key request does not grant a token yet, it simply marks the fact that a token has been requested and why. + +Tyk enables you to manage this flow in a few ways: + +- Auto-approve the key request. +- Have an admin approve the key-request. +- Hand off to a third-party system to manage the key-request (e.g. for billing or additional user validation). This is done via WebHooks or via the "Redirect Key Request" Portal Setting. + +## Key Approval +Once a key request is created, one of two things can be done to it: + +- It can be approved: Covered below +- It can be declined: In which case the request is deleted. + +A key request can be created using the Dashboard API too, in fact, the Key Request mechanism is a great way to create a mapping between an identity (a developer) and a token, and managing that process. + +### Secure Key Approval + +By default, the Key Approval flow is straight forward. Once a Key Request is approved, the Developer will be notified via an email which contains the API Key. + +As of Dashboard version `3.1.0`, it is now possible to turn on a more secure key approval flow. Once the "Request Key Approval" setting is enabled, we see an additional setting: +<img src="/img/dashboard/portal-management/secure_key_approval_setting.png" alt="secure_key_approval_setting" /> + +With this feature turn on, we prevent the API key from being sent in plain text via email. Instead, the once a key request is approved, the Developer will be sent a confirmation link in an email that directs them to the Portal: +<img src="/img/dashboard/portal-management/secure_key_approval_email.png" alt="secure_key_approval_email" /> + +After clicking the `Generate Key` link and logging into the Portal, the key becomes available to the user: +<img src="/img/dashboard/portal-management/secure_key_approval_generate.png" alt="secure_key_approval_generate" /> \ No newline at end of file diff --git a/tyk-developer-portal/tyk-portal-classic/keycloak-dcr.mdx b/tyk-developer-portal/tyk-portal-classic/keycloak-dcr.mdx new file mode 100644 index 00000000..9349f80e --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/keycloak-dcr.mdx @@ -0,0 +1,159 @@ +--- +title: "Step by step guide using Keycloak" +order: 1 +noindex: True +sidebarTitle: "Step by step guide using Keycloak" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +We are going walk you through a basic integration of Tyk with Keycloak using the [OpenID Connect Dynamic Client Registration protocol](https://tools.ietf.org/html/rfc7591). Our current implementation provides support for the client credentials flow with support for <Tooltip tip="JSON Web Tokens">JWT</Tooltip>. To the developer it works like this: + +1. An API with its corresponding security policy is created in Tyk. It is then added to the Developer Portal Catalog. + +2. A developer signs up and creates a Dynamic Client Registration provider using the Developer Portal. +Tyk sends the Dynamic Client Registration call to your <Tooltip tip="Identity Provider">IDP</Tooltip>. The IDP replies with the client ID and secret. + +3. Using the previous information, the developer (or your application) triggers a call to the token endpoint of the IDP. +The developer (or your application) then triggers a call to Tyk, using the token that was generated by the IDP. Tyk validates this token using the <Tooltip tip="JSON Web Key Sets">JWKS</Tooltip> provided by the IDP. + +### Requirements + +- A [Keycloak](https://www.keycloak.org/) instance. +- A [Tyk Self Managed installation](/tyk-self-managed/install) (Gateway + Dashboard). + +### Getting started with Keycloak + +To get started with Dynamic Client Registration in Keycloak you'll need to generate an [initial access token](https://openid.net/specs/openid-connect-registration-1_0.html#Terminology) using the Keycloak Administration Console. After logging in, click **Realm settings** under **Configure** and select the **Client Registration** tab: + +<img src="/img/dcr/keycloak/step_1.png" alt="Step 1" /> + +To generate an initial access token, click **Create** and set the expiration time and maximum number of clients to be created using this token: + +<img src="/img/dcr/keycloak/step_2.png" alt="Step 2" /> + +Click **Save** and the token will be created. Keep it safe as you'll use this token to configure Tyk. + +### Setting up Tyk + +Now you're ready to set up Tyk. For compatibility reasons, check your `tyk_analytics.conf` and make sure that a proper `oauth_redirect_uri_separator` parameter is set. You may use the following value: + +```json + "oauth_redirect_uri_separator": ";", +``` + +**Note:** If you're using a self-signed certificate on your Keycloak instance, you will need to set additional flags on both gateway and dashboard. For skipping DCR endpoint SSL verification, add the following flag to `tyk_analytics.conf`: + +```json + "dcr_ssl_insecure_skip_verify": true +``` + +Also add the following flag to `tyk.conf`, this will instruct the gateway to skip SSL verification when the JWT middleware is in use, particularly when JWKS are retrieved from your IDP: + +```json + "jwt_ssl_insecure_skip_verify": true +``` + +Remember to restart the services after applying the above changes. + +Open the Tyk Dashboard and click **APIs** under **System Management**. Create a new API called "Keycloak API": + +<img src="/img/dcr/keycloak/step_3.png" alt="Step 3" /> + +Complete first part of the API creation form, then click **Configure API** and set the Authentication mode as in the image below: + +<img src="/img/dcr/keycloak/step_4.png" alt="Step 4" /> + + +<Note> +Where do I get the proper JWKS URI for my Keycloak environment? + +The JWKS URI is a required field in the `.well-known/openid-configuration` endpoint of your OpenID Connect Provider metadata. Please see the [OpenID spec](https://openid.net/specs/openid-connect-discovery-1_0.html#ProviderConfigurationResponse) for further information. +</Note> + + + + +For the **Identity Source** field use `"sub"` and for **Policy Field Name** use `"pol"`. + +1. Click **Save** +2. Select **Policies** under **System Management** +3. Click **Create a Policy** and call it **Keycloak Policy**. Use the default values for this policy. +4. In the **Access rights** section, select your previously created **Keycloak API**. You will also need to enter an expiration setting for your keys. + +After the policy is created, switch back to the API settings and make sure that the API is using your **Keycloak API** policy: + +<img src="/img/dcr/keycloak/step_5.png" alt="Step 5" /> + +Now you're ready to add this API to the Developer Portal. +1. Click **Catalog** under **Portal Management** on the navigation menu. +2. Click **Add New API**, enter a name for it and select the newly created policy. Again, you will use **Keycloak Policy**: + +<img src="/img/dcr/keycloak/step_6.png" alt="Step 6" /> + +1. Click **Save** then open the API added again +2. Open the **Settings** tab. +3. In **API Details** select the **Override global settings** option. + + + + <Note> + Tyk lets you set global portal settings that apply to **all portal-listed APIs**, in this guide we assume you’re enabling and setting up DCR for a single API. In case you want to enable DCR for all the APIs, you should go to the **Settings** section under **Portal Management**, and in the **API Access** tab you can enter your DCR settings there. + </Note> + + +4. Scroll down to the DCR section and enter the following settings: + +<img src="/img/dcr/keycloak/step_7.png" alt="Step 7" /> + +**Providers:** Different providers might implement the standard in slightly different ways, Tyk provides a specific driver for each one. For IDPs that aren’t on the list use the **Other** option. + +**Grant Types:** The [OAuth 2.0 grant types](/api-management/authentication/oauth-2) that will be used by the client, see the [specification](https://openid.net/specs/openid-connect-registration-1_0.html#rfc.section.2) for more details. + +**Token Endpoint Auth Method:** defines the way the client will authenticate against the token endpoint. + +**Response Types:** OAuth 2.0 response types that will be used by the client. + +**Identity Provider Host:** Base IDP URL, e.g. `https://keycloak:8443/` + +**Client Registration Endpoint:** OpenID Connect client registration endpoint. This value is found in your well-known discovery document as `registration_endpoint`. The well-known location URL is typically `https://keycloak:8443/.well-known/openid-configuration` + +**Initial Registration Access Token:** the token that’s used to register new clients, this was generated in the early steps of the guide. + +### Testing the flow + +Now that both Tyk and Keycloak are ready we can test the complete flow. + +1. Click **Developers** under **Portal Management** +2. Click on **Add developer** and create a developer user. + +After the developer is created, open your Developer Portal, click on the **OAuth Clients** navigation bar button and follow the wizard: + +<img src="/img/dcr/keycloak/step_8.png" alt="Step 8" /> + +Click **Create first OAuth Client**. You’ll see your previously created **Keycloak API**, select it and click **Save and continue**. The following screen will require you to enter a client name. It’s also possible to set redirect URLs if you also plan to use this client for other flow types. This setting can be left blank for the purposes of this guide. + +<img src="/img/dcr/keycloak/step_9.png" alt="Step 9" /> + +Once you click **Create**, Tyk will trigger a registration on your IDP and the details of your client will be displayed: + +<img src="/img/dcr/keycloak/step_10.png" alt="Step 10" /> + +If you check the Keycloak dashboard you will see this client too: + +<img src="/img/dcr/keycloak/step_11.png" alt="Step 11" /> + +The next step is to generate a token and use it for accessing your **Keycloak API**. We'll use Postman for this. You will need your token URL which is also the well-known URL for your organization. +For this guide we use `https://keycloak:8443/auth/realms/master/protocol/openid-connect/token` + +Your Postman request should contain the following body, where `"client_id"` and `"client_secret"` are the credentials you got from the developer portal: + +<img src="/img/dcr/keycloak/step_12.png" alt="Step 12" /> + +Note that we aren’t using any additional headers for this request, the client credentials are enough. + +Once we get a response from the IDP, we can copy the `"access_token"` and use it to access our **Keycloak API**, this request will be proxied by Tyk: + +<img src="/img/dcr/keycloak/step_13.png" alt="Step 13" /> diff --git a/tyk-developer-portal/tyk-portal-classic/monetise.mdx b/tyk-developer-portal/tyk-portal-classic/monetise.mdx new file mode 100644 index 00000000..452c3d0e --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/monetise.mdx @@ -0,0 +1,35 @@ +--- +title: "Monetize" +order: 11 +noindex: True +sidebarTitle: "Monetising your APIs" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +Out of the box, the Tyk Developer Portal does not have a billing component, however, this does not mean that it is not possible to enable monetization within a Portal developer access flow. + +### The Developer Key Request Flow + +When a developer enrolls for API access with a Tyk portal system, they will: + +1. Sign up +2. Select a catalog entry to participate in +3. Submit a key request form +4. Receive their token + +With Tyk, it is possible to prevent step 4, which auto-enables the key, and instead have the developer redirected to a third party app. This app can then handle any transactional process such as taking a credit card number or pre-validating the developer, before returning the developer to the Portal. + +When Tyk hands off to the redirected app, it will also add the key request ID to the request, so the application that handles the transaction can then use the Tyk Dashboard REST API to approve the key request (triggering the email that notifies the developer of their token, as well as notifying the calling application of the raw token), closing the loop. + +To enable the developer hand-off in a Tyk Portal, from the **Portal Settings** enable the redirect option: + +<img src="/img/dashboard/portal-management/portal_redirect_2.5.png" alt="Redirect key requests form" /> + +## Example Using Stripe + +In this video, we walk you through setting up Stripe to take payments via your Tyk Developer Portal. + +<iframe width="560" height="315" src="https://www.youtube.com/embed/k5b3FURaULk" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> \ No newline at end of file diff --git a/tyk-developer-portal/tyk-portal-classic/okta-dcr.mdx b/tyk-developer-portal/tyk-portal-classic/okta-dcr.mdx new file mode 100644 index 00000000..82fff33b --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/okta-dcr.mdx @@ -0,0 +1,172 @@ +--- +title: "Step by step guide using Okta" +order: 2 +noindex: True +sidebarTitle: "Step by step guide using Okta" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +## Introduction + +We are going walk you through a basic integration of Tyk with Okta using the [OpenID Connect Dynamic Client Registration protocol](https://tools.ietf.org/html/rfc7591). Our current implementation provides support for the client credentials flow with support for <Tooltip tip="JSON Web Tokens">JWT</Tooltip>. + +The user journey is as follow: + +1. A developer signs up and creates a Dynamic Client Registration provider using the Developer Portal. + +2. Tyk sends the Dynamic Client Registration call to your <Tooltip tip="Identity Provider">IDP</Tooltip>. The IDP replies with the client ID and secret. + +3. Using that information, the developer (or the application) triggers a call to the token endpoint of the IDP. + +4. The developer (or the application) then triggers a call to Tyk, using the token that was generated by the IDP. Tyk validates this token using the <Tooltip tip="JSON Web Key Sets">JWKS</Tooltip> provided by the IDP. + +### Requirements + +- An OKTA account (a [trial account](https://www.okta.com/free-trial/) should be enough). +- A [Tyk Self Managed installation](/tyk-self-managed/install) (Gateway + Dashboard). + +### Getting started with OKTA + +First signup to OKTA, the initial screen looks like: + +<img src="/img/dcr/okta/step_1.png" alt="Step 1" /> + +The first thing you’ll need for our integration is an API token from OKTA, the OpenID specification also calls this an [Initial Access Token](https://openid.net/specs/openid-connect-registration-1_0.html#Terminology) to differentiate it from other tokens that are used with this protocol. To create this token, click **API** option from the **Security** menu on the navigation bar: + +<img src="/img/dcr/okta/step_2.png" alt="Step 2" /> + +From the API section, select the **Tokens** tab and click **Create Token** and enter a name for the token. For this guide we’re calling it "Tyk Integration": + +<img src="/img/dcr/okta/step_3.png" alt="Step 3" /> + +Click **Create Token**. Keep it safe as you'll use this token to configure Tyk. + +Next you need to create a scope, from the **Authorization servers** tab in the API section, click **Add Scope**. You need to select the **Set as default scope** option: + +<img src="/img/dcr/okta/step_4.png" alt="Step 4" /> + +### Setting up Tyk + +Now you're ready to set up Tyk. For compatibility reasons, check your `tyk_analytics.conf` and make sure that a proper `oauth_redirect_uri_separator` parameter is set. You may use the following value: + +```json + "oauth_redirect_uri_separator": ";", +``` + +Remember to restart the service after applying the above change. + +Now open the Tyk Dashboard and click **APIs** under **System Management**. Create a new API called "OKTA API": + +<img src="/img/dcr/okta/step_5.png" alt="Step 5" /> + +Complete first part of the API creation form, then click **Configure API** and set the Authentication mode as in the image below: + +<img src="/img/dcr/okta/step_6.png" alt="Step 6" /> + + +<Note> +Where do I get the proper JWKS URI for my Keycloak environment? + +From the OKTA Dashboard, open the **API** section under **Security**, take the base URL from the default Authorization Server and append the `/v1/keys` suffix, e.g. `https://tyk-testing.okta.com/oauth2/default/v1/keys`. +</Note> + + +For the **Identity Source** field use `"sub"` and for **Policy Field Name** use `"pol"`. + +1. Click **Save** +2. Select **Policies** under **System Management** +3. Click **Create a Policy** and call it **OKTA Policy**. Use the default values for this policy. +4. In the **Access rights** section, select your previously created **OKTA API**. You will also need to enter an expiration setting for your keys. + +After the policy is created, switch back to the API settings and make sure that the API is using your **OKTA Policy** policy: + +<img src="/img/dcr/okta/step_7.png" alt="Step 7" /> + +Now you're ready to add this API to the Developer Portal. +1. Click **Catalog** under **Portal Management** on the navigation menu. +2. Click **Add New API**, enter a name for it and select the newly created policy. Again, you will use **OKTA API**: + +<img src="/img/dcr/okta/step_8.png" alt="Step 8" /> + +1. Click **Save** then open the API added again +2. Open the **Settings** tab. +3. In **API Details** select the **Override global settings** option. + + + + <Note> + Tyk lets you set global portal settings that apply to **all portal-listed APIs**, in this guide we assume you’re enabling and setting up DCR for a single API. In case you want to enable DCR for all the APIs, you should go to the **Settings** section under **Portal Management**, and in the **API Access** tab you can enter your DCR settings there. + </Note> + + +4. Scroll down to the DCR section and enter the following settings: + + +<img src="/img/dashboard/portal-management/dcr-okta-grant-types.png" alt="Okta Grant Types" /> + + +**Providers:** Different providers might implement the standard in slightly different ways, Tyk provides a specific driver for each one. For IDPs that aren’t on the list use the "Other" option. For this guide, pick "OKTA". + +**Grant Types:** The grant types that will be used by the client. See the [specification](https://openid.net/specs/openid-connect-registration-1_0.html#rfc.section.2) for more details. You need to enter the following grant types: + * Client Credentials + * Implicit + * Authorization Code + +**Token Endpoint Auth Method:** defines the way the client will authenticate against the token endpoint. Use "Client Secret - Post". + +**Response Types:** OAuth 2.0 response types that will be used by the client. Set **Token**. + +**Identity Provider Host:** Base IDP URL, e.g. `https://tyk-testing.okta.com/` + +**Client Registration Endpoint:** OpenID Connect client registration endpoint. The value we use is `https://tyk-testing.okta.com/oauth2/v1/clients` + +This value is found in your well-known discovery document as `registration_endpoint`. The well-known location URL is typically `https://tyk-testing.okta.com/.well-known/openid-configuration` (replace "tyk-testing" with your org.). + +**Initial Registration Access Token:** the token that’s used to register new clients, this was generated in the early steps of the guide. + + +<Note> +A note on grant types and response types in OKTA + +It’s important to note that OKTA’s DCR endpoint supports a parameter called `"application_type"`, the application types aren’t standard across all IDPs, while the initial specification mentions `"native"` or `"web"` types, some IDPs implement their own. In the current implementation Tyk supports the usage of the `"web"` application type which is necessary in supporting the client credentials flow that’s described in this guide, as well as others, this is set automatically when OKTA is set as the provider. Currently, the ability to change the application type is available with the Enterprise Developer Portal. +</Note> + + +### Testing the flow + +Now that both Tyk and OKTA are ready we can test the complete flow. + +1. Click **Developers** under **Portal Management** +2. Click on **Add developer** and create a developer user. + +After the developer is created, open your Developer Portal, click on the **OAuth Clients** navigation bar button and follow the wizard: + +<img src="/img/dcr/okta/step_10.png" alt="Step 10" /> + +Click **Create first OAuth Client**. You’ll see your previously created **OKTA API**, select it and click **Save and continue**. The following screen will require you to enter a client name. It’s also possible to set redirect URLs if you also plan to use this client for other flow types. This setting can be left blank for the purposes of this guide. + +<img src="/img/dcr/okta/step_11.png" alt="Step 11" /> + +Once you click **Create**, Tyk will trigger a registration on your IDP and the details of your client will be displayed: + +<img src="/img/dcr/okta/step_12.png" alt="Step 12" /> + +If you check the OKTA dashboard you will see this client too: + +<img src="/img/dcr/okta/step_13.png" alt="Step 13" /> + +The next step is to generate a token and use it for accessing our **OKTA API**. We'll use Postman for this. You will need your token URL which is also the well-known URL for your organization. +For this guide you'll use `https://[org].okta.com/oauth2/default/v1/token` + +Your Postman request should contain the following body, where `"client_id"` and `"client_secret"` are the credentials you got from the developer portal: + +<img src="/img/dcr/okta/step_14.png" alt="Step 14" /> + +Note that we aren’t using any additional header for this request, the client credentials are enough. We’re also passing our previously created `"tyk"` scope as value. + +Once we get a response from the IDP, we can copy the `"access_token"` and use it to access our **OKTA API**, this request will be proxied by Tyk: + +<img src="/img/dcr/okta/step_15.png" alt="Step 15" /> diff --git a/tyk-developer-portal/tyk-portal-classic/portal-concepts.mdx b/tyk-developer-portal/tyk-portal-classic/portal-concepts.mdx new file mode 100644 index 00000000..40589d30 --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/portal-concepts.mdx @@ -0,0 +1,94 @@ +--- +title: "Portal Concepts" +order: 1 +noindex: True +sidebarTitle: "Portal Concepts" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +## API Catalog + +The API Catalog is a list of APIs that you have published to your portal. + +The API Catalog entry is not a one-to-one map between an API you manage in Tyk, since you might want to compose multiple managed services into a single public-facing API Facade, a catalog entry is actually an entry that maps against a security policy. + +From the API Catalog, a user can either: + +- View the documentation for the API +- Request for a token to the API + +When a developer requests a token, a new Auth token is generated on the linked policy, instead of the actual API, since you may wish to publish multi-tier access to the same API (E.g. Bronze / Silver / Gold). + +## Key Requests + +A key request is a record that is generated when a developer requests an access token for an API published in the API Catalog. The Key request encompasses the following information: + +Read more about them in the [Key Request section](/tyk-developer-portal/tyk-portal-classic/key-requests) + +### Multiple APIs for a single Key Request + +New for v1.9, a developer can now request access to multiple APIs with a single key request. The APIs you group together via a single key should all be of the same authentication type. + +<img src="/img/dashboard/portal-management/multi-api-per-request.png" alt="Multiple APIs per Key Request" /> + +To enable this functionality, select **Enable subscribing to multiple APIs with a single key** from the Portal Management Settings. + +<img src="/img/dashboard/portal-management/multi-api-setting.png" alt="Multiple APIs" /> + +### Edit APIs associated with a single Key Request + +New for v1.9.4, if you have **Enable subscribing to multiple APIs with a single key** selected you can edit the APIs associated with the Key. You can perform the following: + +* Remove access to existing APIs +* Subscribe to new APIs (of the same authentication type as the existing ones). + + <img src="/img/dashboard/system-management/modify_key_approval.png" alt="Edit APIs" /> + + +If a new API requires key approval, the new key request will be generated, and access to this API will be granted after your admin approves it. + + +## Policies + +In the context of the developer portal, a security policy is the main "element" being exposed to public access. The policy is the same as a standard policy, and the policy forms the baseline template that gets used when the portal generates a token for the developer. + +Security policies are used instead of a one-to-one mapping because they encapsulate all the information needed for a public API program: + +1. Rate limits +2. Quota +3. Access Lists (What APIs and which versions are permitted) +4. Granular access (Which methods and paths are allowed, e.g. you may want to only expose read-only access to the portal, so only GET requests are allowed) +5. Multi-policy-management (With a Key, you can assign more than one policy to an APIs and each policy will have it's own counter). + +Within the developer portal admin area, under a developer record, you will see their subscriptions. Those subscriptions represent the tokens they have and their policy level access. It is possible to then "upgrade" or "downgrade" a developers access without actually managing their token, but just assigning a new policy to that token. + +## Documentation + +Within the portal, documentation is what a developer can use to learn how to access and use your APIs. + +The developer portal supports two types of documentation, and will render them differently: + +1. API Blueprint - this is rendered to HTML templates using Jade and Aglio. +2. Swagger/OpenAPI (OpenAPI 2.0 and 3.0 are supported) - either by pasting your Swagger JSON or YAML content into the code editor, or by linking to any public facing Swagger URL. The URL version can be rendered using [Swagger UI](https://swagger.io/tools/swagger-ui/) which offers a sandbox environment where developers can interact with your API from the browser. + + + + <Note> + Support for API Blueprint is being deprecated. See [Importing APIs](/api-management/gateway-config-managing-classic#api-blueprint-is-being-deprecated) for more details. + </Note> + + +Within an API Catalog entry, documentation must be attached to the catalog entry for it to be published. + +## Developers + +Within the developer portal, a developer is an end-user that has access to the developer portal section of the portal website. This user is completely separate from Tyk Dashboard users and they do not ever intersect (they are also stored separately). + +A developer record consists of some basic sign-up information and a set of admin-definable fields that get attached to the developer as metadata. + +Within the developer view of the Tyk Dashboard, it is possible to manage all access of a developer, including the access levels of their tokens. + + diff --git a/tyk-developer-portal/tyk-portal-classic/portal-events-notifications.mdx b/tyk-developer-portal/tyk-portal-classic/portal-events-notifications.mdx new file mode 100644 index 00000000..5d4addb4 --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/portal-events-notifications.mdx @@ -0,0 +1,111 @@ +--- +title: "Portal events and notifications" +order: 9 +noindex: True +sidebarTitle: "Events and Notifications" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +Tyk enables you to actively monitor both user and organization quotas. These active notifications are managed in the same way as webhooks and provides an easy way to notify your stakeholders, your own organization or the API end user when certain thresholds have been reached for their token. + +### Tyk Cloud Users + +Monitors are disabled by default in Tyk Cloud. Portal events are enabled and can be defined by raising a support ticket. + +### How to Enable Monitors + +See [Monitors](/api-management/gateway-events#monitoring-quota-consumption) for details of how to configure quota consumption monitors. + +### Portal Events + +The Tyk Dashboard and the Portal now support email notifications powered by Mandrill, Sendgrid, Mailgun and Amazon SES. + +#### How Email Notifications Work + +If you have enabled email notifications, the Portal will attempt to send notifications regarding a user's sign-up status or key request status to their username email address. These templates can be found in the `portal/email_templates` folder. + +The templates are available as text based or HTML. See the standard included ones to see the various template fields that can be customized. + +### Extra Dashboard And Portal Events + +The Dashboard and Portal also support a certain level of events that you can use to notify your system of various things that have happened in the Portal. + +To configure them, add an `event_options` section to an Organization when you are creating them. See [Creating an Organization via the Dashboard Admin API](/api-management/dashboard-configuration#create-an-organization) for more details. + +Within this object, you can then register webhooks or/and an email address to notify when an event occurs: + +```{.copyWrapper} expandable +event_options: { + api_event: { + webhook: "http://posttestserver.com/post.php?dir=tyk-events", + email: "test@test.com" + }, + key_event: { + webhook: "http://posttestserver.com/post.php?dir=tyk-key-events", + email: "test@test.com" + }, + key_request_event: { + webhook: "http://posttestserver.com/post.php?dir=tyk-key-events", + email: "test@test.com" + } +} +``` + +The following events are supported: + +* `api_event`: When an API is created, updated or deleted. + +* `key_event`: When a key is created, updated or deleted. + +* `key_request_event`: When a Portal key request is created or updated. + +Sample **Webhook** Payload for a **Key Request** Event: +```{.json} expandable +{ + "event": "key_request_event.submitted", + "data": { + "id": "5e543dd0f56e1a4affdd7acd", + "org_id": "5e2743567c1f8800018bdf35", + "for_plan": "5e2744897c1f8800018bdf3b", + "apply_policies": [ + "5e2744897c1f8800018bdf3b" + ], + "by_user": "5e430ef68131890001b83d2e", + "approved": false, + "date_created": "2020-02-24T16:19:12.175113-05:00", + "portal_developer": { + "id": "5e430ef68131890001b83d2e", + "email": "dev@dev.ca", + "date_created": "2020-02-11T15:30:46.003-05:00", + "inactive": false, + "org_id": "5e2743567c1f8800018bdf35", + "keys": { + "6dc2dfc0": [ + "5e431f938131890001b83d30" + ] + }, + "subscriptions": { + "5e431f938131890001b83d30": "6dc2dfc0" + }, + "last_login_date": "2020-02-11T16:43:39.858-05:00" + }, + "catalogue_entry": { + "name":"frontend APIs", + "short_description":"", + "long_description":"", + "show":true, + "api_id":"", + "policy_id":"5e2744897c1f8800018bdf3b", + "documentation":"5e3b477a7c1f8800013603c6", + "version":"v2", + "is_keyless":false, + "config":{ + + } + } + } +} +``` \ No newline at end of file diff --git a/tyk-developer-portal/tyk-portal-classic/portal-oauth-clients.mdx b/tyk-developer-portal/tyk-portal-classic/portal-oauth-clients.mdx new file mode 100644 index 00000000..00daaaf8 --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/portal-oauth-clients.mdx @@ -0,0 +1,54 @@ +--- +title: "Portal OAuth Clients" +order: 10 +noindex: True +sidebarTitle: "Portal OAuth Clients" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +From Tyk Dashboard v1.8, you can now create and manage OAuth clients from the Developer Portal. + +## Prerequisites + +1. An API created in your Dashboard using Tyk's ability to act as a OAuth provider. You need to have [OAuth 2.0](/api-management/authentication/oauth-2) selected as the Authentication mode. See [Create an API](/api-management/gateway-config-managing-classic#create-an-api) for more details. +2. A Policy created in your Dashboard with the API created above selected in the **Access Rights > Add access rule** drop-down. See [Create a Security Policy](/api-management/gateway-config-managing-classic#secure-an-api) for more details. +3. A Portal Catalog entry for the API created above with the Policy you created selected from the **Available policies** drop-down. See [Create a Portal Entry](/getting-started/tutorials/publish-api) for more details. +4. A developer account created in your Developer Portal. + +## Create the OAuth Client from the Portal + +1. Login to your Portal: + +<img src="/img/dashboard/portal-management/dev_portal_homev1.8.png" alt="Developer Portal Home Screen" /> + +2. Select **OAuth Clients** from the top menu +3. If this is the first OAuth Client you are creating, the screen will be as below: + +<img src="/img/dashboard/portal-management/portal_first-oauth_client.png" alt="Developer OAuth Home Screen" /> + +4. Click **Create first OAuth Client** +5. Hover over the API you added to the Catalog with OAuth Authentication mode from the drop-down list: + +<img src="/img/dashboard/portal-management/portal_oauth_select_api2.png" alt="Select API Screen" /> + +6. Click **Select API** +7. Then click **Save and continue**: + +<img src="/img/dashboard/portal-management/portal_oauth_connected_api2.png" alt="Save" /> + +8. You can now add details about your application, and set the redirect URL to the application. If you want to use this client for more than one application, you can add other redirect URLs as necessary. +9. Click **Create** + +<img src="/img/dashboard/portal-management/create_portal_oauth_client.png" alt="Create" /> + +10. You need to copy and save the displayed Client Secret, as you will not be able to view it from the Portal again. The secret is stored on the Dashboard and are listed for each developer under the **Portal Management > Developers** menu. + +<img src="/img/dashboard/portal-management/oauth_client_secrets.png" alt="secret" /> + + +## Revoke OAuth Client Tokens + +See [Revoke OAuth Tokens](/api-management/authentication/oauth-2#revoking-access-tokens) for more details. \ No newline at end of file diff --git a/tyk-developer-portal/tyk-portal-classic/streams.mdx b/tyk-developer-portal/tyk-portal-classic/streams.mdx new file mode 100644 index 00000000..1b65d2a1 --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/streams.mdx @@ -0,0 +1,18 @@ +--- +title: "Streams" +'og:description': "How to publish Tyk Streams APIs to your Tyk Developer Portal" +tags: ['streaming', 'events', 'event-driven architecture', 'event-driven architectures', 'kafka'] +order: 12 +noindex: True +sidebarTitle: "Streams with Classic Portal" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +As of Tyk v5.7.0, you can now publish Tyk Streams APIs to the Tyk Developer Portal. + +## How To Set Up + +Simply create a [Tyk Streams API](/api-management/event-driven-apis#getting-started), create a [Policy](/api-management/gateway-config-managing-classic#create-a-security-policy-with-the-dashboard) which protects it, and then [publish it to the Developer Portal Catalog](/portal/overview/getting-started#publish-an-api-product). \ No newline at end of file diff --git a/tyk-developer-portal/tyk-portal-classic/tyk-portal-classic/customise/customise-with-templates.mdx b/tyk-developer-portal/tyk-portal-classic/tyk-portal-classic/customise/customise-with-templates.mdx new file mode 100644 index 00000000..47a79014 --- /dev/null +++ b/tyk-developer-portal/tyk-portal-classic/tyk-portal-classic/customise/customise-with-templates.mdx @@ -0,0 +1,106 @@ +--- +title: "Customize Page Templates" +order: 2 +noindex: True +sidebarTitle: "Customise Page Templates" +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + +The Tyk Developer Portal can be fully customized using templates. The templates for the Portal are only available to Self-Managed users currently. These templates are located in the `/opt/tyk-dashboard/portal` folder of your Tyk installation. + +All templates are based on Twitter Bootstrap and are standard HTML with some Golang Template snippets to handle dynamic content rendering. + + +<Note> +The Portal process (`tyk-analytics`) must be restarted for template changes to take effect. This is because the application caches templates on startup. +</Note> + + + +### Adding new templates + +The Tyk content editor enables you to specify a template name to use when rendering templates. two are provided by default: + +* Default Home Page Template +* Default Page Template + +The third option is "Custom" and this allows you to enter a template name into the field editor that will set the template name to use on render. + +To set a new template name up in your Tyk installation, you will need to add the file to the `portal` folder and ensure it starts and ends with the templates directive: + +``` +{{ define "customPage" }} +<!--- HTML IN HERE --/> +{{ end }} +``` + +In the above snippet, we've created the `customPage` template, this can then be used as the template name in the CMS form when generating for the page type. + +### Content fields + +You'll notice that in existing templates, certain fields are mapped out as content: + +``` +{{.Page.Fields.PanelThreeContent | markDown }} +``` + +When you generate your own templates, you can use your own field names so that you can manage the content from inside the CMS: + +<img src="/img/dashboard/portal-management/page_settings_2.5.png" alt="Custom templates section" /> + +In the above example, you would be able to read the `MyContent` field by requesting it from the page data in the template like so: + +``` +{{.Page.Fields.MyContent| markDown }} +``` + +You do not need to pipe the content through the Markdown filter, but it is advised as it gives you a lot more freedom with regards to how to style content blocks. + +In existing page types, these content fields are already set out. + +### Dynamic Customization + +Portal templates now have access to the Developer object, its subscriptions and issued keys meta-data, providing the ability to conditionally show or hide content inside the Portal, based on the attributes described below. + +The current logged in Developer can be accessed using `.UserData` variable with the following fields: + +* Id - Internal developer ID +* Email - Developer email +* OrgID - Tyk Organization ID +* Subscriptions - Map containing subscriptions where key is a policy ID and value is an API key +* Fields - Map containing custom developer fields +* OauthClients - Map containing list of registered oAuth clients, where Key is the policy ID. + +The current logged in Developer detailed subscription object can be accessed using the `.APIS` variable, containing map, where the key is PolicyID and value of the following format: + +* APIDescription - API definition + * ID - Internal API id + * Name - API name + * More fields: https://github.com/TykTechnologies/tyk/blob/master/apidef/api_definitions.go#L320 +* APIKey - API key +* PolicyData - Policy object + * ID - Internal Policy ID + * Name - Policy Name + * More fields: https://github.com/TykTechnologies/tyk/blob/master/user/policy.go#L5 +* KeyMetaData - Key metadata of map type + +## Example +You have different teams of developers, and for each team we want to show them a different list of APIs. In this case, for each developer, we need to set a custom `team` field, and assert it in a template like this: + +``` expandable +{{if .UserData.Fields.Team == `internal`}} +… Display internal APIs … +{{end}} +{{if .UserData.Fields.Team == `public`}} +… Display public set of APIs … +{{end}} +``` + +### Add Files for Downloading + +If you want to have files available for download from your portal you can add them to your `/tyk-dashboard/portal/portal-assets` directory. + +You can then refer to them by using a `/opt/tyk-dashboard/...` link. diff --git a/tyk-gateway-api.mdx b/tyk-gateway-api.mdx new file mode 100644 index 00000000..0e30a29b --- /dev/null +++ b/tyk-gateway-api.mdx @@ -0,0 +1,7 @@ +--- +title: "Tyk Gateway API" +tags: ['OpenAPI Spec', 'OpenAPI Specification', 'OAS', 'REST', 'Tyk Gateway OpenAPI Spec', 'Tyk Gateway OAS', 'API Gateway OAS', 'API Gateway REST'] +order: 3 +sidebarTitle: "Gateway" +--- + diff --git a/tyk-governance/api-labeling.mdx b/tyk-governance/api-labeling.mdx new file mode 100644 index 00000000..ccf7ee14 --- /dev/null +++ b/tyk-governance/api-labeling.mdx @@ -0,0 +1,175 @@ +--- +title: "API Labeling and Categorization" +tags: ['Tyk Governance', 'API Labeling', 'API Categorization'] +sidebarTitle: "API Labeling and Categorization" +--- + +[Overview](#overview) | [Quick Start](#quick-start) | [How It Works](#how-it-works) | [Use Cases](#use-cases) | [Best Practices](#best-practices-and-recommendations) | [FAQs](#faqs) | [Troubleshooting](#troubleshooting) + +## Overview + +API Labeling and Categorization enables you to organize, classify, and filter your APIs using customizable metadata tags. This feature allows you to create a structured taxonomy for your API landscape, making it easier to search, filter, and apply governance policies based on business context, technical characteristics, or organizational ownership. + +### Key Benefits + +- Enables structured organization of APIs by business domain, criticality, and other dimensions +- Facilitates efficient search and filtering of APIs in large inventories +- Provides consistent metadata across APIs from different sources +- Supports governance policy application based on API characteristics (Governance Policy feature is coming soon) +- Enables reporting and analytics based on business context (Reporting feature is coming soon) + +## Quick Start + +In this tutorial, we'll explore how to use API labeling to categorize and filter APIs in your organization's API Repository. + +### Prerequisites + +- Access to the Tyk Governance Hub +- Governance Admin access for creating new label definitions (Note: only admin access is available at the moment) + +### Step-by-Step + +1. **Access the API Repository** + + Navigate to the API Repository section in your Tyk Governance dashboard. + +2. **Explore default labels** + + Tyk Governance comes with pre-configured default labels such as "Business Domain" and "API Criticality". + +3. **Apply labels to APIs** + + Select an API and click "Edit" to apply or modify labels: + + - Set "Business Domain" to an appropriate value (e.g., "Finance", "Customer", "Product") + - Assign "API Criticality" based on the API's importance (Tier 1 for mission-critical, Tier 2 for important, Tier 3 for non-critical) + - Add any custom labels that have been defined by your Governance Admin + +4. **Filter APIs using labels** + + Use the search and filter functionality to find APIs based on their labels: + + - Filter to show only Tier 1 APIs + - Search for APIs in a specific business domain + - Combine multiple label filters for precise results + +5. **Create a custom label (Admin only)** + + Governance Admin users can create custom labels programmatically using the API: + + Example using cURL: + +```bash expandable + curl -X POST https://your-governancel-instance.tyk.io/api/labels/ \ + -H "Content-Type: application/json" \ + -H "X-API-Key: YOUR_ADMIN_TOKEN" \ + -d '{ + "name": "compliance", + "values": ["PCI-DSS", "GDPR", "HIPAA"] + }' +``` + +A successful request will return a 200 OK status code and the newly created label object: + +```json + { + "id": "64a1b2c3d4e5f6a7b8c9d0e1", + "name": "compliance", + "values": ["PCI-DSS", "GDPR", "HIPAA"] + } +``` + +**Notes**: +- The name field is required and must be unique +- The values field is optional. If provided, it defines the allowed values for this label +- If values is empty, the label will accept any value (free text) +- Only users with admin privileges can create labels +- Once created, labels can be applied to APIs using the `/api/{api-id}/labels` endpoint +- After creating a custom label, it will be available for selection when labeling APIs, either through the UI or via the API labeling endpoints. + +### Validation + +- Labeled APIs will display their labels in the API details view +- Filtering by labels will show only matching APIs +- New custom labels will be available for application to APIs + +## How It Works + +API Labeling and Categorization works through a flexible key-value metadata system that allows both structured and free-form classification of APIs. + +### Labeling System Architecture + +1. **Bootstrap Default Labels**: During initial setup, Tyk Governance creates default label definitions such as "Business Domain" and "API Criticality". +2. **Label Definition**: Each label has: + - A unique key (e.g., "business_domain") + - A display name (e.g., "Business Domain") + - A value type (free text or predefined values) + - Optional predefined values (e.g., "Finance", "HR", "Operations") + +3. **Label Application**: Labels are applied to APIs as key-value pairs: + - Key: The label identifier (e.g., "business_domain") + - Value: The specific value for this API (e.g., "Finance") + +4. **Label Storage**: Labels are stored as metadata with each API in the repository database. +5. **Search and Filter**: Tyk Governance indexes labels to enable efficient filtering and searching. + +## Use Cases + +### Governance Policy Application + +Apply different governance rules based on API criticality tiers. For example, Tier 1 (mission-critical) APIs might require stricter security controls, more thorough documentation, and formal change management processes. + +### Compliance Management + +Tag APIs with relevant compliance requirements (PCI-DSS, GDPR, HIPAA) to ensure appropriate controls are applied and to facilitate compliance reporting and audits. + +### Team Ownership and Responsibility + +Label APIs by owning team or department to clarify responsibility for maintenance, support, and governance compliance. + +### API Lifecycle Management + +Use labels to indicate lifecycle stage (Development, Testing, Production, Deprecated) to manage API transitions and communicate status to consumers. + +## Best Practices and Recommendations + +- **Establish a clear labeling taxonomy** before implementing across your organization +- **Keep predefined value lists manageable** – too many options create confusion and inconsistency +- **Use hierarchical naming for related labels** (e.g., security.authentication.method, security.data.classification) +- **Document the meaning and intended use** of each label for consistent application +- **Assign label management responsibility** to a specific role or team to maintain consistency +- **Review and update labels periodically** to ensure they remain relevant as your API landscape evolves +- **Include label application in API onboarding workflows** to ensure consistent metadata from the start +- **Use consistent labeling conventions** across all APIs to facilitate effective filtering and governance +- **Combine multiple labels in filters** for more precise API discovery +- **Use criticality and domain labels** as the foundation of your governance strategy + +## FAQs + +<AccordionGroup> +<Accordion title={'Can I create custom labels with my own predefined values?'}> +Yes, Governance Administrators can create custom labels with either free text values or a predefined list of acceptable values. +</Accordion> + +<Accordion title={'How do labels differ from tags?'}> +Labels are structured key-value pairs that can be validated and used for governance, while tags are typically simpler, unstructured text values used primarily for search. +</Accordion> + +<Accordion title={'Are labels from source systems preserved during discovery?'}> +Yes, the discovery process attempts to map source system metadata to corresponding labels in the governance hub where possible. +</Accordion> +</AccordionGroup> + +## Troubleshooting + +<Expandable title={'Labels not appearing in filter options'}> +- Ensure the label has been properly defined by a Governance Admin +- Check that at least one API has been tagged with this label +- Refresh the browser cache if the label was recently added +</Expandable> + +<Expandable title={'Cannot add a specific label value'}> +- For predefined value labels, check that the value you're trying to add is in the allowed list +- Verify you have sufficient permissions to modify the API's labels +- Ensure the label hasn't been deprecated or replaced +</Expandable> \ No newline at end of file diff --git a/tyk-governance/api-repository.mdx b/tyk-governance/api-repository.mdx new file mode 100644 index 00000000..f52d66d3 --- /dev/null +++ b/tyk-governance/api-repository.mdx @@ -0,0 +1,126 @@ +--- +title: "API Repository" +tags: ['Tyk Governance', 'API Repository'] +sidebarTitle: "Federated API Repository" +--- + +[Overview](#overview) | [Quick Start](#quick-start) | [How It Works](#how-it-works) | [Use Cases](#use-cases) | [Best Practices](#best-practices-and-recommendations) | [FAQs](#faqs) | [Troubleshooting](#troubleshooting) + +## Overview + +API Repository automatically discovers and catalogs APIs across multiple sources (Tyk, AWS API Gateway, etc) to create a comprehensive inventory of all APIs in your organization. This feature addresses API sprawl, identifies shadow APIs, and provides complete visibility into your API landscape. + +### Key Benefits + +- Creates a single source of truth for all APIs across the organization +- Identifies security risks from undocumented or unmanaged APIs +- Enables better resource management and prevents duplication +- Provides visibility into API ownership and usage patterns + +### Dependencies + +- Requires the governance agent for API discovery from non-Tyk Cloud managed control planes and non-Tyk platforms. + +## Quick Start + +In this tutorial, we'll explore how to use the API Repository to view and manage discovered APIs in your organization. + +### Prerequisites + +- Access to the Tyk Governance Hub +- Governance agent deployed and connected to your API providers (non Tyk Cloud sources only) + +For detailed installation and configuration instructions, please refer to the [Installation and Setup](/tyk-governance/installation) page. + +### Step-by-Step + +1. **Access the API Repository** + + Navigate to the API Repository section in your Tyk Governance Hub to view discovered APIs. + +2. **Explore the API inventory** + + The dashboard provides a comprehensive view of all discovered APIs across your organization, with filtering and search capabilities. + + <img src="/img/governance/api-list.png" alt="" /> + +3. **Examine API details** + + Click on any API to view detailed information including specifications, ownership, authentication methods, and governance status. + + <img src="/img/governance/api-details.png" alt="" /> + +## How It Works + +The API Repository works by deploying agents that connect to various API sources, extract metadata, and synchronize this information with the central governance hub. Think of it as an automated API census that continuously updates your API inventory. + +### Discovery Process + +1. **Agent Deployment**: Agents are deployed to connect with various API sources. +2. **API Source Connection**: Agents authenticate and connect to configured API sources. +3. **Metadata Extraction**: Agents extract API metadata including routes, authentication methods, and specifications. +4. **Synchronization**: Extracted data is sent to the governance hub through secure gRPC streams. +5. **Inventory Creation**: APIs are cataloged in a centralized repository with relevant metadata. +6. **Classification**: APIs can be tagged and categorized based on extracted and custom metadata. +7. **Continuous Updates**: Regular scans maintain an up-to-date inventory and identify changes. + +## Use Cases + +### Centralizing API Inventory Across Multiple Gateways + +When your organization uses multiple API gateways (Tyk, AWS, etc.), maintaining a single view of all APIs becomes challenging. API Discovery automatically aggregates APIs from all sources into a unified inventory, providing a complete picture of your API landscape without manual tracking. + +### Identifying and Managing Shadow APIs + +Shadow APIsβ€”those created outside official processesβ€”pose security and governance risks. The discovery feature continuously scans your infrastructure to identify undocumented APIs, allowing you to bring them under governance or decommission them as appropriate. + +### Streamlining API Onboarding with Automated Discovery + +For organizations with many APIs, manual registration is time-consuming and error-prone. Automated discovery accelerates the onboarding process by automatically detecting new APIs and pre-populating their metadata, reducing the time to bring APIs under governance. + +### Tracking API Changes for Compliance and Audit + +When APIs change without proper documentation, it creates compliance risks. The continuous discovery process detects changes to existing APIs, maintaining an accurate, up-to-date inventory that serves as an audit trail for compliance purposes. + +### Enabling API Reuse Through Comprehensive Cataloging + +Developers often recreate APIs because they're unaware of existing ones. A complete API inventory with rich metadata enables developers to discover and reuse existing APIs, reducing duplication and development costs. + +## Best Practices and Recommendations + +- **Configure all relevant API sources** to ensure complete coverage of your API landscape +- **Implement a review process** for newly discovered APIs to ensure proper classification and ownership assignment +- **Integrate discovery with your CI/CD pipeline** to automatically synchronize new APIs as they're deployed +- **Establish clear ownership** for each API to ensure accountability for governance and maintenance + +## FAQs + +<Expandable title={'How secure is the API discovery process?'}> +The discovery process uses secure authentication methods for each provider and transmits data via encrypted channels. The agent requires minimal permissionsβ€”just enough to read API configurations. +</Expandable> + +<Expandable title={'Will discovery impact the performance of my API gateways?'}> +The discovery process is designed to be lightweight and non-intrusive. It primarily reads configuration data rather than analyzing traffic, minimizing any performance impact. +</Expandable> + +## Troubleshooting + +<AccordionGroup> +<Accordion title={"APIs from a specific source aren't being discovered"}> +- Check the agent logs for authentication errors +- Verify the provider configuration in the governance agent config +- Ensure the agent has network access to the API source +</Accordion> + +<Accordion title={'Discovered APIs are missing metadata'}> +- Some API sources may not expose all metadata +- Check if the API definition in the source is complete +- Consider enhancing the API definition at the source +</Accordion> + +<Accordion title={'Agent fails to connect to the governance hub'}> +- Verify the governance URL and token in the agent configuration +- Check network connectivity between the agent and governance hub +- Examine the agent logs for specific connection errors +</Accordion> +</AccordionGroup> diff --git a/tyk-governance/core-concepts.mdx b/tyk-governance/core-concepts.mdx new file mode 100644 index 00000000..78792839 --- /dev/null +++ b/tyk-governance/core-concepts.mdx @@ -0,0 +1,288 @@ +--- +title: "Core Concepts" +tags: ['Tyk Governance'] +sidebarTitle: "Core Concepts" +--- + +This section provides a detailed explanation of the key technical concepts that form the foundation of Tyk Governance. + +## What is Tyk Governance? + +Tyk Governance is a comprehensive API Governance hub designed to provide centralized visibility, control, and policy enforcement across distributed API ecosystems. It enables organizations to establish and maintain consistent standards, security practices, and compliance requirements across multiple API gateways and management platforms. + +At its core, Tyk Governance is a federated control plane that sits above your existing API infrastructure, regardless of whether you're using Tyk exclusively or a mix of different API management solutions. It collects, analyzes, and governs API definitions from various sources, ensuring they adhere to your organization's standards and best practices. + +```mermaid +flowchart LR + subgraph "Tyk Governance hub" + GS["Governance Service"] --- RE["Rule Engine"] + GS --- AR["API Repository"] + GS --- RP["Reporting"] + end + + subgraph "API Ecosystem" + TG["Tyk Dashboard"] --- AG1["Agent"] + AWS["AWS API Gateway"] --- AG2["Agent"] + OTHER["Other sources (Future support)"] --- AG3["Agent"] + end + + AG1 --> GS + AG2 --> GS + AG3 --> GS +``` + +## Federated API Management + +Federated API management refers to the practice of managing APIs across multiple, distributed platforms while maintaining consistent governance, visibility, and control. + +Organizations struggle with API sprawl, with many lacking visibility into their total number of APIs. Multiple gateways across teams create security vulnerabilities, governance gaps, and inefficiency. This "API debt" results in inconsistent security protocols, missed reuse opportunities, and increased risk from shadow APIs. Large enterprises need a solution that balances centralized governance with team autonomyβ€”enabling visibility across different gateways while allowing teams to use their preferred tools. The ideal approach provides unified oversight without forcing consolidation, ensuring compliance while preserving innovation and operational independence. + +### Tyk Governance Solutions + +Tyk Governance addresses these challenges through: + +1. **Unified API Repository**: A central inventory of all APIs across different providers. +2. **Cross-Platform Policy Enforcement**: Consistent application of governance policies regardless of the underlying API provider. +3. **Automated Compliance Checking**: Continuous validation of APIs against organizational standards and regulatory requirements. +4. **Maturity Assessment**: Evaluation and scoring of APIs based on design, security, documentation, and performance criteria. +5. **Centralized Reporting**: Comprehensive visibility into API compliance and governance status. + +## Governance Rulesets + +A governance ruleset in Tyk Governance is a set of rules and standards that APIs must adhere to. These rulesets define the requirements for API design, security, documentation, and operational characteristics. + +### Ruleset Components + +1. **Rules**: Individual checks that validate specific aspects of an API definition. +2. **Severity Levels**: Categorization of rules by importance (error, warning, info). +3. **Validation Functions**: The specific logic used to evaluate API definitions against rules. +4. **Remediation Guidance**: Instructions on how to fix issues when rules are violated. + +### Spectral Ruleset Compatibility + +Tyk Governance rulesets are compatible with the [Spectral ruleset](https://meta.stoplight.io/docs/spectral/01baf06bdd05a-rulesets) format, a widely adopted API linting and governance standard. This compatibility offers several advantages: + +1. **Familiar Format**: Teams already using Spectral can easily migrate their existing rulesets. +2. **Ecosystem Integration**: Leverage the broader ecosystem of pre-built Spectral rules. +3. **Extensibility**: Create custom rules using the same format and functions as Spectral. +4. **IDE Integration**: Use existing Spectral plugins for popular code editors. + +The Spectral-compatible format allows for declarative rule definitions with given/then patterns, custom functions, and detailed error messaging. + +### Basic Ruleset Examples + +```yaml expandable +# Security ruleset requiring HTTPS +owasp-security-hosts-https-oas3: + description: All server interactions MUST use the https protocol + severity: error + then: + function: owaspHostsHttps + +# Rate limiting ruleset +rate-limit-exists: + description: Ensure rateLimit exists under upstream + severity: error + given: "$['x-tyk-api-gateway'].upstream" + then: + - field: rateLimit + function: truthy +``` + +Rulesets can be customized to meet organizational needs and evolve as API best practices and security requirements change. + +## Supported API Providers + +Tyk Governance is designed to work with a wide range of API management platforms, allowing organizations to maintain governance regardless of their existing API infrastructure. + +### API Provider Compatibility + +| API Provider | Tested Version | Supported API Types | Supported Features | +| :--------------- | :-------------- | :------------- | :------------------------------------------------- | +| Tyk Dashboard | 5.3+ | Tyk OAS | Complete integration with all governance features | +| AWS API Gateway | All | Rest APIs | API definition export, OAS schema export | +| Azure | - | - | Coming Soon | +| Kong | - | - | Coming Soon | +| WSO2 | - | - | Coming Soon | + +### Integration Capabilities + +Tyk Governance integrates with these providers through specialized agents that: + +1. Connect to the platform's management APIs +2. Extract API definitions and configurations +3. Convert proprietary formats to OpenAPI Specification (OAS) +4. Apply Tyk-specific extensions where applicable +5. Synchronize definitions with the central governance repository + +### Future API Provider Support + +The Tyk Governance roadmap includes plans to expand support to additional platforms and API Types. + +## How It Works + +Tyk Governance operates through a distributed architecture that combines a centralized cloud-hosted governance service with distributed agents that run in your environments. + +### Technical Architecture + +```mermaid +flowchart LR + subgraph "Tyk Cloud" + GS["Governance Service"] --- DB[(Database)] + GS --- RE["Rule Engine"] + GS <--> A4["Agent 4"] --- P4["Cloud Control Plane"] + end + + subgraph "Customer Environment" + A1["Agent 1"] --- P1["Tyk Dashboard (Self-Managed)"] + A2["Agent 2"] --- P2["AWS API Gateway"] + end + + A1 <-->|"TLS + Auth"| GS + A2 <-->|"TLS + Auth"| GS +``` + +### Hosted Service Model + +The Governance Core is a Tyk Cloud hosted and managed service, providing several benefits: + +1. **Zero Infrastructure Overhead**: No need to deploy and maintain governance infrastructure. +2. **Automatic Updates**: Always access the latest features and security patches. +3. **Scalable Performance**: Handles growing API ecosystems without additional configuration. +4. **High Availability**: Built-in redundancy and failover capabilities. + +### Customer-Hosted Agents + +While the core service is cloud-hosted, customers can host their own agents within their environments: + +1. **Credential Isolation**: Tyk Governance never directly accesses your API providers; all credentials remain within your environment. +2. **Network Security**: The agent requires accepting inbound traffic from the cloud-based governance dashboard. All communication between agents and the dashboard is secured via TLS encryption. +3. **Deployment Flexibility**: Deploy agents in any environment where they can access your API platforms. +4. **Lightweight Footprint**: Agents have minimal resource requirements and can run in containers or VMs. + +### Process Sequence + +```mermaid +sequenceDiagram + participant Agent + participant Governance + participant RuleEngine + + Agent->>Governance: Register with governance hub + Governance->>Agent: Issue authentication token + Governance->>Agent: Send sync request + Agent->>Agent: Get APIs from API providers + Agent->>Governance: Stream API definitions + Governance->>Governance: Store in repository + Governance->>RuleEngine: Validate against rules + RuleEngine->>Governance: Return validation results + Governance->>Governance: Generate reports +``` + +### Synchronization Mechanisms + +Tyk Governance uses a secure bidirectional streaming protocol for efficient synchronization: + +1. **Registration**: Agents register with the Governance hub and establish a secure connection. +2. **Heartbeat**: Agents maintain a health check stream to indicate their status. +3. **Sync Request**: The Governance hub can trigger a sync operation on demand or on schedule. +4. **Streaming Response**: Agents stream API definitions back to the governance hub as they are extracted. +5. **Incremental Updates**: Only changed APIs are synchronized to minimize network traffic. + +### Security Measures + +The synchronization between agents and the Governance service includes multiple security layers: + +1. **TLS Encryption**: All communications are encrypted using TLS 1.2+ to prevent eavesdropping. +2. **Authentication Tokens**: Agents authenticate using secure tokens that can be rotated and revoked. +3. **Minimal Privilege**: Agents use read-only access to API platforms whenever possible. +4. **Data Minimization**: Only API definitions and metadata are transmitted, not actual API traffic or payloads. +5. **Audit Logging**: All synchronization activities are logged for security monitoring. + +### Data Exchange + +The information exchanged between agents and the Governance service includes: + +1. **From Agent to Governance**: + - API definitions in OpenAPI format + - API metadata (name, version, endpoints, security schemes) + - Provider-specific configuration converted to standard formats + - Agent status and capability information + - Sync operation status and results + +2. **From Governance to Agent**: + - Sync requests and configuration + - Authentication tokens and renewal information + +Notably, the following are NOT transmitted: + +- API keys or credentials for accessing APIs +- Actual API request/response payloads +- Customer data processed by APIs +- Internal network information beyond what's in API definitions + +## Glossary of Terms + +### Agent + +A component that connects to API Providers (Tyk, AWS API Gateway, etc.) to extract and sync API definitions. + +### API Maturity + +A measure of how well an API adheres to best practices in design, security, documentation, and performance. + +### API Provider + +A system or platform where APIs are hosted or managed, which Tyk Governance can discover and monitor. Examples include Tyk Dashboard and AWS API Gateway. + +### API Repository + +A federated catalog that aggregates APIs from multiple API providers, providing a centralized view of all APIs within an organization. + +### Federated API Management + +An approach to managing APIs across multiple platforms and environments while maintaining centralized governance. + +### Label + +A key-value pair assigned to APIs or API providers for categorization and governance purposes. Examples include `domain:storefront`, `environment:production`, or `pii:true`. + +### Ruleset + +A collection of governance rules that can be applied to APIs to enforce best practices and compliance requirements. + +### Rule + +A specific condition that can be evaluated against APIs to ensure they meet governance standards. Rules include severity levels, messages, and descriptions. + +### Ruleset Template + +A predefined ruleset containing common governance rules that can be applied as a starting point for governance policies. + +### Governance Report + +A summary of API compliance with governance rules, identifying violations and suggesting remediations. + +### Violation + +An instance where an API fails to meet defined governance standards, categorized by severity level. + +### Compliance + +The degree to which an API adheres to defined governance policies and standards. + +### Remediation + +The structured process of addressing and resolving API governance violations. + +### Risk Level + +A summary metric that reflects API governance compliance across multiple APIs, considering their adherence to selected governance rulesets. + +### Sync + +The process of extracting API definitions from management platforms and updating the governance repository. + +### Tyk-OAS Governance Extensions + +Tyk-specific extensions to the OpenAPI Specification that enable advanced governance features. \ No newline at end of file diff --git a/tyk-governance/installation.mdx b/tyk-governance/installation.mdx new file mode 100644 index 00000000..f2e910d4 --- /dev/null +++ b/tyk-governance/installation.mdx @@ -0,0 +1,498 @@ +--- +title: "Installation and Setup" +tags: ['Tyk Governance'] +sidebarTitle: "Installation" +--- + +This section moves from concepts to hands-on implementation, providing the practical steps needed to start with Tyk Governance. + +## Prerequisites + +Before beginning the installation and setup process for Tyk Governance, ensure your environment meets the following requirements: + +### License Requirements +- Valid Tyk license with Governance feature enabled + +### System Requirements + +**For Tyk Governance Hub:** +- No local system requirements as Tyk Governance is fully hosted and managed by Tyk in the cloud + +**For Tyk Governance Agent:** +- Minimum 1 CPU core +- 512MB RAM +- 1GB available disk space +- Linux-based operating system (Ubuntu 18.04+, CentOS 7+, or equivalent) +- Docker (if using containerized deployment) +- Kubernetes (optional, for orchestrated deployments) + +### Permission Requirements + +**For Agent Installation:** +- Read access to your API providers (Tyk, AWS API Gateway, etc.) +- Ability to create and manage containers or services in your environment +- Network configuration permissions to establish outbound connections +- Permission to create and manage secrets for storing API credentials + +### Network Requirements + +**For Tyk Governance Agent:** +- Inbound access from the Tyk Governance Hub (default 50051 for gRPC) +- Outbound access to your API providers +- Outbound HTTPS (port 443) access to the Tyk Governance Hub +- If API Provider gateways run on different networks, network routes must allow the agent to communicate with those networks + +## System Architecture + +Tyk Governance follows a cloud-hosted service model with customer-deployed agents, creating a secure and flexible architecture that respects your network boundaries while providing centralized governance. + +### High-Level Architecture + +```mermaid +flowchart LR + subgraph "Tyk Cloud" + GS["Governance Service"] --- DB[(Database)] + GS --- RE["Rule Engine"] + GS <--> A4["Agent 3"] --- P3["Cloud Control Plane"] + end + + subgraph "Customer Environment" + A1["Agent 1"] --- P1["Tyk Dashboard (Self-Managed)"] + A2["Agent 2"] --- P2["AWS API Gateway"] + end + + A1 <-->|"TLS + Auth"| GS + A2 <-->|"TLS + Auth"| GS +``` + +### Deployment Models + +#### Tyk Cloud with Automatic Agent + +```mermaid +flowchart LR + subgraph "Tyk Cloud" + GS["Governance Service"] + CP["Cloud Control Plane"] + AG["Auto-deployed Agent"] + + GS --- AG + AG --- CP + end + + User["User"] --> GS +``` + +**When to use this model:** +- You exclusively use Tyk Cloud for API management +- You want the simplest possible setup with minimal configuration +- You don't have any APIs on other platforms that need governance + +#### Tyk Cloud with Customer-Deployed Agents + +```mermaid +flowchart LR + subgraph "Tyk Cloud" + GS["Governance Service"] + end + + subgraph "Customer Environment" + A1["Agent"] --- TG["Tyk Dashboard (Self-Managed)"] + A2["Agent"] --- AWS["AWS API Gateway"] + end + + A1 <--> GS + A2 <--> GS + + User["User"] --> GS +``` + +**When to use this model:** +- You use self-managed Tyk deployments (not Tyk Cloud) +- You use AWS API Gateway or other supported providers +- You need to govern APIs across providers that aren't in Tyk Cloud + +#### Hybrid Deployment + +```mermaid +flowchart LR + subgraph "Tyk Cloud" + GS["Governance Service"] + CP["Cloud Control Plane"] + AG["Auto-deployed Agent"] + + GS --- AG + AG --- CP + end + + subgraph "Customer Environment" + A2["Agent"] --- AWS["AWS API Gateway"] + end + + A2 <--> GS + + User["User"] --> GS +``` + +**When to use this model:** +- You use a combination of Tyk Cloud and other API platforms +- You have a mix of cloud and on-premises API deployments +- You need comprehensive governance across your entire API ecosystem + +## Installation + +The installation process for Tyk Governance varies depending on whether you're an existing Tyk Cloud customer and which deployment model you use. + +### Requesting Access to Tyk Governance + +1. **Contact Tyk for Access** + - Reach out to your Tyk Account Manager or visit [tyk.io/contact-book-a-demo](https://tyk.io/contact-book-a-demo/) + - Specify that you're interested in access to Tyk Governance + - Provide information about your current API management environment + +2. **Receive Access Credentials** + - After your request is processed, you'll receive an email with: + - URL to access the Tyk Governance Hub + - Admin credentials for initial login + - Instructions for next steps + +3. **Initial Login** + - Navigate to the provided Governance hub URL + - Enter the admin credentials from the email + - You'll be prompted to change your password on first login + +### Enabling Governance Feature for Cloud Control Planes + +For existing Tyk Cloud managed control planes, enabling governance is straightforward: + +1. **Log in to Tyk Cloud Dashboard** + - Navigate to your Tyk Cloud dashboard + - Ensure you have administrative privileges + +2. **Access Control Plane Settings** + - Select the Control Plane you want to enable governance for + - Click on "Edit Details" button + +3. **Enable Governance Feature** + - Locate the "Governance Agent" toggle + - Enable the feature + - Save your changes + +4. **Verification** + - An agent will be automatically deployed for your Tyk Control Plane + - You can now access the Governance dashboard via "Governance" in the Cloud UI sidebar + +### Installing a Local Agent + +For environments where you need to install agents manually (non-Tyk platforms or on-premises deployments), follow these steps: + +**Prerequisites for Agent Installation:** +- Access to the Governance hub to generate agent tokens +- Network connectivity between the agent and both the Governance hub and your API provider +- Docker or Kubernetes for container-based deployment (recommended) + +#### Generate Agent Token + +Since the UI for generating agent tokens is unavailable, you'll need to use the API to create a token. After receiving your Governance hub credentials, follow these steps: + +1. **Obtain an API Key**: + - Log in to the Governance hub using the credentials provided in your welcome email + - Check your Access key under the "Settings > User Profile" section + + <img src="/img/governance/user-profile.png" alt="" /> + +2. **Create an Agent using the API**: + +```bash expandable +# Replace these values with your actual information +GOVERNANCE_URL="https://your-governance-instance.tyk.io" +API_KEY="your-access-key" +AGENT_NAME="My AWS Agent (US)" + +# Create agent first +curl -s -X POST --location "${GOVERNANCE_URL}/api/agents/" \ + -H "X-API-Key: ${API_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "'"${AGENT_NAME}"'" + }' +``` + +Example response which shows that an agent is created in INACTIVE state: + +```json +{"id":"a51d9bd0-bafe-4749-8285-e18641b151f2","name":"My AWS agent (US)","last_heartbeat":"0001-01-01T00:00:00Z","status":"INACTIVE","providers":null} +``` + +```bash +# Extract agent ID from response +AGENT_ID="a51d9bd0-bafe-4749-8285-e18641b151f2" +``` + +3. **Generate an Agent Token using the API**: + +Now you can generate an access token for the agent. + +```bash expandable +# API call to create an agent token +curl -X POST "${GOVERNANCE_URL}/api/auth/token/" \ + -H "X-API-Key: ${API_KEY}" \ + -H "Content-Type: application/json" \ + -d '{ + "agent_id": "'"${AGENT_ID}"'" + }' +``` + + Example response: + + ```json expandable + { + "token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...", + } + ``` + +4. **Save the token securely**: + + - Copy the `token` value from the response + - Store it securely, as you'll need it for agent configuration + - Note: This token cannot be retrieved later, so make sure to save it + +#### Prepare Configuration + +Create a configuration file named `agent-config.yaml` with the following structure: + +```yaml +#============================================================================== +# Tyk Governance Agent Configuration +#============================================================================== + +# Your Tyk Governance license key - required for agent authentication +# This is provided by Tyk when you subscribe to the Governance service +licenseKey: "your-tyk-governace-license-key" + +# Configuration for connecting to the Tyk Governance dashboard/service +governanceDashboard: + server: + # The gRPC endpoint URL of the Tyk Governance service + # Format: hostname:port (without protocol) + # This is in the format of prefixing "grpc-" to your Governance Hub URL. + url: "grpc-your-governance-instance.tyk.io:443" + + auth: + # Authentication token for this agent + # Generated via API call to /auth/token endpoint + # This token identifies and authorizes this specific agent + token: "my-agent-token" + +#============================================================================== +# API Provider Configurations +#============================================================================== +# List of API providers this agent will connect to +# Each agent can connect to multiple providers of different types +instances: + #-------------------------------------------------------------------------- + # Tyk Provider Configuration + #-------------------------------------------------------------------------- + - name: "tyk-provider" # Descriptive name for this provider instance + type: "tyk" # Provider type: must be "tyk" for Tyk Dashboard + config: + # The URL of your Tyk Dashboard + # For Kubernetes deployments, this might be an internal service URL + host: "http://dashboard-svc-tyk-stack-tyk-dashboard.tyk.svc.cluster.local:3000" + + # API key with read access to the Tyk Dashboard + # Can be obtained in Tyk Dashboard under "User" > "User Details": "Tyk Dashboard API Access Credentials" + # Requires read permissions for APIs and policies + auth: "your-auth-key" + + #-------------------------------------------------------------------------- + # AWS API Gateway Provider Configuration + #-------------------------------------------------------------------------- + - name: "aws-provider" # Descriptive name for this AWS API Gateway instance + type: "aws" # Provider type: must be "aws" for AWS API Gateway + config: + # AWS IAM credentials with permissions to list and get API Gateway resources + # Recommended: Use an IAM role with minimal required permissions + accessKeyId: "your-aws-access-key-id" + accessKeySecret: "your-aws-access-key-secret" + + # AWS region where your API Gateway APIs are deployed + # Example: us-east-1, eu-west-1, ap-southeast-2, etc. + region: "us-east-1" + + # Optional: Temporary session token if using temporary credentials + # Required only when using AWS STS temporary credentials + sessionToken: "your-aws-session-token" + +#============================================================================== +# Agent Settings +#============================================================================== + +# Log level controls verbosity of agent logs +# Options: debug, info, warn, error +# Recommended: info for production, debug for troubleshooting +logLevel: debug + +# Health probe configuration for monitoring agent health +# Used by container orchestration systems like Kubernetes +healthProbe: + server: + # Port on which the health probe server will listen + # Ensure this port is not used by other services + port: 5959 +``` + +#### Deploy the Agent + + **Docker Deployment:** + +```bash +# Replace it with your Tyk Governance license key +LICENSE_KEY="tyk-governance-license-key" +VERSION="latest" + +docker run -d --name tyk-governance-agent \ + -v $(pwd)/agent-config.yaml:/app/config.yaml \ + -e TYK_AGENT_LICENSEKEY="$LICENSE_KEY" \ + tykio/governance-agent:$VERSION +``` + + **Kubernetes Deployment:** + +1. Create a Kubernetes secret for the configuration: + +```bash +kubectl create secret generic agent-config \ + --from-file=config.yaml=./agent-config.yaml \ + -n your-namespace +``` + +2. Apply the following manifest: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: governance-agent + namespace: your-namespace # Replace with your namespace +spec: + replicas: 1 + selector: + matchLabels: + app: tyk-governance-agent + template: + metadata: + labels: + app: tyk-governance-agent + spec: + containers: + - name: agent + image: tykio/governance-agent:latest # Replace with an available version tag + env: + - name: TYK_AGENT_LICENSEKEY + value: your-governance-license #Replace with your license key + ports: + - name: health + containerPort: 5959 + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: health + readinessProbe: + httpGet: + path: /live + port: health + volumeMounts: + - mountPath: /app/config.yaml + name: agent-config + subPath: config.yaml + volumes: + - name: agent-config + secret: + secretName: agent-config + items: + - key: config.yaml + path: config.yaml + +``` + +Apply with: + +```bash +kubectl apply -f agent-deployment.yaml +``` expandable + +#### Verify Agent Connection + +1. Check if the agent is running properly: + + ```bash + # For Docker + docker logs tyk-governance-agent + + # For Kubernetes + kubectl logs -l app=tyk-governance-agent -n your-namespace + ``` + + 2. Look for log messages indicating a successful connection: + +``` +Starting license validation... +License validated successfully. Valid till: ... +starting agent +agent started successfully +waiting agent to establish health check +starting health probes HTTP server","addr":":5959 +authenticated and established health stream +health check established, waiting for sync stream +agent registered successfully and established sync stream with governance dashboard +waiting for sync requests from the dashboard +``` + +#### Trigger Initial Sync + +1. In the Governance hub, navigate to "API Repository" +2. Click the "ReSync" button to initiate synchronisation + +<img src="/img/governance/sync-repository.png" alt="" /> + +3. Monitor the sync progress in the UI or refresh the page manually. + +## Examples + +The following examples demonstrate common deployment scenarios and configurations for Tyk Governance. + +### Example 1: Tyk Cloud with Automatic Agent + +This is the simplest deployment model for existing Tyk Cloud customers. + +**Configuration Steps:** + +1. Requesting Access to Tyk Governance +2. Enable the Governance feature in Tyk Cloud Control Plane as described in [Enabling Governance Feature for Cloud Control Planes](#enabling-governance-feature-for-cloud-control-planes) +3. Wait for automatic agent deployment +4. Access the Governance Hub from the Cloud UI sidebar +5. Navigate to "API Repository" to view your automatically discovered APIs +6. Trigger "ReSync" to pull updates from the control planes + +**Expected Outcome:** +- All APIs from your Tyk Control Plane will be automatically discovered and displayed in the API Repository + +### Example 2: Multi-Platform Governance with Custom Agents + +This example demonstrates how to set up governance across multiple API providers. + +**Configuration Steps:** + +1. Requesting Access to Tyk Governance +2. Generate agent tokens for each provider as described in [Installing a Local Agent](#installing-a-local-agent) +3. Create configuration files for each agent +4. Deploy each agent using Docker or Kubernetes as described in [Installing a Local Agent](#installing-a-local-agent) +5. Verify agent connections +6. Access the Governance Hub with the provided URL +7. Navigate to "API Repository" to view your automatically discovered APIs +8. Trigger "ReSync" to pull updates from all agents + +**Expected Outcome:** +- APIs from all providers will be discovered and displayed in a unified repository diff --git a/tyk-governance/overview.mdx b/tyk-governance/overview.mdx new file mode 100644 index 00000000..f7b143e4 --- /dev/null +++ b/tyk-governance/overview.mdx @@ -0,0 +1,64 @@ +--- +title: "Tyk Governance Overview" +tags: ['Tyk Governance'] +sidebarTitle: "Overview" +--- + +## Overview + +Tyk Governance is a universal API governance hub that enables organizations to establish, enforce, and monitor governance policies across multiple API platforms and gateways. It solves the challenge of fragmented API management by providing a centralized approach to governance, regardless of where your APIs are hosted or which technologies they use. + +In today's complex API ecosystems, organizations struggle with inconsistent standards, security vulnerabilities, and compliance gaps across different API platforms. Tyk Governance bridges these gaps by creating a unified governance layer that works seamlessly with Tyk and extends to third-party API platforms, such as AWS API Gateway. + +<img src="/img/governance/governance-overview.png" alt="Tyk Governance provides centralized visibility and organizations across different API platforms" /> + +## Key Benefits + +* **Universal Governance** - Define and enforce consistent policies across multiple API platforms and styles (REST, GraphQL, event-driven) from a single control plane +* **Reduced Duplication** - Identify redundant or shadow APIs across different departments, reducing maintenance costs and security risks +* **Early Feedback** - Catch governance violations during design and development, not after deployment, reducing rework by up to 60% +* **Collaborative Improvement** - Enable teams to work together with shared visibility and clear ownership of APIs across the organization +* **Measurable API Maturity** - Track and improve API quality with quantifiable metrics across technical excellence, business impact, and developer experience + +## Who Should Use Tyk Governance + +Tyk Governance transforms API governance from a fragmented, post-deployment concern into a proactive, continuous, and scalable process across your entire API ecosystem. + +```mermaid +flowchart LR + A[Enterprise Architects and Security Leads] -->|Define Policies| B[Tyk Governance] + C[Platform Engineers] -->|Integrate Tools| B + B -->|Provide Feedback| D[API Developers] + D -->|Create Compliant APIs| B + B -->|Compliance Reporting| A[Enterprise Architects and Security Leads] +``` + +### Enterprise Architects & Security Leads + +Enterprise architects and security leads use Tyk Governance to establish organization-wide standards and ensure strategic alignment of the API program. They benefit from: + +* Centralized visibility across all API platforms +* Comprehensive compliance reporting +* The ability to define tiered governance policies based on API criticality + +**Example:** An enterprise architect at a financial services company uses Tyk Governance to ensure all customer-facing APIs comply with security and regulatory requirements, while allowing internal APIs to follow a lighter governance model. + +### Platform Engineers + +Platform engineers leverage Tyk Governance to build and maintain internal developer platforms that streamline API development. They value: + +* Seamless integration of governance into CI/CD pipelines +* Self-service tools that empower developers +* Unified monitoring across environments + +**Example:** A platform engineer integrates Tyk Governance into Tyk Dashboard and other API platforms, providing API templates that automatically incorporate security best practices and compliance requirements. + +### API Developers + +API developers rely on Tyk Governance to design and implement APIs that meet organizational standards from day one. They appreciate: + +* Clear guidance on governance requirements +* Real-time feedback during development +* Reduced rework and faster release cycles + +**Example:** An API developer receives immediate feedback that their new payment API is missing required rate-limiting policies, allowing them to fix the issue before submitting for review. \ No newline at end of file diff --git a/tyk-identity-broker/tib-rest-api.mdx b/tyk-identity-broker/tib-rest-api.mdx new file mode 100644 index 00000000..0d99821d --- /dev/null +++ b/tyk-identity-broker/tib-rest-api.mdx @@ -0,0 +1,274 @@ +--- +title: "TIB REST API" +sidebarTitle: "Tyk Identity Broker" +order: 0 +--- + +The Tyk Identity Broker (TIB) has an API to allow policies to be created, updated, removed and listed for programmatic and automated access. TIB also has a "flush" feature that enables you to flush the current configuration to disk for use when the client starts again. + +TIB does not store profiles in a shared store, so if you have multiple TIB instances, they need to be configured individually (for now). Since we don't expect TIB stores to change often, this is acceptable. + +Starting from Tyk Dashboard 3, TIB is built-in to the dashboard. TIB endpoints are exposed as part of dashboard API on the `/api/tib/` prefix. So if in the guide below external TIB API endpoint is `/api/profiles` the similar endpoint on the dashboard API will be `/api/tib/profiles`. + + +## List Profiles + +```{.copyWrapper} expandable +GET /api/profiles/ +Authorization: test-secret + +{ + "Status": "ok", + "ID": "", + "Data": [ + { + "ActionType": "GenerateTemporaryAuthToken", + "ID": "11", + "IdentityHandlerConfig": { + "DashboardCredential": "822f2b1c75dc4a4a522944caa757976a", + "DisableOneTokenPerAPI": false, + "TokenAuth": { + "BaseAPIID": "e1d21f942ec746ed416ab97fe1bf07e8" + } + }, + "MatchedPolicyID": "5654566b30c55e3904000003", + "OrgID": "53ac07777cbb8c2d53000002", + "ProviderConfig": { + "ExrtactUserNameFromBasicAuthHeader": true, + "OKCode": 200, + "OKRegex": "origin", + "OKResponse": "ewogICJvcmlnaW4iOiAiNjIuMjMyLjExNC4yNTAsIDE3OC42Mi4xMS42MiwgMTc4LjYyLjExLjYyIgp9Cg==", + "TargetHost": "http://sharrow.tyk.io/ba-1/" + }, + "ProviderConstraints": { + "Domain": "", + "Group": "" + }, + "ProviderName": "ProxyProvider", + "ReturnURL": "", + "Type": "passthrough" + }, + { + "ActionType": "GenerateOAuthTokenForClient", + "ID": "6", + "IdentityHandlerConfig": { + "DashboardCredential": "{DASHBAORD-API-ID}", + "DisableOneTokenPerAPI": false, + "OAuth": { + "APIListenPath": "{API-LISTEN-PATH}", + "BaseAPIID": "{BASE-API-ID}", + "ClientId": "{TYK-OAUTH-CLIENT-ID}", + "RedirectURI": "http://{APP-DOMAIN}:{PORT}/{AUTH-SUCCESS-PATH}", + "ResponseType": "token", + "Secret": "{TYK-OAUTH-CLIENT-SECRET}" + } + }, + "MatchedPolicyID": "POLICY-ID", + "OrgID": "53ac07777cbb8c2d53000002", + "ProviderConfig": { + "FailureRedirect": "http://{APP-DOMAIN}:{PORT}/failure", + "LDAPAttributes": [], + "LDAPUseSSL": false, + "LDAPPort": "389", + "LDAPServer": "localhost", + "LDAPUserDN": "cn=*USERNAME*,cn=dashboard,ou=Group,dc=ldap,dc=tyk-ldap-test,dc=com" + } + "ProviderName": "ADProvider", + "ReturnURL": "", + "Type": "passthrough" + } + ] +} +``` + +## <a name="add-profile"></a> Add Profile + +### Sample Request + +```{.copyWrapper} expandable +POST /api/profiles/{id} +Authorization: test-secret + +{ + "ActionType": "GenerateTemporaryAuthToken", + "ID": "11", + "IdentityHandlerConfig": { + "DashboardCredential": "822f2b1c75dc4a4a522944caa757976a", + "DisableOneTokenPerAPI": false, + "TokenAuth": { + "BaseAPIID": "e1d21f942ec746ed416ab97fe1bf07e8" + } + }, + "MatchedPolicyID": "5654566b30c55e3904000003", + "OrgID": "53ac07777cbb8c2d53000002", + "ProviderConfig": { + "ExrtactUserNameFromBasicAuthHeader": true, + "OKCode": 200, + "OKRegex": "origin", + "OKResponse": "ewogICJvcmlnaW4iOiAiNjIuMjMyLjExNC4yNTAsIDE3OC42Mi4xMS42MiwgMTc4LjYyLjExLjYyIgp9Cg==", + "TargetHost": "http://sharrow.tyk.io/ba-1/" + }, + "ProviderConstraints": { + "Domain": "", + "Group": "" + }, + "ProviderName": "ProxyProvider", + "ReturnURL": "", + "Type": "passthrough" +} +``` + +### Sample Response + +``` expandable +{ + "Status": "ok", + "ID": "11", + "Data": { + "ID": "11", + "OrgID": "53ac07777cbb8c2d53000002", + "ActionType": "GenerateTemporaryAuthToken", + "MatchedPolicyID": "5654566b30c55e3904000003", + "Type": "passthrough", + "ProviderName": "ProxyProvider", + "ProviderConfig": { + "ExrtactUserNameFromBasicAuthHeader": true, + "OKCode": 200, + "OKRegex": "origin", + "OKResponse": "ewogICJvcmlnaW4iOiAiNjIuMjMyLjExNC4yNTAsIDE3OC42Mi4xMS42MiwgMTc4LjYyLjExLjYyIgp9Cg==", + "TargetHost": "http://sharrow.tyk.io/ba-1/" + }, + "IdentityHandlerConfig": { + "DashboardCredential": "822f2b1c75dc4a4a522944caa757976a", + "DisableOneTokenPerAPI": false, + "TokenAuth": { + "BaseAPIID": "e1d21f942ec746ed416ab97fe1bf07e8" + } + }, + "ProviderConstraints": { + "Domain": "", + "Group": "" + }, + "ReturnURL": "" + } +} +``` + +## <a name="update-profile"></a> Update Profile + +### Sample Request + +```{.copyWrapper} expandable +PUT /api/profiles/{id} +Authorization: test-secret + +{ + "ActionType": "GenerateTemporaryAuthToken", + "ID": "11", + "IdentityHandlerConfig": { + "DashboardCredential": "822f2b1c75dc4a4a522944caa757976a", + "DisableOneTokenPerAPI": false, + "TokenAuth": { + "BaseAPIID": "e1d21f942ec746ed416ab97fe1bf07e8" + } + }, + "MatchedPolicyID": "5654566b30c55e3904000003", + "OrgID": "53ac07777cbb8c2d53000002", + "ProviderConfig": { + "ExrtactUserNameFromBasicAuthHeader": true, + "OKCode": 200, + "OKRegex": "origin", + "OKResponse": "ewogICJvcmlnaW4iOiAiNjIuMjMyLjExNC4yNTAsIDE3OC42Mi4xMS42MiwgMTc4LjYyLjExLjYyIgp9Cg==", + "TargetHost": "http://sharrow.tyk.io/ba-1/" + }, + "ProviderConstraints": { + "Domain": "", + "Group": "" + }, + "ProviderName": "ProxyProvider", + "ReturnURL": "", + "Type": "passthrough" +} +``` + +### Sample Response + +``` expandable +{ + "Status": "ok", + "ID": "11", + "Data": { + "ID": "11", + "OrgID": "53ac07777cbb8c2d53000002", + "ActionType": "GenerateTemporaryAuthToken", + "MatchedPolicyID": "5654566b30c55e3904000003", + "Type": "passthrough", + "ProviderName": "ProxyProvider", + "ProviderConfig": { + "ExrtactUserNameFromBasicAuthHeader": true, + "OKCode": 200, + "OKRegex": "origin", + "OKResponse": "ewogICJvcmlnaW4iOiAiNjIuMjMyLjExNC4yNTAsIDE3OC42Mi4xMS42MiwgMTc4LjYyLjExLjYyIgp9Cg==", + "TargetHost": "http://sharrow.tyk.io/ba-1/" + }, + "IdentityHandlerConfig": { + "DashboardCredential": "822f2b1c75dc4a4a522944caa757976a", + "DisableOneTokenPerAPI": false, + "TokenAuth": { + "BaseAPIID": "e1d21f942ec746ed416ab97fe1bf07e8" + } + }, + "ProviderConstraints": { + "Domain": "", + "Group": "" + }, + "ReturnURL": "" + } +} +``` + +## <a name="delete-profile"></a> Delete Profile + +### Sample Request + +```{.copyWrapper} +Delete /api/profiles/{id} +Authorization: test-secret + +[emtpy body] + +``` + +### Sample Response + +``` +{ + "Status": "ok", + "ID": "200", + "Data": {} +} +``` + +## <a name="save-profile"></a> Save Profiles to Disk + +### Sample Request + +```{.copyWrapper} +POST /api/profiles/save +Authorization: test-secret +[empty body] +``` + +### Sample Response + +``` +{ + "Status": "ok", + "ID": "", + "Data": {} +} +``` + +## <a name="outcome"></a> Outcome + +The existing `profiles.json` file will be backed up to a new file, and the current profiles data in memory will be flushed to disk as the new `profiles.json` file. Backups are time stamped (e.g. `profiles_backup_1452677499.json`). diff --git a/tyk-mdcb-api.mdx b/tyk-mdcb-api.mdx new file mode 100644 index 00000000..19059d3d --- /dev/null +++ b/tyk-mdcb-api.mdx @@ -0,0 +1,7 @@ +--- +title: "Tyk MDCB API" +tags: ['OpenAPI Spec', 'OpenAPI Specification', 'OAS', 'REST', 'Tyk MDCB OpenAPI Spec', 'Tyk MDCB OAS', 'MDCB API REST'] +order: 3 +sidebarTitle: "Multi Data Center Bridge" +--- + diff --git a/tyk-multi-data-centre/mdcb-configuration-options.mdx b/tyk-multi-data-centre/mdcb-configuration-options.mdx new file mode 100644 index 00000000..b9d65ee7 --- /dev/null +++ b/tyk-multi-data-centre/mdcb-configuration-options.mdx @@ -0,0 +1,36 @@ +--- +title: "MDCB Configuration options" +'og:description': "Each of the config options that are available when deploying MDCB." +tags: ['MDCB', 'configuration options', 'MDCB configuration options'] +order: 3 +sidebarTitle: "Multi Data Center Bridge" +--- + +import MdcbConfig from '/snippets/mdcb-config.mdx'; + +## Tyk MDCB Configuration + +The Tyk MDCB server is configured primarily via the `tyk_sink.conf` file, this file resides in `/opt/tyk-sink` on most systems, but can also live anywhere and be directly targeted with the `-c` flag. + +### Environment Variables + +Environment variables (env var) can be used to override the settings defined in the configuration file. Where an environment variable is specified, its value will take precedence over the value in the configuration file. + +### Default Ports + +| Application | Port | +| :------------------------- | :---------------- | +|MongoDB | 27017 | +|Redis | 6379 | +|**Tyk Dashboard** | | +|Developer Portal | 3000 | +|Admin Dashboard | 3000 | +|Admin Dashboard API | 3000 | +|**Tyk Gateway** | | +|Management API | 8080 | +|**MDCB** | | +|RPC services | 9090 | +|HTTP endpoints | 8181 | + + +<MdcbConfig/> diff --git a/tyk-open-source.mdx b/tyk-open-source.mdx new file mode 100644 index 00000000..d0b8ec41 --- /dev/null +++ b/tyk-open-source.mdx @@ -0,0 +1,33 @@ +--- +title: "Tyk Open Source" +'og:description': "This page serves as a comprehensive guide to Tyk Open Source" +sidebarTitle: "Overview" +tags: ['installation', 'migration', 'open source'] +--- + +## What is Tyk Open Source + +Open source is at the heart of what we do. Anything that is API Gateway-related lives in the Gateway, or is critical for the Gateway to work is open and freely available via our [Github](https://github.com/TykTechnologies/tyk). + +The Tyk Gateway is fully open-source. It's all the same Gateway that's used by you (the community!), by our enterprise products, as well as our SaaS. + +Our commitment to open source also delivers a host of benefits for our users: sign up for free with Tyk, receive securely packaged open source packages, get started guides, access to our community and all of the latest open source information. + + +<Note> +Tyk OSS, Tyk Open Source, Tyk Gateway, Tyk CE +</Note> + + +<img src="/img/diagrams/oss-flow.png" alt="OSS-Guide" /> + +## What Does Tyk Open Source Include? + +The Tyk Team has created and maintains the following components, which are fully Open Source and available under Mozilla Public License 2.0 (MPL). Star the Tyk components you use by clicking the appropriate button: + +* [Tyk Gateway](/tyk-oss-gateway) - Fully fledged API Gateway (Start here!) - <a href="https://github.com/TykTechnologies/tyk" className="github-button" target="_blank" rel="noopener noreferrer">⭐ Star TykTechnologies/tyk</a> +* [Tyk Pump](/tyk-pump) - Send API analytics data to your chosen backend - <a href="https://github.com/TykTechnologies/tyk-pump" className="github-button" target="_blank" rel="noopener noreferrer">⭐ Star TykTechnologies/tyk-pump</a> +* [Tyk Identity Broker](/api-management/external-service-integration#what-is-tyk-identity-broker-tib) - Connect your third-party IdP systems - <a href="https://github.com/TykTechnologies/api-management/external-service-integration#what-is-tyk-identity-broker-tib" className="github-button" target="_blank" rel="noopener noreferrer">⭐ Star TykTechnologies/api-management/external-service-integration#what-is-tyk-identity-broker-tib</a> +* [Tyk Helm Chart](/product-stack/tyk-charts/overview) - Deploy Tyk in K8S - <a href="https://github.com/TykTechnologies/tyk-charts" className="github-button" target="_blank" rel="noopener noreferrer">⭐ Star TykTechnologies/tyk-charts</a> + +You can find additional FAQs regarding the MPL license [here](https://www.mozilla.org/en-US/MPL/2.0/FAQ/). diff --git a/tyk-oss-gateway.mdx b/tyk-oss-gateway.mdx new file mode 100644 index 00000000..c3fa3740 --- /dev/null +++ b/tyk-oss-gateway.mdx @@ -0,0 +1,15 @@ +--- +title: "Tyk Gateway Open Source (OSS)" +order: 1 +sidebarTitle: "Tyk Gateway" +--- + +import TykGatewayFeaturesInclude from '/snippets/tyk-gateway-features-include.mdx'; + +## What is the Tyk Gateway? + +<TykGatewayFeaturesInclude/> + +## Deployment Options + +Refer the [deployment options](/apim) page. \ No newline at end of file diff --git a/tyk-oss-gateway/configuration.mdx b/tyk-oss-gateway/configuration.mdx new file mode 100644 index 00000000..ea529eec --- /dev/null +++ b/tyk-oss-gateway/configuration.mdx @@ -0,0 +1,25 @@ +--- +title: "Tyk Gateway Configuration Options" +order: 1 +sidebarTitle: "Gateway" +--- + +import GatewayConfig from '/snippets/gateway-config.mdx'; + +You can use environment variables to override the config file for the Tyk Gateway. The Gateway configuration file can be found in the `tyk-gateway` folder and by default is called `tyk.conf`, though it can be renamed and specified using the `--conf` flag. Environment variables are created from the dot notation versions of the JSON objects contained with the config files. +To understand how the environment variables notation works, see [Environment Variables](/tyk-oss-gateway/configuration). + +All the Gateway environment variables have the prefix `TYK_GW_`. The environment variables will take precedence over the values in the configuration file. + +### tyk lint + +In **v2.4** we have added a new `tyk lint` command which will validate your `tyk.conf` file and validate it for syntax correctness, misspelled attribute names or format of values. The Syntax can be: + +`tyk lint` or `tyk --conf=path lint` + +If `--conf` is not used, the first of the following paths to exist is used: + +`./tyk.conf` +`/etc/tyk/tyk.conf` + +<GatewayConfig/> diff --git a/tyk-overview.mdx b/tyk-overview.mdx new file mode 100644 index 00000000..6509d12a --- /dev/null +++ b/tyk-overview.mdx @@ -0,0 +1,67 @@ +--- +title: "Tyk Overview" +'og:description': "Explaining Tyk at a high level" +tags: ['Tyk API Management', 'Getting Started', 'Tutorials'] +order: 5 +sidebarTitle: "Tyk Overview" +--- + +Testing Changes + +APIs are central to enabling software integration, data exchange, and automation. However, as organizations scale their API ecosystems, they face mounting challenges around security, reliability, and performance. Tyk exists to simplify and strengthen this process. With a focus on efficient, secure, and scalable API management, Tyk provides a powerful solution for companies looking to streamline API operations, enforce robust security standards, and gain deep visibility into their API usage. + +## Why Tyk Exists: The Need for API Management +The demand for APIs has exploded over the last decade, with companies using them to enable everything from mobile apps and IoT devices to microservices architectures and third-party integrations. But with this growth come significant challenges: + +- **Security Risks**: Exposing services through APIs introduces new security vulnerabilities that need constant management and monitoring. +- **Scalability**: As usage grows, APIs need to be resilient, able to handle high traffic, and scalable across global regions. +- **Complexity in Integration**: Integrating various backend services, identity providers, and front-end applications can become an overwhelming task. +- **Monitoring and Performance**: API performance monitoring, traffic management, and analytics are crucial to optimize API usage and provide reliable service. + +Tyk exists to address these challenges by providing an API management platform that’s secure, scalable, flexible, and easy to use. With Tyk, organizations can confidently manage the entire lifecycle of their APIs, from initial design to deployment and ongoing monitoring. + +## What Problem Does Tyk Solve? + +Tyk is designed to solve several critical issues that organizations face with APIs: + +1. **Unified API Management** + Tyk centralizes all aspects of API management, offering tools for routing, load balancing, security, and performance. This unified approach helps teams streamline API operations and reduce operational overhead. + +2. **Enhanced Security and Compliance** + APIs are vulnerable to numerous security threats. Tyk addresses these concerns by supporting a wide array of security protocols, including OAuth2.0, JWT, HMAC, and OpenID Connect. Additionally, Tyk enables organizations to enforce fine-grained access control policies, rate limiting, and quotas to safeguard API access. + +3. **Scalability for High-Volume Traffic** + Tyk provides a high-performance API gateway that can handle substantial traffic loads while maintaining low latency, ensuring that APIs can scale as demand increases. Tyk’s Multi Data Centre Bridge (MDCB) further enhances scalability by distributing traffic across multiple regions, providing high availability and low latency globally. + +4. **Seamless Integration and Flexibility** + Tyk’s open-source architecture and compatibility with Kubernetes, Docker, and cloud platforms make it easy to integrate within existing infrastructures. With Tyk, teams can operate in hybrid or multi-cloud environments, deploy APIs as Kubernetes-native resources, and leverage CI/CD pipelines for seamless updates. + +5. **Developer and Consumer Enablement** + Through the Tyk Developer Portal, developers can discover and access APIs easily, enabling faster adoption and integration. With detailed documentation, developer self-service features, and API analytics, Tyk empowers both API providers and consumers to make the most of their API ecosystem. + +## How Tyk’s Components Work Together + +Tyk offers a comprehensive suite of components designed to address every aspect of the API lifecycle: + +- **[Tyk Gateway](/tyk-oss-gateway)**: The core of Tyk’s platform, providing high-performance API routing, traffic management, and security. +- **[Tyk Dashboard](/tyk-dashboard)**: A graphical control panel that simplifies API management, configuration, and monitoring. +- **[Tyk Developer Portal](/tyk-developer-portal)**: A self-service portal that enables developers to access, understand, and integrate with APIs. +- **[Tyk Multi Data Centre Bridge (MDCB)](/tyk-multi-data-centre)**: Allows centralized control over APIs distributed across multiple data centers or cloud regions. +- **[Tyk Pump](/tyk-pump)**: Collects and streams analytics from the Tyk Gateway to various storage backends for performance monitoring and reporting. +- **[Tyk Operator](/api-management/automations/operator#what-is-tyk-operator)**: Kubernetes-native API management that allows teams to manage APIs as Kubernetes resources. +- **[Tyk Streams](/api-management/event-driven-apis#)**: Enables real-time data streaming and push-based communication for applications requiring live data. +- **[Tyk Sync](/api-management/automations/sync)**: Synchronizes API configurations across environments, supporting DevOps practices and CI/CD workflows. +- **[Tyk Identity Broker](/api-management/external-service-integration#what-is-tyk-identity-broker-tib)**: Integrates with external identity providers for single sign-on (SSO) and centralized identity management. +- **[Tyk Helm Charts](/product-stack/tyk-charts/overview)**: Simplifies the deployment of Tyk components within Kubernetes environments. +- **[Universal Data Graph](/api-management/data-graph#overview)**: Provides a single GraphQL endpoint that aggregates data from multiple sources, simplifying access to complex data. + +Each component plays a specific role in managing the API lifecycle, from initial deployment and configuration to real-time data streaming and developer access. Together, they create a cohesive API management ecosystem that can handle the unique challenges of production environments. + +You can learn more about the components that make up Tyk, [here](/tyk-components). + +## Why Use Tyk? + +In summary, Tyk offers a complete API management solution designed for modern, production-grade API operations. With its open-source core, robust security options, high performance, and flexible deployment models, Tyk provides everything an organization needs to manage, scale, and secure their APIs. + +Whether you’re a startup looking to build a simple API or a global enterprise deploying complex, multi-region architectures, Tyk has the tools to support your growth at every stage. If you face problems with scaling your solutions, learn more about how Tyk can support you by [getting started with Tyk Cloud](/getting-started/create-account). + diff --git a/tyk-portal-api.mdx b/tyk-portal-api.mdx new file mode 100644 index 00000000..799868e2 --- /dev/null +++ b/tyk-portal-api.mdx @@ -0,0 +1,20 @@ +--- +title: "Classic Portal API" +'og:description': "Landing page for the Tyk Classic Portal API documentation" +sidebarTitle: "Overview" +tags: ['Tyk Classic Portal API', 'Classic Portal API'] +noindex: True +--- + +import LegacyClassicPortalApi from '/snippets/legacy-classic-portal-api.mdx'; + +<LegacyClassicPortalApi/> + + +This section describes the Tyk Classic Portal API endpoints. It includes the following: + +* [Portal Keys](/tyk-apis/tyk-portal-api/portal-keys) +* [Portal Policies](/tyk-apis/tyk-dashboard-api/portal-policies) +* [Portal Developers](/tyk-apis/tyk-portal-api/portal-developers) +* [Portal Configuration](/tyk-apis/tyk-portal-api/portal-configuration) +* [Portal Documentation](/tyk-apis/tyk-portal-api/portal-documentation) \ No newline at end of file diff --git a/tyk-pump/tyk-pump-configuration/tyk-pump-environment-variables.mdx b/tyk-pump/tyk-pump-configuration/tyk-pump-environment-variables.mdx new file mode 100644 index 00000000..d271509c --- /dev/null +++ b/tyk-pump/tyk-pump-configuration/tyk-pump-environment-variables.mdx @@ -0,0 +1,16 @@ +--- +title: "Tyk Pump Environment Variables" +'og:description': "Using Environment Variables to configure your Tyk Pump" +tags: ['Tyk Pump', 'Envoronment Variables', 'Configuration'] +order: 6 +sidebarTitle: "Pump" +--- + +import PumpConfig from '/snippets/pump-config.mdx'; + +You can use environment variables to override the config file for the Tyk Pump. Environment variables are created from the dot notation versions of the JSON objects contained with the config files. +To understand how the environment variables notation works, see [Environment Variables](/tyk-oss-gateway/configuration). + +All the Pump environment variables have the prefix `TYK_PMP_`. The environment variables will take precedence over the values in the configuration file. + +<PumpConfig/> diff --git a/tyk-self-managed.mdx b/tyk-self-managed.mdx new file mode 100644 index 00000000..c2eadd71 --- /dev/null +++ b/tyk-self-managed.mdx @@ -0,0 +1,82 @@ +--- +title: "Tyk Self Managed" +'og:description': "This page serves as a comprehensive guide to migrating workloads to Tyk Self Managed" +sidebarTitle: "Overview" +tags: ['installation', 'migration', 'self managed'] +--- + +## What is Tyk Self-Managed + +Tyk Self-Managed allows you to easily install our Full Lifecycle API Management solution in your own infrastructure. There is no calling home, and there are no usage limits. You have full control. + +## What Does Tyk Self-Managed Include? +The full Tyk Self-Managed system consists of: + +* [Tyk Gateway](/tyk-oss-gateway): Tyk Gateway is provided β€˜Batteries-included’, with no feature lockout. It is an open source enterprise API Gateway, supporting REST, GraphQL, TCP and gRPC protocols, that protects, secures and processes your APIs. +* [Tyk Dashboard](/tyk-dashboard): The management Dashboard and integration API manage a cluster of Tyk Gateways and also show analytics and features of the [Developer portal](/tyk-developer-portal). The Dashboard also provides the API Developer Portal, a customizable developer portal for your API documentation, developer auto-enrollment and usage tracking. +* [Tyk Pump](/tyk-pump): Tyk Pump handles moving analytics data between your gateways and your Dashboard (amongst other data sinks). The Tyk Pump is an open source analytics purger that moves the data generated by your Tyk nodes to any back-end. +* [Tyk Identity Broker](/api-management/external-service-integration#what-is-tyk-identity-broker-tib) (Optional): Tyk Identify Broker handles integrations with third-party IDP's. It (TIB) is a component providing a bridge between various Identity Management Systems such as LDAP, Social OAuth (e.g. GPlus, Twitter, GitHub) or Basic Authentication providers, to your Tyk installation. +* [Tyk Multi-Data Center Bridge](/tyk-multi-data-centre) (Optional, add-on): Tyk Multi-Data Center Bridge allows for the configuration of a Tyk ecosystem that spans many data centers and clouds. It also (MDCB) acts as a broker between Tyk Gateway Instances that are isolated from one another and typically have their own Redis DB. + +<img src="/img/diagrams/diagram_docs_pump-data-flow@2x.png" alt="Tyk Self-Managed Archtecture" /> + +## Tyk Self Managed Pricing + +Refer the [pricing page](https://tyk.io/pricing/) + +## Tyk Dependencies and Database Support + +### MongoDB / PostgreSQL + +Tyk Dashboard requires a persistent datastore for its operations. By default MongoDB is used. From Tyk v4.0, we also support PostgreSQL. See [Database Options](/api-management/dashboard-configuration#supported-database) for a list of versions and drop-in replacements we support. + +### Redis + +Tyk Gateway requires Redis for its operations. Here is the list of supported versions: + +**Supported Versions** +- Tyk 5.3 supports Redis 6.2.x, 7.0.x, and 7.2.x +- Tyk 5.2.x and earlier supports Redis 6.0.x and Redis 6.2.x only. + +Visit the [Gateway page](/tyk-oss-gateway) for more info. + +### Tyk Gateway Architecture + +The Tyk Gateway can run completely independently, requiring only a Redis database, and can scale horizontally: + +<img src="/img/diagrams/oss-architecture.png" alt="Open Source Architecture" /> + + +### Init Systems + +Tyk packages support SysVinit Linux init systems, [systemd](https://www.freedesktop.org/wiki/Software/systemd/) and Upstart (both 0.6.x and 1.x, [FYI - Ubuntu stopped supporting Upstart] upstart(https://askubuntu.com/questions/1024120/will-ubuntu-18-04-lts-still-support-upstart-or-do-we-have-to-change-to-systemd)). +During package installation only one is chosen depending on the operating system support, e.g.: + +* CentOS 6, RHEL 6, Amazon Linux ship with Upstart 0.6.x +* Ubuntu 14.04, Debian Jessie with Upstart 1.x +* CentOS 7, RHEL 7, Ubuntu 16.04, Debian Stretch are running with systemd +* Certain older distros may only provide SysVinit but all of them typically provide compatibility with its scripts + +Note that any init scripts of your choosing can be used instead of automatically detected ones by copying them from the `install/inits` directory inside the package directory. + +This init system variance implies there are different ways to manage the services and collect service logs. + +#### Upstart +For Upstart, service management can be performed through the `initctl` or a set of `start`, `stop`, `restart` and `status` commands. Upstart 1.x also works with the `service` command. + +#### systemd +For systemd, either `systemctl` or `service` commands may be utilized. + +The `service` command can usually be used with SysVinit scripts, as well as invoking them directly. + + +<Note> +* Upstart 0.6.x and SysVinit: log files are located in `/var/logs` for every respective service, e.g. `/var/logs/tyk-gateway.stderr` and `/var/logs/tyk-gateway.stdout` +* Upstart 1.x: by default everything is stored in `/var/logs/upstart` directory, e.g. `/var/logs/upstart/tyk-gateway.log` +* systemd utilizes its own logging mechanism called journald, which is usable via the `journalctl` command, e.g. `journalctl -u tyk-gateway` + + +Please consult with respective init system documentation for more details on how to use and configure it. +</Note> + + diff --git a/tyk-self-managed/install.mdx b/tyk-self-managed/install.mdx new file mode 100644 index 00000000..7dc2df8f --- /dev/null +++ b/tyk-self-managed/install.mdx @@ -0,0 +1,3141 @@ +--- +title: "Installation Options for Tyk Self-Managed" +'og:description': "Explore the various installation options for Tyk Self-Managed, including Docker, Kubernetes, Ansible, and more." +sidebarTitle: "Installation Options" +--- + +import { ResponsiveGrid } from '/snippets/ResponsiveGrid.mdx'; + +## Introduction + +<ResponsiveGrid> + + +<Card href="/tyk-self-managed/install#install-with-docker" img="/img/docker.png"> +**Read time: 10 mins** + +Install with Docker +</Card> + +<Card href="/tyk-self-managed/install#install-on-kubernetes" img="/img/k8s.png"> +**Read time: 10 mins** + +Install on K8s +</Card> + +<Card href="/tyk-self-managed/install#install-with-ansible" img="/img/ansible.png"> +**Read time: 10 mins** + +Install with Ansible +</Card> + +<Card href="/tyk-self-managed/install#install-tyk-on-redhat-rhel-centos" img="/img/redhat-logo2.png"> +**Read time: 10 mins** + +Install on Red Hat +</Card> + +<Card href="/tyk-self-managed/install#install-tyk-on-debian-or-ubuntu" img="/img/debian-nd-753.png"> +**Read time: 10 mins** + +Install on Ubuntu +</Card> + +<Card href="/tyk-self-managed/install#install-on-aws-marketplace" img="/img/aws.png"> +**Read time: 10 mins** + +Install on Amazon AWS +</Card> + +<Card href="/tyk-self-managed/install#install-on-heroku" img="/img/heroku-logo.png"> +**Read time: 10 mins** + +Install Tyk on Heroku +</Card> + +<Card href="/tyk-self-managed/install#install-on-microsoft-azure" img="/img/azure-2.png"> +**Read time: 10 mins** + +Install on Microsoft Azure +</Card> + + +</ResponsiveGrid> + +## Install on Kubernetes + +The main way to install *Tyk Self-Managed* in a Kubernetes cluster is via Helm charts. +We are actively working to add flexibility and more user flows to our chart. Please reach out +to our teams on support or the cummunity forum if you have questions, requests or suggestions for improvements. + +Get started with one of our quick start guides: + +- [Quick Start with PostgreSQL](#install-tyk-stack-with-helm-chart-postgresql) +- [Quick Start with MongoDB](/tyk-self-managed/install#install-tyk-stack-with-helm-chart-mongodb) + +Or go to [Tyk Stack helm chart](/product-stack/tyk-charts/tyk-stack-chart) for detailed installation instructions and configuration options. + +### Install Tyk Stack with Helm Chart (PostgreSQL) + +The following guides provide instructions to install Redis, PostgreSQL, and Tyk stack with default configurations. It is intended for quick start only. For production, you should install and configure Redis and PostgreSQL separately. + +**Prerequisites** + +* [Kubernetes 1.19+](https://kubernetes.io/docs/setup/) +* [Helm 3+](https://helm.sh/docs/intro/install/) + +**Quick Start** + +The following quick start guide explains how to use the Tyk Stack Helm chart to configure a Dashboard that includes: +- Redis for key storage +- PostgreSQL for app config +- Tyk Pump to send analytics to PostgreSQL. It also opens a metrics endpoint where Prometheus (if available) can scrape from. + +At the end of this quickstart Tyk Dashboard should be accessible through service `dashboard-svc-tyk-tyk-dashboard` at port `3000`. You can login to Dashboard using the admin email and password to start managing APIs. Tyk Gateway will be accessible through service `gateway-svc-tyk-tyk-gateway.tyk.svc` at port `8080`. + +**1. Setup required credentials** + +First, you need to provide Tyk license, admin email and password, and API keys. We recommend to store them in secrets. + +```bash expandable +NAMESPACE=tyk +REDIS_BITNAMI_CHART_VERSION=19.0.2 +POSTGRES_BITNAMI_CHART_VERSION=12.12.10 + +API_SECRET=changeit +ADMIN_KEY=changeit +TYK_LICENSE=changeit +ADMIN_EMAIL=admin@default.com +ADMIN_PASSWORD=changeit + +kubectl create namespace $NAMESPACE + +kubectl create secret generic my-secrets -n $NAMESPACE \ + --from-literal=APISecret=$API_SECRET \ + --from-literal=AdminSecret=$ADMIN_KEY \ + --from-literal=DashLicense=$TYK_LICENSE + +kubectl create secret generic admin-secrets -n $NAMESPACE \ + --from-literal=adminUserFirstName=Admin \ + --from-literal=adminUserLastName=User \ + --from-literal=adminUserEmail=$ADMIN_EMAIL \ + --from-literal=adminUserPassword=$ADMIN_PASSWORD +``` + +**2. Install Redis (if you don't already have Redis installed)** + +If you do not already have Redis installed, you may use these charts provided by Bitnami. + +```bash +helm upgrade tyk-redis oci://registry-1.docker.io/bitnamicharts/redis -n $NAMESPACE --install --version $REDIS_BITNAMI_CHART_VERSION +``` +Follow the notes from the installation output to get connection details and password. The DNS name of your Redis as set by Bitnami is `tyk-redis-master.tyk.svc:6379` (Tyk needs the name including the port) + +The Bitnami chart also creates a secret `tyk-redis` which stores the connection password in `redis-password`. We will make use of this secret in installation later. + +**3. Install PostgreSQL (if you don't already have PostgreSQL installed)** + +If you do not already have PostgreSQL installed, you may use these charts provided by Bitnami. + +```bash +helm upgrade tyk-postgres oci://registry-1.docker.io/bitnamicharts/postgresql --set "auth.database=tyk_analytics" -n $NAMESPACE --install --version $POSTGRES_BITNAMI_CHART_VERSION +``` + +Follow the notes from the installation output to get connection details. + +We require the PostgreSQL connection string for Tyk installation. This can be stored in a secret and will be used in installation later. + +```bash +POSTGRESQLURL=host=tyk-postgres-postgresql.$NAMESPACE.svc\ port=5432\ user=postgres\ password=$(kubectl get secret --namespace $NAMESPACE tyk-postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d)\ database=tyk_analytics\ sslmode=disable + +kubectl create secret generic postgres-secrets -n $NAMESPACE --from-literal=postgresUrl="$POSTGRESQLURL" +``` + + + +<Note> +Ensure that you are installing PostgreSQL versions that are supported by Tyk. Please consult the list of [supported versions](/api-management/dashboard-configuration#supported-database) that are compatible with Tyk. +</Note> + + +**4. Install Tyk** +```bash expandable +helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ + +helm repo update + +helm upgrade tyk tyk-helm/tyk-stack -n $NAMESPACE \ + --install \ + --set global.adminUser.useSecretName=admin-secrets \ + --set global.secrets.useSecretName=my-secrets \ + --set global.redis.addrs="{tyk-redis-master.$NAMESPACE.svc:6379}" \ + --set global.redis.passSecret.name=tyk-redis \ + --set global.redis.passSecret.keyName=redis-password \ + --set global.postgres.connectionStringSecret.name=postgres-secrets \ + --set global.postgres.connectionStringSecret.keyName=postgresUrl +``` + +**5. Done!** + +Now Tyk Dashboard should be accessible through service `dashboard-svc-tyk-tyk-dashboard` at port `3000`. You can login to Dashboard using the admin email and password to start managing APIs. Tyk Gateway will be accessible through service `gateway-svc-tyk-tyk-gateway.tyk.svc` at port `8080`. + +You are now ready to [create an API](/api-management/gateway-config-managing-classic#create-an-api). + +For the complete installation guide and configuration options, please see [Tyk Stack Helm Chart](/product-stack/tyk-charts/tyk-stack-chart). + +### Install Tyk Stack with Helm Chart (MongoDB) + +The following guides provide instructions to install Redis, MongoDB, and Tyk stack with default configurations. It is intended for quick start only. For production, you should install and configure Redis and MongoDB separately. + +**Prerequisites** + +* [Kubernetes 1.19+](https://kubernetes.io/docs/setup/) +* [Helm 3+](https://helm.sh/docs/intro/install/) + + + + <Note> + If you want to enable Tyk Enterprise Developer Portal, please use [PostgreSQL](#install-tyk-stack-with-helm-chart-postgresql). MongoDB is not supported in Developer Portal. + </Note> + + +**Quick Start** + +The following quick start guide explains how to use the Tyk Stack Helm chart to configure a Dashboard that includes: +- Redis for key storage +- MongoDB for app config +- Tyk Pump to send analytics to MongoDB. It also opens a metrics endpoint where Prometheus (if available) can scrape from. + +At the end of this quickstart Tyk Dashboard should be accessible through service `dashboard-svc-tyk-tyk-dashboard` at port `3000`. You can login to Dashboard using the admin email and password to start managing APIs. Tyk Gateway will be accessible through service `gateway-svc-tyk-tyk-gateway.tyk.svc` at port `8080`. + +**1. Setup required credentials** + +First, you need to provide Tyk license, admin email and password, and API keys. We recommend to store them in secrets. +```bash expandable +NAMESPACE=tyk +REDIS_BITNAMI_CHART_VERSION=19.0.2 +MONGO_BITNAMI_CHART_VERSION=15.1.3 + +API_SECRET=changeit +ADMIN_KEY=changeit +TYK_LICENSE=changeit +ADMIN_EMAIL=admin@default.com +ADMIN_PASSWORD=changeit + +kubectl create namespace $NAMESPACE + +kubectl create secret generic my-secrets -n $NAMESPACE \ + --from-literal=APISecret=$API_SECRET \ + --from-literal=AdminSecret=$ADMIN_KEY \ + --from-literal=DashLicense=$TYK_LICENSE + +kubectl create secret generic admin-secrets -n $NAMESPACE \ + --from-literal=adminUserFirstName=Admin \ + --from-literal=adminUserLastName=User \ + --from-literal=adminUserEmail=$ADMIN_EMAIL \ + --from-literal=adminUserPassword=$ADMIN_PASSWORD +``` + +**2. Install Redis (if you don't have a Redis instance)** + +If you do not already have Redis installed, you may use these charts provided by Bitnami. + +```bash +helm upgrade tyk-redis oci://registry-1.docker.io/bitnamicharts/redis -n $NAMESPACE --install --version $REDIS_BITNAMI_CHART_VERSION +``` +Follow the notes from the installation output to get connection details and password. The DNS name of your Redis as set by Bitnami is +`tyk-redis-master.tyk.svc:6379` (Tyk needs the name including the port) + +The Bitnami chart also creates a secret `tyk-redis` which stores the connection password in `redis-password`. We will make use of this secret in installation later. + + +<Note> +Please make sure you are installing Redis versions that are supported by Tyk. Please refer to Tyk docs to get list of [supported versions](/planning-for-production/database-settings#redis). +</Note> + + +**3. Install MongoDB (if you don't have a MongoDB instance)** + +If you do not already have MongoDB installed, you may use these charts provided by Bitnami. + +```bash +helm upgrade tyk-mongo oci://registry-1.docker.io/bitnamicharts/mongodb -n $NAMESPACE --install --version $MONGO_BITNAMI_CHART_VERSION +``` + + +<Note> +Please make sure you are installing MongoDB versions that are supported by Tyk. Please refer to Tyk docs to get list of [supported versions](/api-management/dashboard-configuration#supported-database). +</Note> + + + +<Note> +Bitnami MongoDB image is not supported on darwin/arm64 architecture. +</Note> + + +We require the MongoDB connection string for Tyk installation. You can store it in a secret and provide the secret in installation later. + +```bash +MONGOURL=mongodb://root:$(kubectl get secret --namespace $NAMESPACE tyk-mongo-mongodb -o jsonpath="{.data.mongodb-root-password}" | base64 -d)@tyk-mongo-mongodb.$NAMESPACE.svc:27017/tyk_analytics?authSource=admin + +kubectl create secret generic mongourl-secrets --from-literal=mongoUrl=$MONGOURL -n $NAMESPACE +``` + + + +<Note> +Ensure that you are installing MongoDB versions that are supported by Tyk. Please consult the list of [supported versions](/api-management/dashboard-configuration#supported-database) that are compatible with Tyk. +</Note> + + +**4. Install Tyk** +```bash expandable +helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ + +helm repo update + +helm upgrade tyk tyk-helm/tyk-stack -n $NAMESPACE \ + --install \ + --set global.adminUser.useSecretName=admin-secrets \ + --set global.secrets.useSecretName=my-secrets \ + --set global.redis.addrs="{tyk-redis-master.$NAMESPACE.svc:6379}" \ + --set global.redis.passSecret.name=tyk-redis \ + --set global.redis.passSecret.keyName=redis-password \ + --set global.mongo.driver=mongo-go \ + --set global.mongo.connectionURLSecret.name=mongourl-secrets \ + --set global.mongo.connectionURLSecret.keyName=mongoUrl \ + --set global.storageType=mongo \ + --set tyk-pump.pump.backend='{prometheus,mongo}' +``` + +**5. Done!** + +Now Tyk Dashboard should be accessible through service `dashboard-svc-tyk-tyk-dashboard` at port `3000`. You can login to Dashboard using the admin email and password to start managing APIs. Tyk Gateway will be accessible through service `gateway-svc-tyk-tyk-gateway.tyk.svc` at port `8080`. + +You are now ready to [create an API](/api-management/gateway-config-managing-classic#create-an-api). + +For the complete installation guide and configuration options, please see [Tyk Stack Helm Chart](/product-stack/tyk-charts/tyk-stack-chart). + + +### Install Tyk Stack on Windows with Helm + + + +<Note> +Installing Tyk on Kubernetes requires a multi-node Tyk license. If you are evaluating Tyk on Kubernetes, [contact us](https://tyk.io/about/contact/) to obtain an temporary license. +</Note> + + + +<Warning> +This deployment is NOT designed for production use or performance testing. The Tyk Pro Docker Demo is our full, [Self-Managed](/tyk-self-managed/install) solution, which includes our Gateway, Dashboard and analytics processing pipeline. + +This demo will run Tyk Self-Managed on your machine, which contains 5 containers: Tyk Gateway, Tyk Dashboard, Tyk Pump, Redis and either MongoDB or one of our supported [SQL databases](/api-management/dashboard-configuration#supported-database). + +This demo is great for proof of concept and demo purposes, but if you want to test performance, you need to move each component to a separate machine. +</Warning> + + + +<Note> +You use this at your own risk. Tyk is not supported on the Windows platform. However you can test it as a proof of concept using our Pro Demo Docker installation. +</Note> + + +**Prerequisites** + +- MS Windows 10 Pro +- [Tyk Helm Chart](https://github.com/TykTechnologies/tyk-helm-chart) +- [Docker Desktop for Windows](https://docs.docker.com/docker-for-windows/install/) running with a signed in [Docker ID](https://docs.docker.com/docker-id/) +- [minikube](https://minikube.sigs.k8s.io/docs/start/) +- [Kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) +- [Helm](https://github.com/helm/helm/releases) +- Git for Windows +- [Python for Windows](https://www.python.org/downloads/windows/) +- PowerShell running as administrator +- Our Pro Demo Docker [GitHub repo](https://github.com/TykTechnologies/tyk-pro-docker-demo) +- A free Tyk Self-Managed [Developer license](https://tyk.io/sign-up) + +Ensure that kubectl and helm prerequisites are configured on your Windows path environment variable + +This demo installation was tested with the following tools/versions: + +* Microsoft Windows 10 Pro v1909 VM on Azure (Standard D2 v3 size) +* Docker Desktop for Windows 2.2.0.0 (Docker engine v19.03.5) +* helm v3.0.3 +* minikube v1.7.1 (k8s v 1.17.2) +* kubectl v 1.17.0 (Note that kubectl is packaged with Docker Desktop for Windows, but the version may be incompatible with k8s) + +**Installation** + +Now you have your prerequisites, follow the instructions from our [Tyk Helm Chart](#use-legacy-helm-chart) page. + +### Use Legacy Helm Chart + + +<Warning> +`tyk-pro` chart is deprecated. Please use our [Tyk Stack helm chart](/product-stack/tyk-charts/tyk-stack-chart) instead. + +We recommend all users migrate to the `tyk-stack` Chart. Please review the [Configuration](/product-stack/tyk-charts/tyk-stack-chart) section of the new helm chart and cross-check with your existing configurations while planning for migration. +</Warning> + + +**Introduction** + +Tyk Helm chart is the preferred (and easiest) way to install **Tyk Self-Managed** on Kubernetes. +The helm chart `tyk-helm/tyk-pro` will install full Tyk platform with **Tyk Manager**, **Tyk Gateways** and **Tyk Pump** into your Kubernetes cluster. You can also choose to enable the installation of **Tyk Operator** (to manage your APIs in a declarative way). + +**Prerequisites** + +1. **Tyk License** + + If you are evaluating Tyk on Kubernetes, [contact us](https://tyk.io/about/contact/) to obtain a temporary license. + +2. **Data stores** + + The following are required for a Tyk Self-Managed installation: + - Redis - Should be installed in the cluster or reachable from inside the cluster (for SaaS option). + You can find instructions for a simple Redis installation bellow. + - MongoDB or SQL - Should be installed in the cluster or be reachable by the **Tyk Manager** (for SaaS option). + + You can find supported MongoDB and SQL versions [here](/planning-for-production/database-settings). + + Installation instructions for Redis and MongoDB/SQL are detailed below. + +3. **Helm** + + Installed [Helm 3](https://helm.sh/) + Tyk Helm Chart is using Helm v3 version (i.e. not Helm v2). + +**Installation** + +As well as our official Helm repo, you can also find it in [ArtifactHub](https://artifacthub.io/packages/helm/tyk-helm/tyk-pro). +[Open in ArtifactHub](https://artifacthub.io/packages/helm/tyk-helm/tyk-pro) + +If you are interested in contributing to our charts, suggesting changes, creating PRs or any other way, +please use [GitHub Tyk-helm-chart repo](https://github.com/TykTechnologies/tyk-helm-chart/tree/master/tyk-pro) +or contact us in [Tyk Community forum](https://community.tyk.io/) or through our sales team. + + +1. **Add Tyk official Helm repo to your local Helm repository** + + ```bash expandable + helm repo add tyk-helm https://helm.tyk.io/public/helm/charts/ + helm repo update + ``` + +2. **Create namespace for your Tyk deployment** + + ```bash + kubectl create namespace tyk + ``` + +3. **Getting the values.yaml of the chart** + + Before we proceed with installation of the chart you need to set some custom values. + To see what options are configurable on a chart and save that options to a custom values.yaml file run: + + ```bash + helm show values tyk-helm/tyk-pro > values.yaml + ``` + +**Installing the data stores** + +For Redis, MongoDB or SQL you can use these rather excellent charts provided by Bitnami + +<Tabs> +<Tab title="Redis"> +<br /> + +**Redis** +```bash +helm install tyk-redis bitnami/redis -n tyk --version 19.0.2 +``` + + +<Note> +Please make sure you are installing Redis versions that are supported by Tyk. Please refer to Tyk docs to get list of [supported versions](/planning-for-production/database-settings#redis). +</Note> + + +Follow the notes from the installation output to get connection details and password. + +```console + Redis(TM) can be accessed on the following DNS names from within your cluster: + + tyk-redis-master.tyk.svc.cluster.local for read/write operations (port 6379) + tyk-redis-replicas.tyk.svc.cluster.local for read-only operations (port 6379) + + export REDIS_PASSWORD=$(kubectl get secret --namespace tyk tyk-redis -o jsonpath="{.data.redis-password}" | base64 --decode) +``` expandable + +The DNS name of your Redis as set by Bitnami is `tyk-redis-master.tyk.svc.cluster.local:6379` (Tyk needs the name including the port) +You can update them in your local `values.yaml` file under `redis.addrs` and `redis.pass` +Alternatively, you can use `--set` flag to set it in Tyk installation. For example `--set redis.pass=$REDIS_PASSWORD` +</Tab> +<Tab title="MongoDB"> +<br /> + +**MongoDB** +```bash +helm install tyk-mongo bitnami/mongodb --set "replicaSet.enabled=true" -n tyk --version 15.1.3 +``` expandable + +<Note> +Bitnami MongoDB images is not supported on darwin/arm64 architecture. +</Note> + + +Follow the notes from the installation output to get connection details and password. The DNS name of your MongoDB as set with Bitnami is `tyk-mongo-mongodb.tyk.svc.cluster.local` and you also need to set the `authSource` parameter to `admin`. The full `mongoURL` should be similar to `mongoURL: mongodb://root:pass@tyk-mongo-mongodb.tyk.svc.cluster.local:27017/tyk_analytics?authSource=admin`. You can update them in your local `values.yaml` file under `mongo.mongoURL` Alternatively, you can use `--set` flag to set it in your Tyk installation. + + +<Note> +**Important Note regarding MongoDB** + +This Helm chart enables the *PodDisruptionBudget* for MongoDB with an arbiter replica-count of 1. If you intend to perform +system maintenance on the node where the MongoDB pod is running and this maintenance requires for the node to be drained, +this action will be prevented due the replica count being 1. Increase the replica count in the helm chart deployment to +a minimum of 2 to remedy this issue. +</Note> + + +</Tab> +<Tab title="SQL"> +<br /> + +**Postgres** +```bash +helm install tyk-postgres bitnami/postgresql --set "auth.database=tyk_analytics" -n tyk --version 12.12.10 +``` expandable + + +<Note> +Please make sure you are installing PostgreSQL versions that are supported by Tyk. Please refer to Tyk docs to get list of [supported versions](/api-management/dashboard-configuration#supported-database). +</Note> + + +Follow the notes from the installation output to get connection details and password. The DNS name of your Postgres service as set by Bitnami is `tyk-postgres-postgresql.tyk.svc.cluster.local`. +You can update connection details in `values.yaml` file under `postgres`. +</Tab> +</Tabs> + +--- + +**Quick Redis and MongoDB PoC installation** + +<Warning> +Another option for Redis and MongoDB, to get started quickly, is to use our **simple-redis** and **simple-mongodb** charts. +Please note that these provided charts must not ever be used in production and for anything +but a quick start evaluation only. Use external redis or Official Redis Helm chart in any other case. +We provide this chart, so you can quickly get up and running, however it is not meant for long term storage of data for example. + +```bash +helm install redis tyk-helm/simple-redis -n tyk +helm install mongo tyk-helm/simple-mongodb -n tyk +``` expandable +</Warning> + + +**License setting** + +For the **Tyk Self-Managed** chart we need to set the license key in your custom `values.yaml` file under `dash.license` field +or use `--set dash.license={YOUR-LICENSE_KEY}` with the `helm install` command. + + +Tyk Self-Managed licensing allow for different numbers of Gateway nodes to connect to a single Dashboard instance. +To ensure that your Gateway pods will not scale beyond your license allowance, please ensure that the Gateway's resource kind is `Deployment` +and the replica count to your license node limit. By default, the chart is configured to work with a single node license: `gateway.kind=Deployment` and `gateway.replicaCount=1`. + + +<Note> +**Please Note** + +There may be intermittent issues on the new pods during the rolling update process, when the total number of online +gateway pods is more than the license limit with lower amounts of Licensed nodes. +</Note> + + +**Installing Tyk Self managed** + +Now we can install the chart using our custom values: + +```bash +helm install tyk-pro tyk-helm/tyk-pro -f ./values.yaml -n tyk --wait +``` expandable + + +<Note> +**Important Note regarding MongoDB** + +The `--wait` argument is important to successfully complete the bootstrap of your **Tyk Manager**. +</Note> + + +**Pump Installation** + +By default pump installation is disabled. You can enable it by setting `pump.enabled` to `true` in `values.yaml` file. +Alternatively, you can use `--set pump.enabled=true` while doing helm install. + +**Quick Pump configuration(Supported from tyk helm v0.10.0)** + +*1. Mongo Pump* + +To configure mongo pump, do following changings in `values.yaml` file: +1. Set `backend` to `mongo`. +2. Set connection string in `mongo.mongoURL`. + +*2. Postgres Pump* + +To configure postgres pump, do following changings in `values.yaml` file: +1. Set `backend` to `postgres`. +2. Set connection string parameters in `postgres` section. + +**Tyk Developer Portal** + +You can disable the bootstrapping of the Developer Portal by the `portal.bootstrap: false` in your local `values.yaml` file. + +**Using TLS** + +You can turn on the TLS option under the gateway section in your local `values.yaml` file which will make your Gateway +listen on port 443 and load up a dummy certificate. You can set your own default certificate by replacing the file in the `certs/` folder. + +**Mounting Files** + +To mount files to any of the Tyk stack components, add the following to the mounts array in the section of that component. +For example: + ```bash + - name: aws-mongo-ssl-cert + filename: rds-combined-ca-bundle.pem + mountPath: /etc/certs +``` + +**Sharding APIs** + +Sharding is the ability for you to decide which of your APIs are loaded on which of your Tyk Gateways. This option is +turned off by default, however, you can turn it on by updating the `gateway.sharding.enabled` option. Once you do that you +will also need to set the `gateway.sharding.tags` field with the tags that you want that particular Gateway to load. (ex. tags: "external,ingress".) +You can then add those tags to your APIs in the API Designer, under the **Advanced Options** tab, and +the **Segment Tags (Node Segmentation)** section in your Tyk Dashboard. +Check [Tyk Gateway Sharding](/api-management/multiple-environments#what-is-api-sharding-) for more details. + + +### Install More Tyk Components + +**Installing Tyk Enterprise Developer Portal** + +If you are deploying the **Tyk Enterprise Developer Portal**, set the appropriate values under the `enterprisePortal` section in your `values.yaml`. Please visit [Tyk Enterprise Developer Portal installation](/portal/install#using-legacy-helm-chart) for a step by step guide. + +>**Note**: Helm chart supports Enterprise Portal v1.2.0+ + +**Installing Tyk Self-managed Control Plane** + +If you are deploying the **Tyk Control plane**, a.k.a **MDCB**, for a **Tyk Multi Data Center Bridge** deployment then you set +the `mdcb.enabled: true` option in the local `values.yaml` to add of the **MDCB** component to your installation. +Check [Tyk Control plane](/tyk-multi-data-centre) for more configuration details. + +This setting enables multi-cluster, multi Data-Center API management from a single dashboard. + + +**Tyk Identity Broker (TIB)** + +The **Tyk Identity Broker** (TIB) is a micro-service portal that provides a bridge between various Identity Management Systems +such as LDAP, OpenID Connect providers and legacy Basic Authentication providers, to your Tyk installation. +See [TIB](/api-management/external-service-integration#installing-tyk-identity-broker-tib) for more details. + +For SSO to **Tyk Manager** and **Tyk developer portal** purposes you do not need to install **TIB**, as its functionality is now +part of the **Tyk Manager**. However, if you want to run it separately (as you used to before this merge) or if you need it + as a broker for the **Tyk Gateway** you can do so. + +Once you have installed your **Tyk Gateway** and **Tyk Manager**, you can configure **TIB** by adding its configuration environment variables +under the `tib.extraEnvs` section and updating the `profile.json` in your `configs` folder. +See our [TIB GitHub repo](https://github.com/TykTechnologies/tyk-identity-broker#how-to-configure-tib). +Once you complete your modifications you can run the following command from the root of the repository to update your helm chart. + +```bash +helm upgrade tyk-pro values.yaml -n tyk +``` + +This chart implies there's a **ConfigMap** with a `profiles.json` definition in it. Please use `tib.configMap.profiles` value +to set the name of this **ConfigMap** (`tyk-tib-profiles-conf` by default). + + + +**Tyk Operator and Ingress** + +For a GitOps workflow used with a **Tyk Self-Managed** installation or setting the Tyk Gateway as a Kubernetes ingress controller, Tyk Operator enables you to manage API definitions, security policies and other Tyk features using Kubernetes manifest files. +To get started go to [Tyk Operator](/api-management/automations/operator). + +## Install on AWS Marketplace + + +Tyk offers a flexible and powerful API management solution through **Tyk Cloud** on the [AWS Marketplace](https://aws.amazon.com/marketplace/pp/prodview-pboluroscnqro). Tyk Cloud is an end-to-end managed API platform where both the control plane and gateways are installed on AWS for a seamless, fully cloud-hosted experience. + +For those who need more deployment flexibility, Tyk Cloud also supports a [Hybrid Gateway](/tyk-cloud#deploy-hybrid-gateways) option. In this setup, the control plane remains hosted and managed by Tyk on AWS, while the gateways can be deployed on your preferred cloud provider or on-premises environmentβ€”allowing you to meet data locality and compliance needs without sacrificing control. + +**Available AWS Deployment Regions** + +You can deploy Tyk Cloud in the following AWS regions: + +- **Singapore**: `aws-ap-southeast-1` +- **Frankfurt, Germany**: `aws-eu-central-1` +- **London, UK**: `aws-eu-west-2` +- **N. Virginia, USA**: `aws-us-east-1` +- **Oregon, USA**: `aws-us-west-2` +- **Australia**: `aws-ap-southeast-2` + +Getting started with Tyk Cloud via the AWS Marketplace is quick and easy. Sign up today to access Tyk’s comprehensive API management tools designed to scale with your needs. + +**Install Tyk on AWS EC2** + + +1. Spin up an [EC2 instance](https://aws.amazon.com/ec2/instance-types/), AWS Linux2 preferably, T2.Medium is fine + - add a public IP + - open up SG access to: + - 3000 for the Tyk Dashboard + - 8080 for the Tyk Gateway + - 22 TCP for SSH + +2. SSH into the instance +`ssh -i mykey.pem ec2-user@public-ec2-ip` + +3. Install Git, Docker, & Docker Compose +Feel free to copy paste these +```.sh expandable +sudo yum update -y +sudo yum install git -y +sudo yum install -y docker +sudo service docker start +sudo usermod -aG docker ec2-user +sudo su +sudo curl -L "https://github.com/docker/compose/releases/download/1.25.5/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose +sudo chmod +x /usr/local/bin/docker-compose +sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose +docker ps +``` + +4. Clone the Tyk Pro Docker repo + +```.bash +git clone https://github.com/TykTechnologies/tyk-pro-docker-demo +cd tyk-pro-docker-demo/ +``` + +5. Add the license key to `confs/tyk_analytics.conf` into the `license_key variable` using "vi" or "nano", etc + +**This is the most common place to have problems.** + +**Look for extra spaces between quotes ("") and the license key. It will not work if there are any.** + +Inside `tyk_analytics.conf`, `license_key` should look something like this, with a real license however: + +` +"license_key": "eyJhbGciOiJSUzI1NiIsInR5cCI...WQ", +` + +6. Run the containers via `docker-compose` + +```.bash +docker-compose up -d +``` + +7. Visit + +``` +http://<public-ec2-ip>:3000 +``` +and fill out the Bootstrap form! +**If you see any page besides the Bootstrap page, you have pasted the license key incorrectly** + +**Enable SSL for the Gateway & Dashboard** + +1. Add the following to `confs/tyk.conf` + +```.json expandable +"policies.policy_connection_string": "https://tyk-dashboard:3000" +"db_app_conf_options.connection_string": "https://tyk-dashboard:3000" +"http_server_options": { + "use_ssl": true, + "certificates": [ + { + "domain_name": "*.yoursite.com", + "cert_file": "./new.cert.cert", + "key_file": "./new.cert.key" + } + ], + "ssl_insecure_skip_verify": true ## YOU ONLY NEED THIS IF YOU ARE USING SELF SIGNED CERTS +} +``` + +2. Add the following to `confs/tyk_analytics.conf` + +```.json expandable +"tyk_api_config.Host": "https://tyk-gateway" +"http_server_options": { + "use_ssl": true, + "certificates": [ + { + "domain_name": "*.yoursite.com", + "cert_file": "./new.cert.cert", + "key_file": "./new.cert.key" + } + ] +} +``` + +3. Generate self-signed Certs: (Or bring your own CA signed) + +``` +openssl req -x509 -newkey rsa:4096 -keyout key.pem -out cert.pem -days 365 -nodes +``` + +4. Mount your certs to containers through `docker-compose.yml` + +```.yaml expandable +tyk-dashboard: + ... + volumes: + - ./cert.pem:/opt/tyk-dashboard/new.cert.cert + - ./key.pem:/opt/tyk-dashboard/new.cert.key +tyk-gateway: + ... + volumes: + - ./cert.pem:/opt/tyk-gateway/new.cert.cert + - ./key.pem:/opt/tyk-gateway/new.cert.key +``` + +5. Restart your containers with the mounted files + +``` +docker-compose up -d tyk-dashboard tyk-gateway +``` + +6. Download the bootstrap script onto EC2 machine + +``` +wget https://raw.githubusercontent.com/sedkis/tyk/master/scripts/bootstrap-ssl.sh +``` + +7. Apply execute permissions to file: + +```chmod +x bootstrap.sh``` + +8. Run the bootstrap script + +```./bootstrap.sh localhost``` expandable + +9. Done! use the generated user and password to log into The Tyk Dashboard + + +## Install with Ansible + + +<Note> +**Requirements** + +[Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) is required to run the following commands. +</Note> + + +**Getting Started** + +1. clone the [tyk-ansible](https://github.com/TykTechnologies/tyk-ansible) repositry + + ```bash + $ git clone https://github.com/TykTechnologies/tyk-ansible + ``` + +2. `cd` into the directory + + ```.bash + $ cd tyk-ansible + ``` + +3. Run initialisation script to initialise environment + + ```bash + $ sh scripts/init.sh + ``` + +4. Modify `hosts.yml` file to update ssh variables to your server(s). You can learn more about the hosts file [here](https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html) + +5. Run ansible-playbook to install the following: + + - Redis + - MongoDB or PostgreSQL + - Tyk Dashboard + - Tyk Gateway + - Tyk Pump + + ```bash + $ ansible-playbook playbook.yaml -t tyk-pro -t redis -t `mongodb` or `pgsql` + ``` + + You can choose to not install Redis, MongoDB or PostgreSQL by removing the `-t redis` or `-t mongodb` or `-t pgsql` However Redis and MongoDB or PostgreSQL are a requirement and need to be installed for the Tyk Pro installation to run. + + + + <Note> + For a production environment, we recommend that the Gateway, Dashboard and Pump are installed on separate machines. If installing multiple Gateways, you should install each on a separate machine. See [Planning for Production](/planning-for-production) For more details. + </Note> + + +**Supported Distributions** + +| Distribution | Version | Supported | +| --------- | :---------: | :---------: | +| Amazon Linux | 2 | βœ… | +| CentOS | 8 | βœ… | +| CentOS | 7 | βœ… | +| Debian | 10 | βœ… | +| Debian | 9 | βœ… | +| RHEL | 8 | βœ… | +| RHEL | 7 | βœ… | +| Ubuntu | 21 | βœ… | +| Ubuntu | 20 | βœ… | +| Ubuntu | 18 | βœ… | +| Ubuntu | 16 | βœ… | + +**Variables** + +- `vars/tyk.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| secrets.APISecret | `352d20ee67be67f6340b4c0605b044b7` | API secret | +| secrets.AdminSecret | `12345` | Admin secret | +| redis.host | | Redis server host if different than the hosts url | +| redis.port | `6379` | Redis server listening port | +| redis.pass | | Redis server password | +| redis.enableCluster | `false` | Enable if redis is running in cluster mode | +| redis.storage.database | `0` | Redis server database | +| redis.tls | `false` | Enable if redis connection is secured with SSL | +| mongo.host | | MongoDB server host if different than the hosts url | +| mongo.port | `27017` | MongoDB server listening port | +| mongo.tls | `false` | Enable if mongo connection is secured with SSL | +| pgsql.host | | PGSQL server host if different than the hosts url | +| pgsql.port | `5432` | PGSQL server listening port | +| pgsql.tls | `false` | Enable if pgsql connection is secured with SSL | +| dash.license | | Dashboard license| +| dash.service.host | | Dashboard server host if different than the hosts url | +| dash.service.port | `3000` | Dashboard server listening port | +| dash.service.proto | `http` | Dashboard server protocol | +| dash.service.tls | `false` | Set to `true` to enable SSL connections | +| gateway.service.host | | Gateway server host if different than the hosts url | +| gateway.service.port | `8080` | Gateway server listening port | +| gateway.service.proto | `http` | Gateway server protocol | +| gateway.service.tls | `false` | Set to `true` to enable SSL connections | +| gateway.sharding.enabled | `false` | Set to `true` to enable filtering (sharding) of APIs | +| gateway.sharding.tags | | The tags to use when filtering (sharding) Tyk Gateway nodes. Tags are processed as OR operations. If you include a non-filter tag (e.g. an identifier such as `node-id-1`, this will become available to your Dashboard analytics) | +| gateway.rpc.connString | | Use this setting to add the URL for your MDCB or load balancer host | +| gateway.rpc.useSSL | `true` | Set this option to `true` to use an SSL RPC connection| +| gateway.rpc.sslInsecureSkipVerify | `true` | Set this option to `true` to allow the certificate validation (certificate chain and hostname) to be skipped. This can be useful if you use a self-signed certificate | +| gateway.rpc.rpcKey | | Your organization ID to connect to the MDCB installation | +| gateway.rpc.apiKey | | This the API key of a user used to authenticate and authorize the Gateway’s access through MDCB. The user should be a standard Dashboard user with minimal privileges so as to reduce any risk if the user is compromised. The suggested security settings are read for Real-time notifications and the remaining options set to deny | +| gateway.rpc.groupId | | This is the `zone` that this instance inhabits, e.g. the cluster/data-center the Gateway lives in. The group ID must be the same across all the Gateways of a data-center/cluster which are also sharing the same Redis instance. This ID should also be unique per cluster (otherwise another Gateway cluster can pick up your keyspace events and your cluster will get zero updates). | + +- `vars/redis.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| redis_bind_interface | `0.0.0.0` | Binding address of Redis | + +Read more about Redis configuration [here](https://github.com/geerlingguy/ansible-role-redis). + +- `vars/mongodb.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| bind_ip | `0.0.0.0` | Binding address of MongoDB | +| mongodb_version | `4.4` | MongoDB version | + +Read more about MongoDB configuration [here](https://github.com/ansible-collections/community.mongodb). + +- `vars/pgsql.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| postgresql_databases[] | `[]` | Array of DBs to be created | +| postgresql_databases[].name | `tyk_analytics` | Database name | +| postgresql_users[] | `[]` | Array of users to be created | +| postgresql_users[`0`].name | `default` | User name | +| postgresql_users[`0`].password | `topsecretpassword` | User password | +| postgresql_global_config_options[] | `[]` | Postgres service config options | +| postgresql_global_config_options[`1`].option | `listen_addresses` | Listen address binding for the service | +| postgresql_global_config_options[`1`].value | `*` | Default value to listen to all addresses | +| postgresql_hba_entries[] | `[]` | Host based authenticaiton list| +| postgresql_hba_entries[`4`].type | `host` | Entry type | +| postgresql_hba_entries[`4`].database | `tyk_analytics` | Which database this entry will give access to | +| postgresql_hba_entries[`4`].user | `default` | What users this gain access from this entry | +| postgresql_hba_entries[`4`].address | `0.0.0.0/0` | What addresses this gain access from this entry | +| postgresql_hba_entries[`4`].auth_method | `md5` | What authentication method to to use for the users | + +Read more about PostgreSQL configuration [here](https://github.com/geerlingguy/ansible-role-postgresql). + + +## Install using Bootstrap CLI + +To list the available flags, execute `tyk-analytics bootstrap -h`: + +``` + usage: tyk-analytics bootstrap [<flags>] + + Bootstrap the Dashboard. + + Flags: + -h, --help Show context-sensitive help (also try --help-long and --help-man). + --version Show application version. + --conf="tyk_analytics.conf" + Load a named configuration file. + --create-org Create a new organisation. + --reuse-org=REUSE-ORG Reuse the organisation with given ID. + --drop-org=DROP-ORG Drop the organisation with given ID. +``` expandable + + +**Description** + +The `bootstrap` command makes bootstrapping easier. It helps you to create organizations and users. The command needs a + config file path. By default, it looks at `tyk_analytics.conf` in the directory where the `tyk-analytics` binary is located. + For example: + + ```tyk-analytics bootstrap``` + + You can also give the path of a custom config file with the `--conf` flag. For example: + + ```tyk-analytics bootstrap --conf some-directory/custom.conf``` + + The tool can work in both auto and interactive modes. You can use the flags while running the command or you can just run + it without flags and use interactive mode. + + +**Environment Variables** + +You can override the config values by environment variables. See [how to configure an environment variable](/tyk-oss-gateway/configuration). + +For example, you can override hostname, port, mongo url, redis host and redis port values by exporting the following variables: + +- **TYK_DB_HOSTCONFIG_HOSTNAME** +- **TYK_DB_LISTENPORT** +- **TYK_DB_MONGOURL** +- **TYK_DB_REDISHOST** +- **TYK_DB_REDISPORT** + + +## Install with Docker + + +Tyk has three containers that are available to set up a Docker installation: + +* [The Tyk Gateway container](https://hub.docker.com/r/tykio/tyk-gateway/) +* [The Tyk Dashboard container](https://hub.docker.com/r/tykio/tyk-dashboard/) +* [The Tyk Pump container](https://hub.docker.com/r/tykio/tyk-pump-docker-pub/) + +All three are required for a full deployment. We recommend that each container is installed on a separate machine for optimum performance. + +From v5.5.0 onwards, these images are based on [distroless](https://github.com/GoogleContainerTools/distroless). This means that you will not be able to obtain a shell with `docker run --rm -it tykio/tyk-gateway:v5.5.0 sh`. The image can be inspected with tools like [dive](https://github.com/wagoodman/dive) or [Docker Desktop](https://www.docker.com/products/docker-desktop/). + +We also have a [Docker Tyk Pro Demo](/deployment-and-operations/tyk-self-managed/tyk-demos-and-pocs/overview#docker-compose-setup), which installs our full Self-Managed solution, which includes our Gateway, Dashboard, and analytics processing pipeline. This demo will run Tyk Self-Managed on your machine. + + +## Install on Heroku + +**Install Tyk API Gateway on Heroku** + +A full Tyk Self-Managed installation can be deployed to Heroku dynos and workers using [Heroku Container Registry and Runtime](https://devcenter.heroku.com/articles/) functionality. This guide will utilize [Tyk Docker images](https://hub.docker.com/u/tykio/) with a small amount of customization as well as an external MongoDB service. + + +**Prerequisites** + +1. Docker daemon installed and running locally +2. [Heroku account](https://www.heroku.com/), the free plan is sufficient for a basic PoC but not recommended for production usage +3. [Heroku CLI](https://devcenter.heroku.com/articles/heroku-cli) installed +4. MongoDB service (such as [Atlas](https://www.mongodb.com/cloud/atlas), [mLab](https://elements.heroku.com/addons/mongolab), or your own deployment), this guide is based on MongoDB Atlas but others should work as well +5. [Tyk License](https://tyk.io/pricing/on-premise/) (note that in case of running multiple gateway dynos, license type must match) +6. Checkout the [Tyk quickstart repository](https://github.com/TykTechnologies/tyk-pro-heroku) from GitHub +7. Python 2 or 3 in order to execute the bootstrap script + +**Creating Heroku Apps** + +We will create two Heroku apps, one for the Tyk Gateway (with [Redis add-on](https://devcenter.heroku.com/articles/heroku-redis) attached to it) and another for the Dashboard and Pump. + +Given Heroku CLI is installed and your Heroku account is available, log into it: +```bash +heroku login +``` + +Now create the Gateway app and note down its name: +```bash +heroku create +``` +``` +Creating app... done, β¬’ infinite-plains-14949 +https://infinite-plains-14949.herokuapp.com/ | https://git.heroku.com/infinite-plains-14949.git +``` + +<Note> +`--space` flag must be added to the command if the app is being created in a private space, see more details in the section on Heroku private spaces (below). +</Note> + + +Provision a Redis add-on (we'll use a `hobby-dev` plan for demonstration purposes but that's not suitable for production), replacing the app name with your own: +```bash +heroku addons:create heroku-redis:hobby-dev -a infinite-plains-14949 +``` +``` expandable +Creating heroku-redis:hobby-dev on β¬’ infinite-plains-14949... free +Your add-on should be available in a few minutes. +! WARNING: Data stored in hobby plans on Heroku Redis are not persisted. +redis-infinite-35445 is being created in the background. The app will restart when complete... +Use heroku addons:info redis-infinite-35445 to check creation progress +Use heroku addons:docs heroku-redis to view documentation +``` + +Once add-on provisioning is done, the info command (replacing the add-on name with your own) will show the following output: +```bash +heroku addons:info redis-infinite-35445 +``` +``` expandable +=== redis-infinite-35445 +Attachments: infinite-plains-14949::REDIS +Installed at: Sun May 18 2018 14:23:21 GMT+0300 (EEST) +Owning app: infinite-plains-14949 +Plan: heroku-redis:hobby-dev +Price: free +State: created +``` + +Time to create the Dashboard app and note down its name as well: +```bash +heroku create +``` +``` +Creating app... done, β¬’ evening-beach-40625 +https://evening-beach-40625.herokuapp.com/ | https://git.heroku.com/evening-beach-40625.git +``` + +Since the Dashboard and Pump need access to the same Redis instance as the gateway, we'll need to share the Gateway app's add-on with this new app: +```bash +heroku addons:attach infinite-plains-14949::REDIS -a evening-beach-40625 +``` +``` +Attaching redis-infinite-35445 to β¬’ evening-beach-40625... done +Setting REDIS config vars and restarting β¬’ evening-beach-40625... done, v3 +``` + +To check that both apps have access to the same Redis add-on, we can utilize the `heroku config` command and check for the Redis endpoint: +```bash +heroku config -a infinite-plains-14949 | grep REDIS_URL +heroku config -a evening-beach-40625 | grep REDIS_URL +``` + +Their outputs should match. + +**Deploy the Dashboard** + +It's recommended to start with the Dashboard so in your Heroku quickstart clone run: +```bash +cd analytics +ls dashboard +``` +``` +bootstrap.sh Dockerfile.web entrypoint.sh tyk_analytics.conf +``` + +You will find it contains a `Dockerfile.web` for the web dyno, a config file for the Dashboard, entrypoint script for the Docker container and a bootstrap script for seeding the dashboard instance with sample data. All these files are editable for your purposes but have sane defaults for a PoC. + + +<Note> +You can use the `FROM` statement in `Dockerfile.web` to use specific dashboard version and upgrade when needed instead of relying on the `latest` tag. +</Note> + + + +The [Dashboard configuration](/tyk-dashboard/configuration) can be changed by either editing the `tyk_analytics.conf` file or injecting them as [environment variables](/tyk-oss-gateway/configuration) via `heroku config`. In this guide we'll use the latter for simplicity of demonstration but there is merit to both methods. + +First let's set the license key: +```bash +heroku config:set TYK_DB_LICENSEKEY="your license key here" -a evening-beach-40625 +``` +``` +Setting TYK_DB_LICENSEKEY and restarting β¬’ evening-beach-40625... done, v4 +TYK_DB_LICENSEKEY: should show your license key here +``` + +Now the MongoDB endpoint (replacing with your actual endpoint): +```bash +heroku config:set TYK_DB_MONGOURL="mongodb://user:pass@mongoprimary.net:27017,mongosecondary.net:27017,mongotertiary.net:27017" -a evening-beach-40625 +``` +``` +Setting TYK_DB_MONGOURL and restarting β¬’ evening-beach-40625... done, v5 +TYK_DB_MONGOURL: mongodb://user:pass@mongoprimary.net:27017,mongosecondary.net:27017,mongotertiary.net:27017 +``` + +And enable SSL for it if your service supports/requires this: +```bash +heroku config:set TYK_DB_MONGOUSESSL="true" -a evening-beach-40625 +``` +``` +Setting TYK_DB_MONGOUSESSL and restarting β¬’ evening-beach-40625... done, v6 +TYK_DB_MONGOUSESSL: true +``` + +Since the Tyk Dashboard needs to access gateways sometimes, we'll need to specify the Gateway endpoint too, which is the Gateway app's URL: +```bash +heroku config:set TYK_DB_TYKAPI_HOST="https://infinite-plains-14949.herokuapp.com" -a evening-beach-40625 +heroku config:set TYK_DB_TYKAPI_PORT="443" -a evening-beach-40625 +``` +``` +Setting TYK_DB_TYKAPI_HOST and restarting β¬’ evening-beach-40625... done, v7 +TYK_DB_TYKAPI_HOST: https://infinite-plains-14949.herokuapp.com +Setting TYK_DB_TYKAPI_PORT and restarting β¬’ evening-beach-40625... done, v8 +TYK_DB_TYKAPI_PORT: 443 +``` + +This is enough for a basic Dashboard setup but we recommend also changing at least node and admin secrets with strong random values, as well as exploring other config options. + +Since the Tyk Pump is also a part of this application (as a worker process), we'll need to configure it too. + +```bash +ls pump +``` +``` +Dockerfile.pump entrypoint.sh pump.conf +``` + +Same principles apply here as well. Here we'll need to configure MongoDB endpoints for all the Pumps (this can also be done in the `pump.conf` file): +```bash +heroku config:set PMP_MONGO_MONGOURL="mongodb://user:pass@mongoprimary.net:27017,mongosecondary.net:27017,mongotertiary.net:27017" -a evening-beach-40625 +heroku config:set PMP_MONGO_MONGOUSESSL="true" + +heroku config:set PMP_MONGOAGG_MONGOURL="mongodb://user:pass@mongoprimary.net:27017,mongosecondary.net:27017,mongotertiary.net:27017" -a evening-beach-40625 +heroku config:set PMP_MONGOAGG_MONGOUSESSL="true" +``` + +With the configuration in place it's finally time to deploy our app to Heroku. + +First, make sure CLI is logged in to Heroku containers registry: +```bash +heroku container:login +``` +``` +Login Succeeded +``` + +Provided you're currently in `analytics` directory of the quickstart repo: +```bash +heroku container:push --recursive -a evening-beach-40625 +``` +``` expandable +=== Building web (/tyk-heroku-docker/analytics/dashboard/Dockerfile.web) +Sending build context to Docker daemon 8.192kB +Step 1/5 : FROM tykio/tyk-dashboard:v1.6.1 + ---> fdbc67b43139 +Step 2/5 : COPY tyk_analytics.conf /opt/tyk-dashboard/tyk_analytics.conf + ---> 89be9913798b +Step 3/5 : COPY entrypoint.sh /opt/tyk-dashboard/entrypoint.sh + ---> c256152bff29 +Step 4/5 : ENTRYPOINT ["/bin/sh", "-c"] + ---> Running in bc9fe7a569c0 +Removing intermediate container bc9fe7a569c0 + ---> f40e6b259230 +Step 5/5 : CMD ["/opt/tyk-dashboard/entrypoint.sh"] + ---> Running in 705273810eea +Removing intermediate container 705273810eea + ---> abe9f10e8b21 +Successfully built abe9f10e8b21 +Successfully tagged registry.heroku.com/evening-beach-40625/web:latest +=== Building pump (/tyk-heroku-docker/analytics/pump/Dockerfile.pump) +Sending build context to Docker daemon 5.12kB +Step 1/5 : FROM tykio/tyk-pump-docker-pub:v0.5.2 + ---> 247c6b5795a9 +Step 2/5 : COPY pump.conf /opt/tyk-pump/pump.conf + ---> 1befeab8f092 +Step 3/5 : COPY entrypoint.sh /opt/tyk-pump/entrypoint.sh + ---> f8ad0681aa70 +Step 4/5 : ENTRYPOINT ["/bin/sh", "-c"] + ---> Running in 0c30d35b9e2b +Removing intermediate container 0c30d35b9e2b + ---> b17bd6a8ed44 +Step 5/5 : CMD ["/opt/tyk-pump/entrypoint.sh"] + ---> Running in a16acb453b62 +Removing intermediate container a16acb453b62 + ---> 47ac9f221d8d +Successfully built 47ac9f221d8d +Successfully tagged registry.heroku.com/evening-beach-40625/pump:latest +=== Pushing web (/tyk-heroku-docker/analytics/dashboard/Dockerfile.web) +The push refers to repository [registry.heroku.com/evening-beach-40625/web] +c60cf00e6e9b: Pushed +11d074829795: Pushed +8b72aa2b2acc: Pushed +ca2feecf234c: Pushed +803aafd71223: Pushed +43efe85a991c: Pushed +latest: digest: sha256:b857afaa69154597558afb2462896275ab667b729072fac224487f140427fa73 size: 1574 +=== Pushing pump (/tyk-heroku-docker/analytics/pump/Dockerfile.pump) +The push refers to repository [registry.heroku.com/evening-beach-40625/pump] +eeddc94b8282: Pushed +37f3b3ce56ab: Pushed +4b61531ec7dc: Pushed +eca9efd615d9: Pushed +0f700064c5a1: Pushed +43efe85a991c: Mounted from evening-beach-40625/web +latest: digest: sha256:f45acaefa3b47a126dd784a888c89e420814ad3031d3d4d4885e340a59aec31c size: 1573 +``` + +This has built Docker images for both dashboard and pump, as well as pushed them to Heroku registry and automatically deployed to the application. + +Provided everything went well (and if not, inspect the application logs), you should be seeing the Dashboard login page at your app URL (e.g "https://evening-beach-40625.herokuapp.com/"). + +However, it doesn't yet have any accounts. It order to populate it please run the `dashboard/bootstrap.sh` script: +```bash +dashboard/bootstrap.sh evening-beach-40625.herokuapp.com +``` +``` expandable +Creating Organization +ORGID: 5b016ca530867500050b9e90 +Adding new user +USER AUTH: a0f7c1e878634a60599dc037489a880f +NEW ID: 5b016ca6dcd0056d702dc40e +Setting password + +DONE +==== +Login at https://evening-beach-40625.herokuapp.com/ +User: c7ze82m8k3@default.com +Pass: test123 +``` + +It will generate a default organization with random admin username and a specified password. The bootstrap script can be edited to suit your needs as well as just editing the user info in the dashboard. + +If this was successful, you should be able to log into your dashboard now. + +The last step in this app is to start the Pump worker dyno since by default only the web dyno is enabled: +```bash +heroku dyno:scale pump=1 -a evening-beach-40625 +``` +``` +Scaling dynos... done, now running pump at 1:Free +``` + +At that point the dyno formation should look like this: +```bash +heroku dyno:scale -a evening-beach-40625 +``` +``` +pump=1:Free web=1:Free +``` + +**Deploy the Gateway** + +The process is very similar for the Tyk Gateway, except it doesn't have a worker process and doesn't need access to MongoDB. + +```bash +cd ../gateway +ls +``` +``` +Dockerfile.web entrypoint.sh tyk.conf +``` + +All these files serve the same purpose as with the Dasboard and the Pump. [Configuration](/tyk-oss-gateway/configuration) can either be edited in `tyk.conf` or [injected](/tyk-oss-gateway/configuration) with `heroku config`. + +To get things going we'll need to set following options for the Dashboard endpoint (substituting the actual endpoint and the app name, now for the gateway app): +```bash +heroku config:set TYK_GW_DBAPPCONFOPTIONS_CONNECTIONSTRING="https://evening-beach-40625.herokuapp.com" -a infinite-plains-14949 +heroku config:set TYK_GW_POLICIES_POLICYCONNECTIONSTRING="https://evening-beach-40625.herokuapp.com" -a infinite-plains-14949 +``` +``` +Setting TYK_GW_DBAPPCONFOPTIONS_CONNECTIONSTRING and restarting β¬’ infinite-plains-14949... done, v4 +TYK_GW_DBAPPCONFOPTIONS_CONNECTIONSTRING: https://evening-beach-40625.herokuapp.com +Setting TYK_GW_POLICIES_POLICYCONNECTIONSTRING and restarting β¬’ infinite-plains-14949... done, v5 +TYK_GW_POLICIES_POLICYCONNECTIONSTRING: https://evening-beach-40625.herokuapp.com +``` + +Since the Redis configuration will be automatically discovered (it's already injected by Heroku), we're ready to deploy: +```bash +heroku container:push --recursive -a infinite-plains-14949 +``` +``` expandable +=== Building web (/tyk-heroku-docker/gateway/Dockerfile.web) +Sending build context to Docker daemon 6.144kB +Step 1/5 : FROM tykio/tyk-gateway:v2.6.1 + ---> f1201002e0b7 +Step 2/5 : COPY tyk.conf /opt/tyk-gateway/tyk.conf + ---> b118611dc36b +Step 3/5 : COPY entrypoint.sh /opt/tyk-gateway/entrypoint.sh + ---> 68ad364030cd +Step 4/5 : ENTRYPOINT ["/bin/sh", "-c"] + ---> Running in 859f4c15a0d2 +Removing intermediate container 859f4c15a0d2 + ---> 5f8c0d1b378a +Step 5/5 : CMD ["/opt/tyk-gateway/entrypoint.sh"] + ---> Running in 44c5e4c87708 +Removing intermediate container 44c5e4c87708 + ---> 86a9eb509968 +Successfully built 86a9eb509968 +Successfully tagged registry.heroku.com/infinite-plains-14949/web:latest +=== Pushing web (/tyk-heroku-docker/gateway/Dockerfile.web) +The push refers to repository [registry.heroku.com/infinite-plains-14949/web] +b8a4c3e3f93c: Pushed +0b7bae5497cd: Pushed +e8964f363bf4: Pushed +379aae48d347: Pushed +ab2b28b92877: Pushed +021ee50b0983: Pushed +43efe85a991c: Mounted from evening-beach-40625/pump +latest: digest: sha256:d67b8f55d729bb56e06fe38e17c2016a36f2edcd4f01760c0e62a13bb3c9ed38 size: 1781 +``` + +Inspect the logs (`heroku logs -a infinite-plains-14949`) to check that deployment was successful, also the node should be registered by the Dashboard in "System Management" -> "Nodes and Licenses" section. + +You're ready to follow the guide on [creating and managing your APIs](/api-management/gateway-config-managing-classic#create-an-api) with this Heroku deployment. + + +<Note> +To use the [geographic log distribution](/api-management/dashboard-configuration#activity-by-location) feature in the Dashboard please supply the GeoLite2 DB in the `gateway` directory, uncomment the marked line in `Dockerfile.web` and set the `analytics_config.enable_geo_ip` setting (or `TYK_GW_ANALYTICSCONFIG_ENABLEGEOIP` env var) to `true`. +</Note> + + +**Heroku Private Spaces** + +Most instructions are valid for [Heroku Private Spaces runtime](https://devcenter.heroku.com/articles/private-spaces). However there are several differences to keep in mind. + +Heroku app creation commands must include the private space name in the `--space` flag, e.g.: +```bash +heroku create --space test-space-virginia +``` + +When deploying to the app, the container must be released manually after pushing the image to the app: +```bash +heroku container:push --recursive -a analytics-app-name +heroku container:release web -a analytics-app-name +heroku container:release pump -a analytics-app-name +``` + +Similarly, the Gateway: +```bash +heroku container:push --recursive -a gateway-app-name +heroku container:release web -a gateway-app-name +``` + +Please allow several minutes for the first deployment to start as additional infrastructure is being created for it. Next deployments are faster. + +Private spaces maintain stable set of IPs that can be used for allowing fixed set of IPs on your upstream side (e.g. on an external database service). Find them using the following command: +```bash +heroku spaces:info --space test-space-virginia +``` + +Alternatively VPC peering can be used with the private spaces if external service supports it. This way exposure to external network can be avoided. For instance, see [MongoDB Atlas guide](https://www.mongodb.com/blog/post/integrating-mongodb-atlas-with-heroku-private-spaces) for setting this up. + +The minimal Heroku Redis add-on plan that installs into your private space is currently `private-7`. Please refer to [Heroku's Redis with private spaces guide](https://devcenter.heroku.com/articles/heroku-redis-and-private-spaces) for more information. + +Apps in private spaces don't enable SSL/TLS by default. It needs to be configured in the app settings along with the domain name for it. If it's not enabled, please make sure that configs that refer to corresponding hosts are using HTTP instead of HTTPS and related ports (80 for HTTP). + +**Gateway Plugins** + +In order to enable [rich plugins](/api-management/plugins/rich-plugins#) for the Gateway, please set the following Heroku config option to either `python` or `lua` depending on the type of plugins used: +```bash +heroku config:set TYK_PLUGINS="python" -a infinite-plains-14949 +``` +``` +Setting TYK_PLUGINS and restarting β¬’ infinite-plains-14949... done, v9 +TYK_PLUGINS: python +``` + +After re-starting the Gateway, the logs should be showing something similar to this: +``` +2018-05-18T13:13:50.272511+00:00 app[web.1]: Tyk will be using python plugins +2018-05-18T13:13:50.311510+00:00 app[web.1]: time="May 18 13:13:50" level=info msg="Setting PYTHONPATH to 'coprocess/python:middleware/python:event_handlers:coprocess/python/proto'" +2018-05-18T13:13:50.311544+00:00 app[web.1]: time="May 18 13:13:50" level=info msg="Initializing interpreter, Py_Initialize()" +2018-05-18T13:13:50.497815+00:00 app[web.1]: time="May 18 13:13:50" level=info msg="Initializing dispatcher" +``` + +Set this variable back to an empty value in order to revert back to the default behavior. + +**Upgrading or Customizing Tyk** + +Since this deployment is based on Docker images and containers, upgrading or making changes to the deployment is as easy as building a new image and pushing it to the registry. + +Specifically, upgrading version of any Tyk components is done by editing the corresponding `Dockerfile` and replacing the base image version tag. E.g. changing `FROM tykio/tyk-gateway:v2.5.4` to `FROM tykio/tyk-gateway:v2.6.1` will pull the Tyk gateway 2.6.1. We highly recommend specifying concrete version tags instead of `latest` for better house keeping. + +Once these changes have been made just run `heroku container:push --recursive -a app_name` on the corresponding directory as shown previously in this guide. This will do all the building and pushing as well as gracefully deploying on your Heroku app. + + +Please refer to [Heroku documentation on containers and registry](https://devcenter.heroku.com/articles/container-registry-and-runtime) for more information. + + +## Install on Microsoft Azure + +[Azure](https://azure.microsoft.com/en-us/explore/) is Microsoft's cloud services platform. It supports both the running of [Ubuntu Servers](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/canonical.0001-com-ubuntu-server-focal?tab=overview), as well as [Docker](https://learn.microsoft.com/en-us/previous-versions/azure/virtual-machines/linux/docker-machine) and [Docker-Compose](https://docs.microsoft.com/en-us/azure/virtual-machines/linux/docker-compose-quickstart). + +For more details, see the [Azure Documentation](https://docs.microsoft.com/en-us/azure/). + +**Tyk Installation Options for Azure ** + +Azure allows you to install Tyk in the following ways: + +**On-Premises** + +1. Via our [Ubuntu Setup](/tyk-self-managed/install#debian-ubuntu-install-gateway) on an installed Ubuntu Server on Azure. +2. Via our [Docker Installation](/deployment-and-operations/tyk-self-managed/tyk-demos-and-pocs/overview#docker-compose-setup) using Azure's Docker support. + +See our video for installing Tyk on Ubuntu via Azure: + +<iframe width="560" height="315" src="https://www.youtube.com/embed/-Q9Lox-DyTU" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe> + +We also have a [blog post](https://tyk.io/blog/getting-started-with-tyk-on-microsoft-azure-and-ubuntu/) that walks you through installing Tyk on Azure. + + +## Install to Google Cloud + +[GCP](https://cloud.google.com/) is Google's Cloud services platform. It supports both the running of [Ubuntu Servers](https://console.cloud.google.com/marketplace/browse?q=ubuntu%2020.04) and [Docker](https://cloud.google.com/build/docs/cloud-builders). + +For more details, see the [Google Cloud Documentation](https://cloud.google.com/docs). + +**Tyk Installation Options for Google CLoud ** + +Google Cloud allows you to install Tyk in the following ways: + +**On-Premises** + +1. Via our [Ubuntu Setup](/tyk-self-managed/install#debian-ubuntu-install-gateway) on an installed Ubuntu Server within Google Cloud. +2. Via our [Docker Installation](/deployment-and-operations/tyk-self-managed/tyk-demos-and-pocs/overview#docker-compose-setup) using Google Cloud's Docker support. + +**Tyk Pump on GCP** + +When running Tyk Pump in GCP using [Cloud Run](https://cloud.google.com/run/docs/overview/what-is-cloud-run) it is available 24/7. However, since it is serverless you also need to ensure that the _CPU always allocated_ option is configured to ensure availability of the analytics. Otherwise, for each request there will be a lag between the Tyk Pump container starting up and having the CPU allocated. Subsequently, the analytics would only be available during this time. + +1. Configure Cloud Run to have the [CPU always allocated](https://cloud.google.com/run/docs/configuring/cpu-allocation#setting) option enabled. Otherwise, the Tyk Pump container needs to warm up, which takes approximately 1 min. Subsequently, by this time the stats are removed from Redis. + +2. Update the Tyk Gateway [configuration](/tyk-oss-gateway/configuration#analytics_configstorage_expiration_time) to keep the stats for 3 mins to allow Tyk Pump to process them. This value should be greater than the Pump [purge delay](/tyk-pump/tyk-pump-configuration/tyk-pump-environment-variables#purge_delay) to ensure the analytics data exists long enough in Redis to be processed by the Pump. + + +<h2 id="install-tyk-on-redhat-rhel-centos">Install Tyk on Red Hat (RHEL / CentOS)</h2> +Select the preferred way of installing Tyk by selecting **Shell** or **Ansible** tab for instructions. +There are 4 components which needs to be installed. Each can be installed via shell or ansible + +### Install Database + +#### Using Shell + +**Supported Distributions** +| Distribution | Version | Supported | +| --------- | :---------: | :---------: | +| CentOS | 7 | βœ… | +| RHEL | 9 | βœ… | +| RHEL | 8 | βœ… | +| RHEL | 7 | βœ… | + + +**Install and Configure Dependencies** +<br /> + +**Redis** + +Tyk Gateway has a [dependency](/planning-for-production/database-settings#redis) on Redis. Follow the steps provided by Red Hat to make the installation of Redis, conducting a [search](https://access.redhat.com/search/?q=redis) for the correct version and distribution. + +**Storage Database** + +Tyk Dashboard has a dependency on a storage database that can be [PostgreSQL](/planning-for-production/database-settings#postgresql) or [MongoDB](/planning-for-production/database-settings#mongodb-sizing-guidelines). + + +**Option 1: Install PostgreSQL** + +Check the PostgreSQL supported [versions](/planning-for-production/database-settings#postgresql). Follow the steps provided by [PostgreSQL](https://www.postgresql.org/download/linux/redhat/) to install it. + +Configure PostgreSQL + +Create a new role/user +```console +sudo -u postgres createuser --interactive +``` +The name of the role can be "tyk" and say yes to make it a superuser + +Create a matching DB with the same name. Postgres authentication system assumes by default that for any role used to log in, that role will have a database with the same name which it can access. +```console +sudo -u postgres createdb tyk +``` +Add another user to be used to log into your operating system + +```console +sudo adduser tyk +``` +Log in to your Database +```console +sudo -u tyk psql +``` +Update the user β€œtyk” to have a password +```console +ALTER ROLE tyk with PASSWORD '123456'; +``` +Create a DB (my example is tyk_analytics) +```console +sudo -u tyk createdb tyk_analytics +``` +**Option 2: Install MongoDB** +<br /> +Check the MongoDB supported [versions](/planning-for-production/database-settings#mongodb-sizing-guidelines). Follow the steps provided by [MongoDB](https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-red-hat/) to install it. + +Optionally initialize the database and enable automatic start: +```console +# Optionally ensure that MongoDB will start following a system reboot +sudo systemctl enable mongod +# start MongoDB server +sudo systemctl start mongod +``` + +#### Using Ansible +You can install Tyk on RHEL or CentOS using our YUM repositories. Follow the guides and tutorials in this section to have Tyk up and running in no time. + +The order is to install Tyk Dashboard, then Tyk Pump and then Tyk Gateway for a full stack. + +- [Dashboard](#install-dashboard) +- [Pump](#install-pump) +- [Gateway](#install-gateway) + + + + <Note> + For a production environment, we recommend that the Tyk Gateway, Tyk Dashboard and Tyk Pump are installed on separate machines. If installing multiple Tyk Gateways, you should install each on a separate machine. See [Planning for Production](/planning-for-production) for more details. + </Note> + + +**Supported Distributions** + +| Distribution | Version | Supported | +| --------- | :---------: | :---------: | +| CentOS | 7 | βœ… | +| RHEL | 8 | βœ… | +| RHEL | 7 | βœ… | + + +**Requirements** + +[Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) - required for running the commands below. + +**Getting Started** + +1. clone the [tyk-ansible](https://github.com/TykTechnologies/tyk-ansible) repositry + + ```console expandable + $ git clone https://github.com/TykTechnologies/tyk-ansible + ``` + +2. `cd` into the directory + + ```console + $ cd tyk-ansible + ``` + +3. Run initialisation script to initialise environment + + ```console + $ sh scripts/init.sh + ``` + +4. Modify `hosts.yml` file to update ssh variables to your server(s). You can learn more about the hosts file [here](https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html) + +5. Run ansible-playbook to install the following: + + - Redis + - MongoDB or PostgreSQL + - Tyk Dashboard + - Tyk Gateway + - Tyk Pump + + ```console + $ ansible-playbook playbook.yaml -t tyk-pro -t redis -t `mongodb` or `pgsql` + ``` + + You can choose to not install Redis, MongoDB or PostgreSQL by removing the `-t redis` or `-t mongodb` or `-t pgsql` However Redis and MongoDB or PostgreSQL are a requirement and need to be installed for the Tyk Pro installation to run. + +**Variables** + +- `vars/tyk.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| secrets.APISecret | `352d20ee67be67f6340b4c0605b044b7` | API secret | +| secrets.AdminSecret | `12345` | Admin secret | +| redis.host | | Redis server host if different than the hosts url | +| redis.port | `6379` | Redis server listening port | +| redis.pass | | Redis server password | +| redis.enableCluster | `false` | Enable if redis is running in cluster mode | +| redis.storage.database | `0` | Redis server database | +| redis.tls | `false` | Enable if redis connection is secured with SSL | +| mongo.host | | MongoDB server host if different than the hosts url | +| mongo.port | `27017` | MongoDB server listening port | +| mongo.tls | `false` | Enable if mongo connection is secured with SSL | +| pgsql.host | | PGSQL server host if different than the hosts url | +| pgsql.port | `5432` | PGSQL server listening port | +| pgsql.tls | `false` | Enable if pgsql connection is secured with SSL | +| dash.license | | Dashboard license| +| dash.service.host | | Dashboard server host if different than the hosts url | +| dash.service.port | `3000` | Dashboard server listening port | +| dash.service.proto | `http` | Dashboard server protocol | +| dash.service.tls | `false` | Set to `true` to enable SSL connections | +| gateway.service.host | | Gateway server host if different than the hosts url | +| gateway.service.port | `8080` | Gateway server listening port | +| gateway.service.proto | `http` | Gateway server protocol | +| gateway.service.tls | `false` | Set to `true` to enable SSL connections | +| gateway.sharding.enabled | `false` | Set to `true` to enable filtering (sharding) of APIs | +| gateway.sharding.tags | | The tags to use when filtering (sharding) Tyk Gateway nodes. Tags are processed as OR operations. If you include a non-filter tag (e.g. an identifier such as `node-id-1`, this will become available to your Dashboard analytics) | +| gateway.rpc.connString | | Use this setting to add the URL for your MDCB or load balancer host | +| gateway.rpc.useSSL | `true` | Set this option to `true` to use an SSL RPC connection| +| gateway.rpc.sslInsecureSkipVerify | `true` | Set this option to `true` to allow the certificate validation (certificate chain and hostname) to be skipped. This can be useful if you use a self-signed certificate | +| gateway.rpc.rpcKey | | Your organization ID to connect to the MDCB installation | +| gateway.rpc.apiKey | | This the API key of a user used to authenticate and authorize the Gateway’s access through MDCB. The user should be a standard Dashboard user with minimal privileges so as to reduce any risk if the user is compromised. The suggested security settings are read for Real-time notifications and the remaining options set to deny | +| gateway.rpc.groupId | | This is the `zone` that this instance inhabits, e.g. the cluster/data-center the Gateway lives in. The group ID must be the same across all the Gateways of a data-center/cluster which are also sharing the same Redis instance. This ID should also be unique per cluster (otherwise another Gateway cluster can pick up your keyspace events and your cluster will get zero updates). | + +- `vars/redis.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| redis_bind_interface | `0.0.0.0` | Binding address of Redis | + +Read more about Redis configuration [here](https://github.com/geerlingguy/ansible-role-redis). + +- `vars/mongodb.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| bind_ip | `0.0.0.0` | Binding address of MongoDB | +| mongodb_version | `4.4` | MongoDB version | + +Read more about MongoDB configuration [here](https://github.com/ansible-collections/community.mongodb). + +- `vars/pgsql.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| postgresql_databases[] | `[]` | Array of DBs to be created | +| postgresql_databases[].name | `tyk_analytics` | Database name | +| postgresql_users[] | `[]` | Array of users to be created | +| postgresql_users[`0`].name | `default` | User name | +| postgresql_users[`0`].password | `topsecretpassword` | User password | +| postgresql_global_config_options[] | `[]` | Postgres service config options | +| postgresql_global_config_options[`1`].option | `listen_addresses` | Listen address binding for the service | +| postgresql_global_config_options[`1`].value | `*` | Default value to listen to all addresses | +| postgresql_hba_entries[] | `[]` | Host based authenticaiton list| +| postgresql_hba_entries[`4`].type | `host` | Entry type | +| postgresql_hba_entries[`4`].database | `tyk_analytics` | Which database this entry will give access to | +| postgresql_hba_entries[`4`].user | `default` | What users this gain access from this entry | +| postgresql_hba_entries[`4`].address | `0.0.0.0/0` | What addresses this gain access from this entry | +| postgresql_hba_entries[`4`].auth_method | `md5` | What authentication method to to use for the users | + +Read more about PostgreSQL configuration [here](https://github.com/geerlingguy/ansible-role-postgresql). + + +### Install Dashboard + +#### Using Shell + +Tyk has its own signed RPMs in a YUM repository hosted by the kind folks at [packagecloud.io](https://packagecloud.io/tyk/tyk-dashboard/install#manual-rpm), which makes it easy, safe and secure to install a trusted distribution of the Tyk Gateway stack. + +This configuration should also work (with some tweaks) for CentOS. + +**Prerequisites** + +* Ensure port `3000` is open: This is used by the Dashboard to provide the GUI and the Classic Developer Portal. +* Follow the steps provided in this link [Getting started on Red Hat (RHEL / CentOS)](#install-tyk-on-redhat-rhel-centos) to install and configure Tyk dependencies. + +1. **Set up YUM Repositories** + + First, install two package management utilities `yum-utils` and a file downloading tool `wget`: + ```bash + sudo yum install yum-utils wget + ``` + Then install Python: + ```bash + sudo yum install python3 + ``` + +2. **Configure and Install the Tyk Dashboard** + + Create a file named `/etc/yum.repos.d/tyk_tyk-dashboard.repo` that contains the repository configuration settings for YUM repositories `tyk_tyk-dashboard` and `tyk_tyk-dashboard-source` used to download packages from the specified URLs, including GPG key verification and SSL settings, on a Linux system. + + Make sure to replace `el` and `8` in the config below with your Linux distribution and version: + ```bash + [tyk_tyk-dashboard] + name=tyk_tyk-dashboard + baseurl=https://packagecloud.io/tyk/tyk-dashboard/el/8/$basearch + repo_gpgcheck=1 + gpgcheck=0 + enabled=1 + gpgkey=https://packagecloud.io/tyk/tyk-dashboard/gpgkey + sslverify=1 + sslcacert=/etc/pki/tls/certs/ca-bundle.crt + metadata_expire=300 + + [tyk_tyk-dashboard-source] + name=tyk_tyk-dashboard-source + baseurl=https://packagecloud.io/tyk/tyk-dashboard/el/8/SRPMS + repo_gpgcheck=1 + gpgcheck=0 + enabled=1 + gpgkey=https://packagecloud.io/tyk/tyk-dashboard/gpgkey + sslverify=1 + sslcacert=/etc/pki/tls/certs/ca-bundle.crt + metadata_expire=300 + ``` + + We'll need to update the YUM package manager's local cache, enabling only the `tyk_tyk-dashboard` repository while disabling all other repositories `--disablerepo='*' --enablerepo='tyk_tyk-dashboard'`, and confirm all prompts `-y`. + ```bash + sudo yum -q makecache -y --disablerepo='*' --enablerepo='tyk_tyk-dashboard' + ``` + + Install Tyk dashboard: + ```bash + sudo yum install -y tyk-dashboard + ``` + +3. **Confirm Redis and MongoDB or PostgreSQL are running** + + Start Redis since it is always required by the Dashboard. + ```bash + sudo service redis start + ``` + Then start either MongoDB or PostgreSQL depending on which one you are using. + ```bash + sudo systemctl start mongod + ``` + ```bash + sudo systemctl start postgresql-13 + ``` + +4. **Configure Tyk Dashboard** + +We can set the Dashboard up with a similar setup command, the script below will get the Dashboard set up for the local instance. +Make sure to use the actual DNS hostname or the public IP of your instance as the last parameter. + +<Tabs> +<Tab title="MongoDB"> + +```bash +sudo /opt/tyk-dashboard/install/setup.sh --listenport=3000 --redishost=<Redis Hostname> --redisport=6379 --mongo=mongodb://<Mongo IP Address>:<Mongo Port>/tyk_analytics --tyk_api_hostname=$HOSTNAME --tyk_node_hostname=http://localhost --tyk_node_port=8080 --portal_root=/portal --domain="XXX.XXX.XXX.XXX" +``` + +Replace `<Redis Hostname>`, `<Mongo IP Address>` and `<Mongo Port>` with your own values to run this script. + +</Tab> +<Tab title="SQL"> + +```bash +sudo /opt/tyk-dashboard/install/setup.sh --listenport=3000 --redishost=<Redis Hostname> --redisport=6379 --storage=postgres --connection_string=postgresql://<User>:<Password>@<PostgreSQL Hostname>:<PostgreSQL Port>/<PostgreSQL DB> --tyk_api_hostname=$HOSTNAME --tyk_node_hostname=http://localhost --tyk_node_port=8080 --portal_root=/portal --domain="XXX.XXX.XXX.XXX" +``` expandable + +Replace `<Redis Hostname>`,`<PostgreSQL Hostname>`,`<PostgreSQL Port>`, `<PostgreSQL User>`, `<PostgreSQL Password>` and `<PostgreSQL DB>` with your own values to run the script. + +</Tab> +</Tabs> + +With these values your are configuring the following: + +* `--listenport=3000`: Tyk Dashboard (and Portal) to listen on port `3000`. +* `--redishost=<hostname>`: Tyk Dashboard should use the local Redis instance. +* `--redisport=6379`: The Tyk Dashboard should use the default port. +* `--domain="XXX.XXX.XXX.XXX"`: Bind the Dashboard to the IP or DNS hostname of this instance (required). +* `--mongo=mongodb://<Mongo IP Address>:<Mongo Port>/tyk_analytics`: Use the local MongoDB (should always be the same as the Gateway). +* `--storage=postgres`: In case, your preferred storage Database is PostgreSQL, use storage type "postgres" and specify connection string. +* `--connection_string=postgresql://<User>:<Password>@<PostgreSQL Host Name>:<PostgreSQL Port>/<PostgreSQL DB>`: Use the PostgreSQL instance provided in the connection string (should always be the same as the gateway). +* `--tyk_api_hostname=$HOSTNAME`: The Tyk Dashboard has no idea what hostname has been given to Tyk, so we need to tell it, in this instance we are just using the local HOSTNAME env variable, but you could set this to the public-hostname/IP of the instance. +* `--tyk_node_hostname=http://localhost`: The Tyk Dashboard needs to see a Tyk node in order to create new tokens, so we need to tell it where we can find one, in this case, use the one installed locally. +* `--tyk_node_port=8080`: Tell the Dashboard that the Tyk node it should communicate with is on port 8080. +* `--portal_root=/portal`: We want the Portal to be shown on /portal of whichever domain we set for the Portal. + +5. **Start Tyk Dashboard** + + ```bash + sudo service tyk-dashboard start + ``` + + + <Note> + To check the logs from the deployment run: + ```bash + sudo journalctl -u tyk-dashboard + ``` + </Note> + + + Notice how we haven't actually started the gateway yet, because this is a Dashboard install, we need to enter a license first. + + +<Note> +When using PostgreSQL you may receive the error: `"failed SASL auth (FATAL: password authentication failed for user...)"`, follow these steps to address the issue: +1. Open the terminal or command prompt on your PostgreSQL server. +2. Navigate to the location of the `pg_hba.conf` file. This file is typically located at `/var/lib/pgsql/13/data/pg_hba.conf`. +3. Open the `pg_hba.conf` file using a text manipulation tool. +4. In the `pg_hba.conf` file, locate the entry corresponding to the user encountering the authentication error. This entry might resemble the following: +```bash +host all all <IP_address>/<netmask> scram-sha-256 +``` +5. In the entry, find the METHOD column. It currently has the value scram-sha-256. +6. Replace scram-sha-256 with md5, so the modified entry looks like this: +```bash +host all all <IP_address>/<netmask> md5 +``` +7. Save the changes you made to the `pg_hba.conf` file. +8. Restart the PostgreSQL service to apply the modifications: +```bash +sudo systemctl restart postgresql-13 +``` expandable +</Note> + + +6. **Enter Dashboard license** + + Add your license in `/var/opt/tyk-dashboard/tyk_analytics.conf` in the `license` field. + + If all is going well, you will be taken to a Dashboard setup screen - we'll get to that soon. + +7. **Restart the Dashboard process** + + Because we've just entered a license via the UI, we need to make sure that these changes get picked up, so to make sure things run smoothly, we restart the Dashboard process (you only need to do this once) and (if you have it installed) then start the gateway: + ```bash + sudo service tyk-dashboard restart + ``` + +8. **Go to the Tyk Dashboard URL** + + Go to the following URL to access to the Tyk Dashboard: + + ```bash + 127.0.0.1:3000 + ``` + + You should get to the Tyk Dashboard Setup screen: + + <img src="/img/dashboard/system-management/bootstrap_screen.png" alt="Tyk Dashboard Bootstrap Screen" /> + +9. **Create your Organization and Default User** + + You need to enter the following: + + * Your **Organization Name** + * Your **Organization Slug** + * Your User **Email Address** + * Your User **First and Last Name** + * A **Password** for your User + * **Re-enter** your user **Password** + + + + + <Note> + For a password, we recommend a combination of alphanumeric characters, with both upper and lower case letters. + </Note> + + + + Click **Bootstrap** to save the details. + +10. **Login to the Dashboard** + + You can now log in to the Tyk Dashboard from `127.0.0.1:3000`, using the username and password created in the Dashboard Setup screen. + + **Configure your Developer Portal** + + To set up your [Developer Portal](/tyk-developer-portal) follow our Self-Managed [tutorial on publishing an API to the Portal Catalog](/getting-started/tutorials/publish-api). + +#### Using Ansible + +**Getting Started** + +1. clone the [tyk-ansible](https://github.com/TykTechnologies/tyk-ansible) repository + +```bash +$ git clone https://github.com/TykTechnologies/tyk-ansible +``` + +2. `cd` into the directory +```.bash +$ cd tyk-ansible +``` + +3. Run initialisation script to initialise environment + +```bash +$ sh scripts/init.sh +``` + +4. Modify `hosts.yml` file to update ssh variables to your server(s). You can learn more about the hosts file [here](https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html) + +5. Run ansible-playbook to install `tyk-dashboard` + +```bash +$ ansible-playbook playbook.yaml -t tyk-dashboard +``` expandable + +**Supported Distributions** + +| Distribution | Version | Supported | +| --------- | :---------: | :---------: | +| Amazon Linux | 2 | βœ… | +| CentOS | 8 | βœ… | +| CentOS | 7 | βœ… | +| RHEL | 8 | βœ… | +| RHEL | 7 | βœ… | + +**Variables** + +- `vars/tyk.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| secrets.APISecret | `352d20ee67be67f6340b4c0605b044b7` | API secret | +| secrets.AdminSecret | `12345` | Admin secret | +| dash.license | | Dashboard license| +| dash.service.host | | Dashboard server host if different than the hosts url | +| dash.service.port | `3000` | Dashboard server listening port | +| dash.service.proto | `http` | Dashboard server protocol | +| dash.service.tls | `false` | Set to `true` to enable SSL connections | + + + +### Install Pump + +#### Using Shell + +Tyk has it's own signed RPMs in a YUM repository hosted by the kind folks at [packagecloud.io](https://packagecloud.io), which makes it easy, safe and secure to install a trusted distribution of the Tyk Gateway stack. + +This tutorial will run on an [Amazon AWS](http://aws.amazon.com) *Red Hat Enterprise Linux 7.1* instance. We will install Tyk Pump with all dependencies stored locally. + +We're installing on a `t2.micro` because this is a tutorial, you'll need more RAM and more cores for better performance. + +This configuration should also work (with some tweaks) for CentOS. + +**Prerequisites** + +We are assuming that Redis and either MongoDB or SQL are installed (these are installed as part of the Tyk Gateway and Dashboard installation guides) + +**Step 1: Set up YUM Repositories** + +First, we need to install some software that allows us to use signed packages: +```bash +sudo yum install pygpgme yum-utils wget +``` + +Next, we need to set up the various repository configurations for Tyk and MongoDB: + +Create a file named `/etc/yum.repos.d/tyk_tyk-pump.repo` that contains the repository configuration below: + +Make sure to replace `el` and `7` in the config below with your Linux distribution and version: +```bash +[tyk_tyk-pump] +name=tyk_tyk-pump +baseurl=https://packagecloud.io/tyk/tyk-pump/el/7/$basearch +repo_gpgcheck=1 +gpgcheck=1 +enabled=1 +gpgkey=https://keyserver.tyk.io/tyk.io.rpm.signing.key.2020 + https://packagecloud.io/tyk/tyk-pump/gpgkey +sslverify=1 +sslcacert=/etc/pki/tls/certs/ca-bundle.crt +metadata_expire=300 +``` + +Finally we'll need to update our local cache, so run: +```bash +sudo yum -q makecache -y --disablerepo='*' --enablerepo='tyk_tyk-pump' +``` + +**Step 2: Install Packages** + +We're ready to go, you can now install the relevant packages using yum: +```bash +sudo yum install -y tyk-pump +``` expandable + +**(You may be asked to accept the GPG key for our repos and when the package installs, hit yes to continue.)** +<br /> + +**Step 3: Configure Tyk Pump** + +If you don't complete this step, you won't see any analytics in your Dashboard, so to enable the analytics service, we need to ensure Tyk Pump is running and configured properly. + + +**Configure Tyk Pump for MongoDB** +<br /> + +<Note> +You need to replace `<hostname>` for `--redishost=<hostname>`, and `<Mongo IP Address>`, `<Mongo Port>` for `--mongo=mongodb://<Mongo IP Address>:<Mongo Port>/` with your own values to run this script. +</Note> + + +```bash +sudo /opt/tyk-pump/install/setup.sh --redishost=<hostname> --redisport=6379 --mongo=mongodb://<IP Address>:<Mongo Port>/tyk_analytics +``` +**Configure Tyk Pump for SQL** +<br /> + +<Note> +You need to replace `<hostname>` for `--redishost=<hostname>`, and `<Postgres Host Name>`,`<Port>`, `<User>`, `<Password>`, `<DB>` for `--postgres="host=<Postgres Host Name> port=<Port> user=<User> password=<Password> dbname=<DB>"` with your own values to run this script. +</Note> + +```bash +sudo /opt/tyk-pump/install/setup.sh --redishost=<hostname> --redisport=6379 --postgres="host=<Postgres Host Name> port=<Port> user=<User> password=<Password> dbname=<DB>" +``` + + +**Step 4: Start Tyk Pump** + +```bash +sudo service tyk-pump start +``` + +That's it, the Pump should now be up and running. + +You can verify if Tyk Pump is running and working by accessing the logs: +```bash +sudo journalctl -u tyk-pump +``` + + + +#### Using Ansible + +**Getting Started** + +1. clone the [tyk-ansible](https://github.com/TykTechnologies/tyk-ansible) repositry + +```bash +$ git clone https://github.com/TykTechnologies/tyk-ansible +``` + +2. `cd` into the directory +```.bash +$ cd tyk-ansible +``` + +3. Run initialisation script to initialise environment + +```bash +$ sh scripts/init.sh +``` + +4. Modify `hosts.yml` file to update ssh variables to your server(s). You can learn more about the hosts file [here](https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html) + +5. Run ansible-playbook to install `tyk-pump` + +```bash +$ ansible-playbook playbook.yaml -t tyk-pump +``` expandable + +**Supported Distributions** + +| Distribution | Version | Supported | +| --------- | :---------: | :---------: | +| Amazon Linux | 2 | βœ… | +| CentOS | 8 | βœ… | +| CentOS | 7 | βœ… | +| RHEL | 8 | βœ… | +| RHEL | 7 | βœ… | + + + +### Install Gateway + +#### Using Shell + +Tyk has it's own signed RPMs in a YUM repository hosted by the kind folks at [packagecloud.io](https://packagecloud.io/tyk/tyk-dashboard/install#manual-rpm), which makes it easy, safe and secure to install a trusted distribution of the Tyk Gateway stack. + +This tutorial will run on an [Amazon AWS](http://aws.amazon.com) *Red Hat Enterprise Linux 7.1* instance. We will install Tyk Gateway with all dependencies stored locally. + +We're installing on a `t2.micro` because this is a tutorial, you'll need more RAM and more cores for better performance. + +This configuration should also work (with some tweaks) for CentOS. + +**Prerequisites** + +* Ensure port `8080` is open: this is used in this guide for Gateway traffic (API traffic to be proxied) +* EPEL (Extra Packages for Enterprise Linux) is a free, community based repository project from Fedora which provides high quality add-on software packages for Linux distribution including RHEL, CentOS, and Scientific Linux. EPEL isn’t a part of RHEL/CentOS but it is designed for major Linux distributions. In our case we need it for Redis. Install EPEL using the instructions here. + +**Step 1: Set up YUM Repositories** + +First, we need to install some software that allows us to use signed packages: +```bash +sudo yum install pygpgme yum-utils wget +``` + +Next, we need to set up the various repository configurations for Tyk and MongoDB: + +**Step 2: Create Tyk Gateway Repository Configuration** + +Create a file named `/etc/yum.repos.d/tyk_tyk-gateway.repo` that contains the repository configuration below https://packagecloud.io/tyk/tyk-gateway/install#manual-rpm: +```bash +[tyk_tyk-gateway] +name=tyk_tyk-gateway +baseurl=https://packagecloud.io/tyk/tyk-gateway/el/7/$basearch +repo_gpgcheck=1 +gpgcheck=1 +enabled=1 +gpgkey=https://keyserver.tyk.io/tyk.io.rpm.signing.key.2020 + https://packagecloud.io/tyk/tyk-gateway/gpgkey +sslverify=1 +sslcacert=/etc/pki/tls/certs/ca-bundle.crt +metadata_expire=300 +``` + +**Step 3: Install Packages** + +We're ready to go, you can now install the relevant packages using yum: +```bash +sudo yum install -y redis tyk-gateway +``` + +*(you may be asked to accept the GPG key for our two repos and when the package installs, hit yes to continue)* + +**Step 4: Start Redis** + +In many cases Redis will not be running, so let's start those: +```bash +sudo service redis start +``` expandable + +When Tyk is finished installing, it will have installed some init scripts, but it will not be running yet. The next step will be to setup the Gateway – thankfully this can be done with three very simple commands. + + + +#### Using Ansible + +**Requirements** + +[Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) - required for running the commands below. Use the **Shell** tab for instructions to install Tyk from a shell. + +**Getting Started** + +1. clone the [tyk-ansible](https://github.com/TykTechnologies/tyk-ansible) repositry + +```bash +$ git clone https://github.com/TykTechnologies/tyk-ansible +``` + +2. `cd` into the directory +```.bash +$ cd tyk-ansible +``` + +3. Run initialisation script to initialise environment + +```bash +$ sh scripts/init.sh +``` + +4. Modify `hosts.yml` file to update ssh variables to your server(s). You can learn more about the hosts file [here](https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html) + +5. Run ansible-playbook to install `tyk-gateway` + +```bash +$ ansible-playbook playbook.yaml -t `tyk-gateway-pro` or `tyk-gateway-hybrid` +``` expandable + +**Supported Distributions** + +| Distribution | Version | Supported | +| --------- | :---------: | :---------: | +| Amazon Linux | 2 | βœ… | +| CentOS | 8 | βœ… | +| CentOS | 7 | βœ… | +| RHEL | 8 | βœ… | +| RHEL | 7 | βœ… | + +**Variables** + +- `vars/tyk.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| secrets.APISecret | `352d20ee67be67f6340b4c0605b044b7` | API secret | +| secrets.AdminSecret | `12345` | Admin secret | +| gateway.service.host | | Gateway server host if different than the hosts url | +| gateway.service.port | `8080` | Gateway server listening port | +| gateway.service.proto | `http` | Gateway server protocol | +| gateway.service.tls | `false` | Set to `true` to enable SSL connections | +| gateway.sharding.enabled | `false` | Set to `true` to enable filtering (sharding) of APIs | +| gateway.sharding.tags | | The tags to use when filtering (sharding) Tyk Gateway nodes. Tags are processed as OR operations. If you include a non-filter tag (e.g. an identifier such as `node-id-1`, this will become available to your Dashboard analytics) | +| gateway.rpc.connString | | Use this setting to add the URL for your MDCB or load balancer host | +| gateway.rpc.useSSL | `true` | Set this option to `true` to use an SSL RPC connection| +| gateway.rpc.sslInsecureSkipVerify | `true` | Set this option to `true` to allow the certificate validation (certificate chain and hostname) to be skipped. This can be useful if you use a self-signed certificate | +| gateway.rpc.rpcKey | | Your organization ID to connect to the MDCB installation | +| gateway.rpc.apiKey | | This the API key of a user used to authenticate and authorize the Gateway’s access through MDCB. The user should be a standard Dashboard user with minimal privileges so as to reduce any risk if the user is compromised. The suggested security settings are read for Real-time notifications and the remaining options set to deny | +| gateway.rpc.groupId | | This is the `zone` that this instance inhabits, e.g. the cluster/data-center the Gateway lives in. The group ID must be the same across all the Gateways of a data-center/cluster which are also sharing the same Redis instance. This ID should also be unique per cluster (otherwise another Gateway cluster can pick up your keyspace events and your cluster will get zero updates). | +##### Configure Tyk Gateway with the Dashboard + +**Prerequisites** + +This configuration assumes that you have already installed Tyk Dashboard, and have decided on the domain names for your Dashboard and your Portal. **They must be different**. For testing purposes, it is easiest to add hosts entries to your (and your servers) `/etc/hosts` file. + +**Set up Tyk Gateway with Quick Start Script** + +You can set up the core settings for Tyk Gateway with a single setup script, however for more involved deployments, you will want to provide your own configuration file. + + +<Note> +You need to replace `<hostname>` for `--redishost=<hostname>`with your own value to run this script. +</Note> + + +```bash +sudo /opt/tyk-gateway/install/setup.sh --dashboard=1 --listenport=8080 --redishost=<hostname> --redisport=6379 +``` expandable + +What we've done here is told the setup script that: + +* `--dashboard=1`: We want to use the Dashboard, since Tyk Gateway gets all it's API Definitions from the Dashboard service, as of v2.3 Tyk will auto-detect the location of the dashboard, we only need to specify that we should use this mode. +* `--listenport=8080`: Tyk should listen on port 8080 for API traffic. +* `--redishost=<hostname>`: Use Redis on the hostname: localhost. +* `--redisport=6379`: Use the default Redis port. + +**Starting Tyk** + +The Tyk Gateway can be started now that it is configured. Use this command to start the Tyk Gateway: +```bash +sudo service tyk-gateway start +``` expandable + +**Pro Tip: Domains with Tyk Gateway** + +Tyk Gateway has full domain support built-in, you can: + +* Set Tyk to listen only on a specific domain for all API traffic. +* Set an API to listen on a specific domain (e.g. api1.com, api2.com). +* Split APIs over a domain using a path (e.g. api.com/api1, api.com/api2, moreapis.com/api1, moreapis.com/api2 etc). +* If you have set a hostname for the Gateway, then all non-domain-bound APIs will be on this hostname + the `listen_path`. + +## Install Tyk on Debian or Ubuntu + +### Install Database + +#### Using Shell + +**Requirements** + +Before installing the Tyk components in the order below, you need to first install Redis and MongoDB/SQL. + +**Getting Started** + +<Tabs> +<Tab title="MongoDB"> +**Install MongoDB 4.0** + +You should follow the [online tutorial for installing MongoDb](https://docs.mongodb.com/v4.0/tutorial/install-mongodb-on-ubuntu/). We will be using version 4.0. As part of the Mongo installation you need to perform the following: + +1. Import the public key +2. Create a list file +3. Reload the package database +4. Install the MongoDB packages +5. Start MongoDB +6. Check the `mongod` service is running + +</Tab> +<Tab title="SQL"> + +**Install SQL** + +You should follow the [online tutorial for installing PostgreSQL](https://www.postgresql.org/download/linux/ubuntu/). We will be using version 13. As part of the PostgreSQL installation you need to perform the following: + +1. Create the file repository configuration +2. Import the repository signing key +3. Update the package lists +4. Install the PostgreSQL packages +5. Start PostgreSQL +6. Check the `postgresql` service is running + +See [SQL configuration](/planning-for-production/database-settings#postgresql) for details on installing SQL in a production environment. +</Tab> +</Tabs> + +**Install Redis** + +```console +$ sudo apt-get install -y redis-server +``` expandable + +**Install Tyk Pro on Ubuntu** + +Installing Tyk on Ubuntu is very straightforward using our APT repositories, follow the guides and tutorials in this section to have Tyk up and running in no time. + +The suggested order would be to install Tyk Dashboard, then Tyk Pump and then Tyk Gateway for a full stack. + +- [Dashboard](/tyk-self-managed/install#Debian-Ubuntu-install-dashboard) +- [Pump](/tyk-self-managed/install#Debian-Ubuntu-install-pump) +- [Gateway](/tyk-self-managed/install#debian-ubuntu-install-gateway) + + + + <Note> + For a production environment, we recommend that the Gateway, Dashboard and Pump are installed on separate machines. If installing multiple Gateways, you should install each on a separate machine. See [Planning for Production](/planning-for-production) For more details. + </Note> + + + +#### Using Ansible + +**Requirements** + + +[Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) - required for running the commands below. Use the **Shell** tab for instructions to install Tyk from a shell. + +**Getting Started** + +1. clone the [tyk-ansible](https://github.com/TykTechnologies/tyk-ansible) repositry + +```console +$ git clone https://github.com/TykTechnologies/tyk-ansible +``` + +2. `cd` into the directory +```console +$ cd tyk-ansible +``` + +3. Run initialisation script to initialise environment + +```console +$ sh scripts/init.sh +``` expandable + +4. Modify `hosts.yml` file to update ssh variables to your server(s). You can learn more about the hosts file [here](https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html) + +5. Run ansible-playbook to install the following: +- Redis +- MongoDB or PostgreSQL +- Tyk Dashboard +- Tyk Gateway +- Tyk Pump + +```console +$ ansible-playbook playbook.yaml -t tyk-pro -t redis -t `mongodb` or `pgsql` +``` expandable + +You can choose to not install Redis, MongoDB or PostgreSQL by removing the `-t redis` or `-t mongodb` or `-t pgsql` However Redis and MongoDB or PostgreSQL are a requirement and need to be installed for the Tyk Pro installation to run. + +**Supported Distributions** + +| Distribution | Version | Supported | +| --------- | :---------: | :---------: | +| Debian | 10 | βœ… | +| Debian | 9 | βœ… | +| Ubuntu | 21 | βœ… | +| Ubuntu | 20 | βœ… | +| Ubuntu | 18 | βœ… | +| Ubuntu | 16 | βœ… | + +**Variables** + +- `vars/tyk.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| secrets.APISecret | `352d20ee67be67f6340b4c0605b044b7` | API secret | +| secrets.AdminSecret | `12345` | Admin secret | +| redis.host | | Redis server host if different than the hosts url | +| redis.port | `6379` | Redis server listening port | +| redis.pass | | Redis server password | +| redis.enableCluster | `false` | Enable if redis is running in cluster mode | +| redis.storage.database | `0` | Redis server database | +| redis.tls | `false` | Enable if redis connection is secured with SSL | +| mongo.host | | MongoDB server host if different than the hosts url | +| mongo.port | `27017` | MongoDB server listening port | +| mongo.tls | `false` | Enable if mongo connection is secured with SSL | +| pgsql.host | | PGSQL server host if different than the hosts url | +| pgsql.port | `5432` | PGSQL server listening port | +| pgsql.tls | `false` | Enable if pgsql connection is secured with SSL | +| dash.license | | Dashboard license| +| dash.service.host | | Dashboard server host if different than the hosts url | +| dash.service.port | `3000` | Dashboard server listening port | +| dash.service.proto | `http` | Dashboard server protocol | +| dash.service.tls | `false` | Set to `true` to enable SSL connections | +| gateway.service.host | | Gateway server host if different than the hosts url | +| gateway.service.port | `8080` | Gateway server listening port | +| gateway.service.proto | `http` | Gateway server protocol | +| gateway.service.tls | `false` | Set to `true` to enable SSL connections | +| gateway.sharding.enabled | `false` | Set to `true` to enable filtering (sharding) of APIs | +| gateway.sharding.tags | | The tags to use when filtering (sharding) Tyk Gateway nodes. Tags are processed as OR operations. If you include a non-filter tag (e.g. an identifier such as `node-id-1`, this will become available to your Dashboard analytics) | +| gateway.rpc.connString | | Use this setting to add the URL for your MDCB or load balancer host | +| gateway.rpc.useSSL | `true` | Set this option to `true` to use an SSL RPC connection| +| gateway.rpc.sslInsecureSkipVerify | `true` | Set this option to `true` to allow the certificate validation (certificate chain and hostname) to be skipped. This can be useful if you use a self-signed certificate | +| gateway.rpc.rpcKey | | Your organization ID to connect to the MDCB installation | +| gateway.rpc.apiKey | | This the API key of a user used to authenticate and authorize the Gateway’s access through MDCB. The user should be a standard Dashboard user with minimal privileges so as to reduce any risk if the user is compromised. The suggested security settings are read for Real-time notifications and the remaining options set to deny | +| gateway.rpc.groupId | | This is the `zone` that this instance inhabits, e.g. the cluster/data-center the Gateway lives in. The group ID must be the same across all the Gateways of a data-center/cluster which are also sharing the same Redis instance. This ID should also be unique per cluster (otherwise another Gateway cluster can pick up your keyspace events and your cluster will get zero updates). | + +- `vars/redis.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| redis_bind_interface | `0.0.0.0` | Binding address of Redis | + +Read more about Redis configuration [here](https://github.com/geerlingguy/ansible-role-redis). + +- `vars/mongodb.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| bind_ip | `0.0.0.0` | Binding address of MongoDB | +| mongodb_version | `4.4` | MongoDB version | + +Read more about MongoDB configuration [here](https://github.com/ansible-collections/community.mongodb). + +- `vars/pgsql.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| postgresql_databases[] | `[]` | Array of DBs to be created | +| postgresql_databases[].name | `tyk_analytics` | Database name | +| postgresql_users[] | `[]` | Array of users to be created | +| postgresql_users[`0`].name | `default` | User name | +| postgresql_users[`0`].password | `topsecretpassword` | User password | +| postgresql_global_config_options[] | `[]` | Postgres service config options | +| postgresql_global_config_options[`1`].option | `listen_addresses` | Listen address binding for the service | +| postgresql_global_config_options[`1`].value | `*` | Default value to listen to all addresses | +| postgresql_hba_entries[] | `[]` | Host based authenticaiton list| +| postgresql_hba_entries[`4`].type | `host` | Entry type | +| postgresql_hba_entries[`4`].database | `tyk_analytics` | Which database this entry will give access to | +| postgresql_hba_entries[`4`].user | `default` | What users this gain access from this entry | +| postgresql_hba_entries[`4`].address | `0.0.0.0/0` | What addresses this gain access from this entry | +| postgresql_hba_entries[`4`].auth_method | `md5` | What authentication method to to use for the users | + +Read more about PostgreSQL configuration [here](https://github.com/geerlingguy/ansible-role-postgresql). + +<h3 id="Debian-Ubuntu-install-dashboard">Install Dashboard</h3> +#### Using Shell + +Tyk has its own APT repositories hosted by the kind folks at [packagecloud.io](https://packagecloud.io/tyk), which makes it easy, safe and secure to install a trusted distribution of the Tyk Gateway stack. + +This tutorial has been tested on Ubuntu 16.04 & 18.04 with few if any modifications. We will install the Tyk Dashboard with all dependencies locally. + +**Prerequisites** +- Have MongoDB/SQL and Redis installed - follow the guide for [installing databases on Debian/Ubuntu](#install-tyk-on-debian-or-ubuntu). +- Ensure port `3000` is available. This is used by the Tyk Dashboard to provide the GUI and the Developer Portal. + +**Step 1: Set up our APT Repositories** + +First, add our GPG key which signs our binaries: + +```bash +curl -L https://packagecloud.io/tyk/tyk-dashboard/gpgkey | sudo apt-key add - +``` + +Run update: + +```bash +sudo apt-get update +``` + +Since our repositories are installed via HTTPS, you will need to make sure APT supports this: + +```bash +sudo apt-get install -y apt-transport-https +``` + +Now lets add the required repos and update again (notice the `-a` flag in the second Tyk commands - this is important!): + +```bash +echo "deb https://packagecloud.io/tyk/tyk-dashboard/ubuntu/ bionic main" | sudo tee /etc/apt/sources.list.d/tyk_tyk-dashboard.list + +echo "deb-src https://packagecloud.io/tyk/tyk-dashboard/ubuntu/ bionic main" | sudo tee -a /etc/apt/sources.list.d/tyk_tyk-dashboard.list + +sudo apt-get update +``` expandable + + +<Note> +`bionic` is the code name for Ubuntu 18.04. Please substitute it with your particular [ubuntu release](https://releases.ubuntu.com/), e.g. `focal`. +</Note> + + +**What we've done here is:** + +- Added the Tyk Dashboard repository +- Updated our package list + +**Step 2: Install the Tyk Dashboard** + +We're now ready to install the Tyk Dashboard. To install run: + +```bash +sudo apt-get install -y tyk-dashboard +``` + +What we've done here is instructed `apt-get` to install the Tyk Dashboard without prompting. Wait for the downloads to complete. + +When the Tyk Dashboard has finished installing, it will have installed some `init` scripts, but it will not be running yet. The next step will be to setup each application - thankfully this can be done with three very simple commands. + +**Verify the origin key (optional)** + +Debian packages are signed with the repository keys. These keys are verified at the time of fetching the package and is taken care of by the `apt` infrastructure. These keys are controlled by PackageCloud, our repository provider. For an additional guarantee, it is possible to verify that the package was indeed created by Tyk by verifying the `origin` certificate that is attached to the package. + +First, you have to fetch Tyk's signing key and import it. + +```bash +wget https://keyserver.tyk.io/tyk.io.deb.signing.key +gpg --import tyk.io.deb.signing.key +``` + +Then, you have to either, +- sign the key with your ultimately trusted key +- trust this key ultimately + +The downloaded package will be available in `/var/cache/apt/archives`. Assuming you found the file `tyk-gateway-2.9.4_amd64.deb` there, you can verify the origin signature. + +```bash +gpg --verify d.deb +gpg: Signature made Wed 04 Mar 2020 03:05:00 IST +gpg: using RSA key F3781522A858A2C43D3BC997CA041CD1466FA2F8 +gpg: Good signature from "Team Tyk (package signing) <team@tyk.io>" [ultimate] +``` expandable + +##### **Configure Tyk Dashboard** + +**Prerequisites for MongoDB** + +You need to ensure the MongoDB and Redis services are running before proceeding. + + +<Note> +You need to replace `<hostname>` for `--redishost=<hostname>`, and `<IP Address>` for `--mongo=mongodb://<IP Address>/` with your own values to run this script. +</Note> + + + +You can set your Tyk Dashboard up with a helper setup command script. This will get the Dashboard set up for the local instance: + +```bash +sudo /opt/tyk-dashboard/install/setup.sh --listenport=3000 --redishost=<hostname> --redisport=6379 --mongo=mongodb://<IP Address>/tyk_analytics --tyk_api_hostname=$HOSTNAME --tyk_node_hostname=http://localhost --tyk_node_port=8080 --portal_root=/portal --domain="XXX.XXX.XXX.XXX" +``` expandable + + +<Note> +Make sure to use the actual DNS hostname or the public IP of your instance as the last parameter. +</Note> + + + +What we have done here is: + +- `--listenport=3000`: Told the Tyk Dashboard (and Portal) to listen on port 3000. +- `--redishost=<hostname>`: The Tyk Dashboard should use the local Redis instance. +- `--redisport=6379`: The Tyk Dashboard should use the default port. +- `--domain="XXX.XXX.XXX.XXX"`: Bind the Tyk Dashboard to the IP or DNS hostname of this instance (required). +- `--mongo=mongodb://<IP Address>/tyk_analytics`: Use the local MongoDB (should always be the same as the gateway). +- `--tyk_api_hostname=$HOSTNAME`: The Tyk Dashboard has no idea what hostname has been given to Tyk, so we need to tell it, in this instance we are just using the local HOSTNAME env variable, but you could set this to the public-hostname/IP of the instance. +- `--tyk_node_hostname=http://localhost`: The Tyk Dashboard needs to see a Tyk node in order to create new tokens, so we need to tell it where we can find one, in this case, use the one installed locally. +- `--tyk_node_port=8080`: Tell the Tyk Dashboard that the Tyk node it should communicate with is on port 8080. +- `--portal_root=/portal`: We want the portal to be shown on `/portal` of whichever domain we set for the portal. + +**Prerequisites for SQL** + +You need to ensure the PostgreSQL and Redis services are running before proceeding. + + +<Note> +You need to replace `<hostname>` for `--redishost=<hostname>`, and `<Postgres Host Name>`, `<Port>`, `<User>`, `<Password>`, `<DB>` for `--connection_string="host=<Postgres Host Name> port=<Port> user=<User> password=<Password> dbname=<DB>"` with your own values to run this script. +</Note> + + + +You can set the Tyk Dashboard up with a helper setup command script. This will get the Dashboard set up for the local instance: + +```bash +sudo /opt/tyk-dashboard/install/setup.sh --listenport=3000 --redishost=<hostname> --redisport=6379 --storage=postgres --connection_string="host=<Postgres Host Name> port=<Port> user=<User> password=<Password> dbname=<DB>" --tyk_api_hostname=$HOSTNAME --tyk_node_hostname=http://localhost --tyk_node_port=8080 --portal_root=/portal --domain="XXX.XXX.XXX.XXX" +``` expandable + + +<Note> +Make sure to use the actual DNS hostname or the public IP of your instance as the last parameter. +</Note> + + + +What we have done here is: + +- `--listenport=3000`: Told the Tyk Dashboard (and Portal) to listen on port 3000. +- `--redishost=<hostname>`: The Tyk Dashboard should use the local Redis instance. +- `--redisport=6379`: The Tyk Dashboard should use the default port. +- `--domain="XXX.XXX.XXX.XXX"`: Bind the dashboard to the IP or DNS hostname of this instance (required). +- `--storage=postgres`: Use storage type postgres. +- `--connection_string="host=<Postgres Host Name> port=<Port> user=<User> password=<Password> dbname=<DB>"`: Use the postgres instance provided in the connection string(should always be the same as the gateway). +- `--tyk_api_hostname=$HOSTNAME`: The Tyk Dashboard has no idea what hostname has been given to Tyk, so we need to tell it, in this instance we are just using the local HOSTNAME env variable, but you could set this to the public-hostname/IP of the instance. +- `--tyk_node_hostname=http://localhost`: The Tyk Dashboard needs to see a Tyk node in order to create new tokens, so we need to tell it where we can find one, in this case, use the one installed locally. +- `--tyk_node_port=8080`: Tell the dashboard that the Tyk node it should communicate with is on port 8080. +- `--portal_root=/portal`: We want the portal to be shown on `/portal` of whichever domain we set for the portal. + + +**Step 1: Enter your Tyk Dashboard License** + +Add your license in `/opt/tyk-dashboard/tyk_analytics.conf` in the `license` field. + +**Step 2: Start the Tyk Dashboard** + +Start the dashboard service, and ensure it will start automatically on system boot. + +```bash +sudo systemctl start tyk-dashboard +sudo systemctl enable tyk-dashboard +``` + +**Step 3: Install your Tyk Gateway** + +Follow the [Gateway installation instructions](#using-shell-7) to connect to your Dashboard instance before you continue on to step 4. + +**Step 4: Bootstrap the Tyk Dashboard with an initial User and Organization** + +Go to: + +```bash +127.0.0.1:3000 +``` expandable + +You should get to the Tyk Dashboard Setup screen: + +<img src="/img/dashboard/system-management/bootstrap_screen.png" alt="Tyk Dashboard Bootstrap Screen" /> + +**Step 5 - Create your Organization and Default User** + +You need to enter the following: + +- Your **Organization Name** +- Your **Organization Slug** +- Your User **Email Address** +- Your User **First and Last Name** +- A **Password** for your User +- **Re-enter** your user **Password** + + + + <Note> + For a password, we recommend a combination of alphanumeric characters, with both upper and lower case + letters. + </Note> + + +Click **Bootstrap** to save the details. + +**Step 6 - Login to the Tyk Dashboard** + +You can now log in to the Tyk Dashboard from `127.0.0.1:3000`, using the username and password created in the Dashboard Setup screen. + +##### **Configure your Developer Portal** + +To set up your [Developer Portal](/tyk-developer-portal) follow our Self-Managed [tutorial on publishing an API to the Portal Catalog](/getting-started/tutorials/publish-api). + +#### Using Ansible + +**Getting Started** + +1. clone the [tyk-ansible](https://github.com/TykTechnologies/tyk-ansible) repositry + +```bash +$ git clone https://github.com/TykTechnologies/tyk-ansible +``` + +2. `cd` into the directory +```.bash +$ cd tyk-ansible +``` + +3. Run initialisation script to initialise environment + +```bash +$ sh scripts/init.sh +``` + +4. Modify `hosts.yml` file to update ssh variables to your server(s). You can learn more about the hosts file [here](https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html) + +5. Run ansible-playbook to install `tyk-dashboard` + +```bash +$ ansible-playbook playbook.yaml -t tyk-dashboard +``` expandable + +**Supported Distributions** + +| Distribution | Version | Supported | +| --------- | :---------: | :---------: | +| Debian | 10 | βœ… | +| Debian | 9 | βœ… | +| Ubuntu | 21 | βœ… | +| Ubuntu | 20 | βœ… | +| Ubuntu | 18 | βœ… | +| Ubuntu | 16 | βœ… | + +**Variables** + +- `vars/tyk.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| secrets.APISecret | `352d20ee67be67f6340b4c0605b044b7` | API secret | +| secrets.AdminSecret | `12345` | Admin secret | +| dash.license | | Dashboard license| +| dash.service.host | | Dashboard server host if different than the hosts url | +| dash.service.port | `3000` | Dashboard server listening port | +| dash.service.proto | `http` | Dashboard server protocol | +| dash.service.tls | `false` | Set to `true` to enable SSL connections | + +<h3 id="Debian-Ubuntu-install-pump">Install Pump</h3> +#### Using Shell + +This tutorial has been tested Ubuntu 16.04 & 18.04 with few if any modifications. + +**Prerequisites** + +- You have installed Redis and either MongoDB or SQL. +- You have installed the Tyk Dashboard. + +**Step 1: Set up our APT repositories** + +First, add our GPG key which signs our binaries: + +```bash +curl -L https://packagecloud.io/tyk/tyk-pump/gpgkey | sudo apt-key add - +``` + +Run update: + +```bash +sudo apt-get update +``` + +Since our repositories are installed via HTTPS, you will need to make sure APT supports this: + +```bash +sudo apt-get install -y apt-transport-https +``` + +Now lets add the required repos and update again (notice the `-a` flag in the second Tyk commands - this is important!): + +```bash +echo "deb https://packagecloud.io/tyk/tyk-pump/ubuntu/ bionic main" | sudo tee /etc/apt/sources.list.d/tyk_tyk-pump.list + +echo "deb-src https://packagecloud.io/tyk/tyk-pump/ubuntu/ bionic main" | sudo tee -a /etc/apt/sources.list.d/tyk_tyk-pump.list + +sudo apt-get update +``` expandable + + +<Note> +`bionic` is the code name for Ubuntu 18.04. Please substitute it with your particular [ubuntu release](https://releases.ubuntu.com/), e.g. `focal`. +</Note> + + +**What you've done here is:** + +- Added the Tyk Pump repository +- Updated our package list + +**Step 2: Install the Tyk Pump** + +You're now ready to install the Tyk Pump. To install it, run: + +```bash +sudo apt-get install -y tyk-pump +``` + +What you've done here is instructed `apt-get` to install Tyk Pump without prompting. Wait for the downloads to complete. + +When Tyk Pump has finished installing, it will have installed some `init` scripts, but it will not be running yet. The next step will be to setup each application using three very simple commands. + +**Verify the origin key (optional)** + +Debian packages are signed with the repository keys. These keys are verified at the time of fetching the package and is taken care of by the `apt` infrastructure. These keys are controlled by PackageCloud, our repository provider. For an additional guarantee, it is possible to verify that the package was indeed created by Tyk by verifying the `origin` certificate that is attached to the package. + +First, you have to fetch Tyk's signing key and import it. + +```bash +wget https://keyserver.tyk.io/tyk.io.deb.signing.key +gpg --import tyk.io.deb.signing.key +``` + +Then, you have to either, +- sign the key with your ultimately trusted key +- trust this key ultimately + +The downloaded package will be available in `/var/cache/apt/archives`. Assuming you found the file `tyk-gateway-2.9.3_amd64.deb` there, you can verify the origin signature. + +```bash +gpg --verify d.deb +gpg: Signature made Wed 04 Mar 2020 03:05:00 IST +gpg: using RSA key F3781522A858A2C43D3BC997CA041CD1466FA2F8 +gpg: Good signature from "Team Tyk (package signing) <team@tyk.io>" [ultimate] +``` expandable + +**Step 3: Configure Tyk Pump** + +If you don't complete this step, you won't see any analytics in your Dashboard, so to enable the analytics service, we need to ensure Tyk Pump is running and configured properly. + +**Option 1: Configure Tyk Pump for MongoDB** +<br /> + +<Note> +You need to replace `<hostname>` for `--redishost=<hostname>`, and `<IP Address>` for `--mongo=mongodb://<IP Address>/` with your own values to run this script. +</Note> + + +```bash +sudo /opt/tyk-pump/install/setup.sh --redishost=<hostname> --redisport=6379 --mongo=mongodb://<IP Address>/tyk_analytics +``` + +**Option 2: Configure Tyk Pump for SQL** +<br /> + +<Note> +You need to replace `<hostname>` for `--redishost=<hostname>`, and `<Postgres Host Name>`,`<Port>`, `<User>`, `<Password>`, `<DB>` for `--postgres="host=<Postgres Host Name> port=<Port> user=<User> password=<Password> dbname=<DB>"` with your own values to run this script. +</Note> + + +```bash +sudo /opt/tyk-pump/install/setup.sh --redishost=<hostname> --redisport=6379 --postgres="host=<Postgres Host Name> port=<Port> user=<User> password=<Password> dbname=<DB>" +``` + +**Step 4: Start Tyk Pump** + +```bash +sudo service tyk-pump start +sudo service tyk-pump enable +``` + +You can verify if Tyk Pump is running and working by tailing the log file: + +```bash +sudo tail -f /var/log/upstart/tyk-pump.log +``` + +#### Using Ansible + +**Install Tyk Pump Through Ansible** + +**Getting Started** +1. clone the [tyk-ansible](https://github.com/TykTechnologies/tyk-ansible) repositry + +```bash +$ git clone https://github.com/TykTechnologies/tyk-ansible +``` + +2. `cd` into the directory +```.bash +$ cd tyk-ansible +``` + +3. Run initialisation script to initialise environment + +```bash +$ sh scripts/init.sh +``` + +4. Modify `hosts.yml` file to update ssh variables to your server(s). You can learn more about the hosts file [here](https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html) + +5. Run ansible-playbook to install `tyk-pump` + +```bash +$ ansible-playbook playbook.yaml -t tyk-pump +``` expandable + +**Supported Distributions** +| Distribution | Version | Supported | +| --------- | :---------: | :---------: | +| Debian | 10 | βœ… | +| Debian | 9 | βœ… | +| Ubuntu | 21 | βœ… | +| Ubuntu | 20 | βœ… | +| Ubuntu | 18 | βœ… | +| Ubuntu | 16 | βœ… | + +<h3 id="debian-ubuntu-install-gateway">Install Gateway</h3> +#### Using Shell + +Tyk has it's own APT repositories hosted by the kind folks at [packagecloud.io][1], which makes it easy, safe and secure to install a trusted distribution of the Tyk Gateway stack. + +This tutorial has been tested on Ubuntu 16.04 & 18.04 with few if any modifications. + +Please note however, that should you wish to write your own plugins in Python, we currently have a Python version dependency of 3.4. Python-3.4 ships with Ubuntu 14.04, however you may need to explicitly install it on newer Ubuntu Operating System releases. + +**Prerequisites** + +* Ensure port `8080` is available. This is used in this guide for Gateway traffic (API traffic to be proxied). +* You have MongoDB and Redis installed. +* You have installed firstly the Tyk Dashboard, then the Tyk Pump. + +**Step 1: Set up our APT Repositories** + +First, add our GPG key which signs our binaries: + +```bash +curl -L https://packagecloud.io/tyk/tyk-gateway/gpgkey | sudo apt-key add - +``` + +Run update: +```bash +sudo apt-get update +``` + +Since our repositories are installed via HTTPS, you will need to make sure APT supports this: +```bash +sudo apt-get install -y apt-transport-https +``` + +Create a file `/etc/apt/sources.list.d/tyk_tyk-gateway.list` with the following contents: +```bash +deb https://packagecloud.io/tyk/tyk-gateway/ubuntu/ bionic main +deb-src https://packagecloud.io/tyk/tyk-gateway/ubuntu/ bionic main +``` + +<Note> +`bionic` is the code name for Ubuntu 18.04. Please substitute it with your particular [ubuntu release](https://releases.ubuntu.com/), e.g. `focal`. +</Note> + + +Now you can refresh the list of packages with: +```bash +sudo apt-get update +``` + +**What we've done here is:** + +* Added the Tyk Gateway repository +* Updated our package list + +**Step 2: Install the Tyk Gateway** + +We're now ready to install the Tyk Gateway. To install it, run: + +```bash +sudo apt-get install -y tyk-gateway +``` +What we've done here is instructed apt-get to install the Tyk Gateway without prompting, wait for the downloads to complete. + +When Tyk has finished installing, it will have installed some init scripts, but will not be running yet. The next step will be to set up the Gateway - thankfully this can be done with three very simple commands, however it does depend on whether you are configuring Tyk Gateway for use with the Dashboard or without (the Community Edition). + +**Verify the origin key (optional)** + +Debian packages are signed with the repository keys. These keys are verified at the time of fetching the package and is taken care of by the `apt` infrastructure. These keys are controlled by PackageCloud, our repository provider. For an additional guarantee, it is possible to verify that the package was indeed created by Tyk by verifying the `origin` certificate that is attached to the package. + +First, you have to fetch Tyk's signing key and import it. + +```bash +wget https://keyserver.tyk.io/tyk.io.deb.signing.key +gpg --import tyk.io.deb.signing.key +``` + +Then, you have to either, +- sign the key with your ultimately trusted key +- trust this key ultimately + +The downloaded package will be available in `/var/cache/apt/archives`. Assuming you found the file `tyk-gateway-2.9.4_amd64.deb` there, you can verify the origin signature. + +```bash +gpg --verify d.deb +gpg: Signature made Wed 04 Mar 2020 03:05:00 IST +gpg: using RSA key F3781522A858A2C43D3BC997CA041CD1466FA2F8 +gpg: Good signature from "Team Tyk (package signing) <team@tyk.io>" [ultimate] +``` expandable + +**Configure Tyk Gateway with Dashboard** + +**Prerequisites** + +This configuration assumes that you have already installed the Tyk Dashboard, and have decided on the domain names for your Dashboard and your Portal. **They must be different**. For testing purposes, it is easiest to add hosts entries to your (and your servers) `/etc/hosts` file. + +**Set up Tyk** + +You can set up the core settings for Tyk Gateway with a single setup script, however for more involved deployments, you will want to provide your own configuration file. + + +<Note> +You need to replace `<hostname>` for `--redishost=<hostname>`with your own value to run this script. +</Note> + + + +```bash +sudo /opt/tyk-gateway/install/setup.sh --dashboard=1 --listenport=8080 --redishost=<hostname> --redisport=6379 +``` expandable + +What we've done here is told the setup script that: + +* `--dashboard=1`: We want to use the Dashboard, since Tyk Gateway gets all it's API Definitions from the Dashboard service, as of v2.3 Tyk will auto-detect the location of the dashboard, we only need to specify that we should use this mode. +* `--listenport=8080`: Tyk should listen on port 8080 for API traffic. +* `--redishost=<hostname>`: Use Redis on your hostname. +* `--redisport=6379`: Use the default Redis port. + +**Starting Tyk** + +The Tyk Gateway can be started now that it is configured. Use this command to start the Tyk Gateway: +```bash +sudo service tyk-gateway start +sudo service tyk-gateway enable +``` expandable + +**Pro Tip: Domains with Tyk Gateway** + +Tyk Gateway has full domain support built-in, you can: + +* Set Tyk to listen only on a specific domain for all API traffic. +* Set an API to listen on a specific domain (e.g. api1.com, api2.com). +* Split APIs over a domain using a path (e.g. api.com/api1, api.com/api2, moreapis.com/api1, moreapis.com/api2 etc). +* If you have set a hostname for the Gateway, then all non-domain-bound APIs will be on this hostname + the `listen_path`. + +[1]: https://packagecloud.io/tyk + + +#### Using Ansible + +**Getting Started** +1. clone the [tyk-ansible](https://github.com/TykTechnologies/tyk-ansible) repositry + +```bash +$ git clone https://github.com/TykTechnologies/tyk-ansible +``` + +2. `cd` into the directory +```.bash +$ cd tyk-ansible +``` + +3. Run initialisation script to initialise environment + +```bash +$ sh scripts/init.sh +``` + +4. Modify `hosts.yml` file to update ssh variables to your server(s). You can learn more about the hosts file [here](https://docs.ansible.com/ansible/latest/user_guide/intro_inventory.html) + +5. Run ansible-playbook to install `tyk-gateway` + +```bash +$ ansible-playbook playbook.yaml -t `tyk-gateway-pro` or `tyk-gateway-hybrid` +``` + +**Supported Distributions** +| Distribution | Version | Supported | +| --------- | :---------: | :---------: | +| Debian | 10 | βœ… | +| Debian | 9 | βœ… | +| Ubuntu | 21 | βœ… | +| Ubuntu | 20 | βœ… | +| Ubuntu | 18 | βœ… | +| Ubuntu | 16 | βœ… | + +**Variables** +- `vars/tyk.yaml` + +| Variable | Default | Comments | +| --------- | :---------: | --------- | +| secrets.APISecret | `352d20ee67be67f6340b4c0605b044b7` | API secret | +| secrets.AdminSecret | `12345` | Admin secret | +| gateway.service.host | | Gateway server host if different than the hosts url | +| gateway.service.port | `8080` | Gateway server listening port | +| gateway.service.proto | `http` | Gateway server protocol | +| gateway.service.tls | `false` | Set to `true` to enable SSL connections | +| gateway.sharding.enabled | `false` | Set to `true` to enable filtering (sharding) of APIs | +| gateway.sharding.tags | | The tags to use when filtering (sharding) Tyk Gateway nodes. Tags are processed as OR operations. If you include a non-filter tag (e.g. an identifier such as `node-id-1`, this will become available to your Dashboard analytics) | +| gateway.rpc.connString | | Use this setting to add the URL for your MDCB or load balancer host | +| gateway.rpc.useSSL | `true` | Set this option to `true` to use an SSL RPC connection| +| gateway.rpc.sslInsecureSkipVerify | `true` | Set this option to `true` to allow the certificate validation (certificate chain and hostname) to be skipped. This can be useful if you use a self-signed certificate | +| gateway.rpc.rpcKey | | Your organization ID to connect to the MDCB installation | +| gateway.rpc.apiKey | | This the API key of a user used to authenticate and authorize the Gateway’s access through MDCB. The user should be a standard Dashboard user with minimal privileges so as to reduce any risk if the user is compromised. The suggested security settings are read for Real-time notifications and the remaining options set to deny | +| gateway.rpc.groupId | | This is the `zone` that this instance inhabits, e.g. the cluster/data-center the Gateway lives in. The group ID must be the same across all the Gateways of a data-center/cluster which are also sharing the same Redis instance. This ID should also be unique per cluster (otherwise another Gateway cluster can pick up your keyspace events and your cluster will get zero updates). | + + diff --git a/tyk-stack.mdx b/tyk-stack.mdx new file mode 100644 index 00000000..ba3e4b72 --- /dev/null +++ b/tyk-stack.mdx @@ -0,0 +1,25 @@ +--- +title: "Tyk Stack" +order: 7 +sidebarTitle: "Tyk Stack" +--- + +import OssProductListInclude from '/snippets/oss-product-list-include.mdx'; + +<OssProductListInclude/> + +## Closed Source + +The following Tyk components, created and maintained by the Tyk Team, are proprietary and closed-source: + +* [Tyk Dashboard](/tyk-dashboard) +* [Tyk Developer Portal](/tyk-developer-portal) +* [Tyk Multi Data Center Bridge](/api-management/mdcb#managing-geographically-distributed-gateways-to-minimize-latency-and-protect-data-sovereignty) +* [Universal Data Graph](/api-management/data-graph#overview) +* [Tyk Operator](/api-management/automations/operator#what-is-tyk-operator) +* [Tyk Sync](/api-management/automations/sync) + +If you plan to deploy and use the above components On-premise, license keys are required. + +## Licensing +Read more about licensing [here](/apim#licensing). \ No newline at end of file diff --git a/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/approve-requests.mdx b/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/approve-requests.mdx new file mode 100644 index 00000000..c959317f --- /dev/null +++ b/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/approve-requests.mdx @@ -0,0 +1,32 @@ +--- +title: "Configure Provisioning Request" +'og:description': "How to configure Provisioning Request in Tyk developer portal" +tags: ['Developer Portal', 'Tyk', 'Dynamic Client Registration'] +sidebarTitle: "Provisioning Request" +--- + +## Approve or Reject Provisioning Request + +When an external developer is looking to access a specific API(s) as a part of an API Product, they will request access via the public facing portal. + +**Prerequisites** + +- A Tyk Enterprise portal installation +- A portal admin app login +- Log in to a provisioning request sent by an external API consumer + +**Step by step instructions** + +1. Log in to the portal admin app +2. Navigate to **Provisioning Requests** +3. Select which request you want to approve +4. Click the **more** symbol and select either **approve** or **reject** + +<img src="/img/dashboard/portal-management/enterprise-portal/approve-request.png" alt="Approve or reject an API provisioning request" /> + +## Configure Auto Approval + +You can auto approve provisioning requests. From the **Plans** section of the admin app, edit a plan and select **Auto-approve provisioning request** from the **Plan Settings**. By default this setting is not selected, requiring manual approval of each request. Click **Save Changes** to enable this setting. + +<img src="/img/dashboard/portal-management/enterprise-portal/auto-approve-requests.png" alt="Auto Approve API provisioning requests" /> + diff --git a/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/configuring-custom-rate-limit-keys.mdx b/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/configuring-custom-rate-limit-keys.mdx new file mode 100644 index 00000000..57f2ba3c --- /dev/null +++ b/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/configuring-custom-rate-limit-keys.mdx @@ -0,0 +1,84 @@ +--- +title: "Configuring Custom Rate Limit Keys in Developer Portal" +'og:description': "How to configure rate limit in Tyk developer portal" +tags: ['Developer Portal', 'Tyk', 'Rate Limit'] +sidebarTitle: "Configure Rate Limits" +--- + +## Introduction + +Different business models may require applying rate limits and quotas not only by credentials but also by other entities, e.g. per application, per developer, per organization etc. +For example, if an API Product is sold to a B2B customer, the quota of API calls is usually applied to all developers and their respective applications combined, in addition to a specific credential. + +To enable this, Tyk introduced support for custom rate limit keys in [Tyk 5.3.0](/developer-support/release-notes/dashboard#530-release-notes). This guide explains how to configure custom rate limit keys. + +**Prerequisites** + +This capability works with [Tyk 5.3.0](/developer-support/release-notes/dashboard#530-release-notes) or higher. + +### Configuring custom rate limit keys for policies in Tyk Dashboard + + +<Note> +If you are using Tyk Developer Portal version 1.13.0 or later, you can configure the custom rate limit keys directly from the Developer Portal in the Advanced settings (optional) colapsible section of the Plan's view (by Credentials metadata). +<img src="/img/dashboard/portal-management/enterprise-portal/portal-plan-advanced-settings.png" alt="Add Plan Advanced Settings" /> +</Note> + + +Custom rate limit keys are applied at a policy level. When a custom rate limit key is specified, quota, rate limit and throttling will be calculated against the specified value and not against a credential ID. + +To specify a custom rate limit key, add to a policy a new metadata field called `rate_limit_pattern`. +In the value field you can specify any value or expression that you want to use as a custom rate limit key for your APIs. +The `rate_limit_pattern` field supports referencing session metadata using `$tyk_meta.FIELD_NAME` syntax. +In addition, it's possible to concatenate multiple values together using the pipe operator (`|`). + +For instance, if you want to specify a rate limit pattern to calculate the rate limit for a combination of developers and plans, where all credentials of a developer using the same plan share the same rate limit, you can use the following expression. +This assumes that the `DeveloperID` and `PlanID` metadata fields are available in a session: + +```gotemplate +$tyk_meta.DeveloperID|$tyk_meta.PlanID +``` + +Here's how it looks like in the Tyk Dashboard UI: + +<img src="/img/dashboard/portal-management/enterprise-portal/configuring-custom-rate-limit-keys.png" alt="Configuring custom rate limit keys" /> + +<br/> + + +<Note> +**Updating credential metadata** + +Please note that the custom rate limit key capability uses only metadata objects, such as credentials metadata available in a session. +Therefore, if the `rate_limit_pattern` relies on credentials metadata, this capability will work only if those values are present. +If, after evaluating the `rate_limit_pattern`, its value is equal to an empty string, the rate limiter behavior defaults to rate limiting by credential IDs. +</Note> + + +### Using custom rate limit keys with the portal + +The Tyk Enterprise Developer Portal facilitates the configuration of various rate limiting options based on a business model for API Products published in the portal. + +To achieve this, the portal, by default, populates the following attributes in the credential metadata, which can be used as part of a custom rate limit key: +- **ApplicationID**: The ID of the application to which the credential belongs. +- **DeveloperID**: The ID of the developer who created the credential. +- **OrganizationID**: The ID of the organization to which the developer belongs. + +Additionally, it's possible to attach [custom attribute values](/portal/customization/user-model#add-custom-attributes-to-the-user-model) defined in a developer profile as metadata fields to credentials. + +When a credential is provisioned by the portal, all the fields described above are added as metadata values to the credential, making them valid options for configuring the rate limit key: + +<img src="/img/dashboard/portal-management/enterprise-portal/credential-metadata.png" alt="Credential's metadata" /> + +This approach allows the portal to seamlessly apply rate limits based on any combination of the aforementioned fields and other custom metadata objects defined in policies used for plans or products. This is in addition to credentials. + +--- + + +<Note> +**Tyk Enterprise Developer Portal** + +If you are interested in getting access contact us at [support@tyk.io](<mailto:support@tyk.io?subject=Tyk Enterprise Portal Beta>) +</Note> + + diff --git a/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/dynamic-client-registration.mdx b/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/dynamic-client-registration.mdx new file mode 100644 index 00000000..f96f3b34 --- /dev/null +++ b/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/dynamic-client-registration.mdx @@ -0,0 +1,393 @@ +--- +title: "Dynamic Client Registration" +'og:description': "How to configure Dynamic Client Registration in Tyk developer portal" +tags: ['Developer Portal', 'Tyk', 'Dynamic Client Registration'] +sidebarTitle: "Dynamic Client Registration" +--- + +## Introduction + +**Why OAuth2.0 is important** + +OAuth 2.0 is a crucial security mechanism for both public and internal APIs, as it provides a secure and standardized way to authenticate and authorize access to protected resources. It enables granular access control and revocation of access when necessary without exposing sensitive login credentials. In short, OAuth 2.0 offers a secure and flexible approach to managing access to APIs. + + +Implementing an OAuth2.0 provider can be a complex process that involves several technical and security considerations. As such, many API providers choose to use specialized identity providers instead of implementing OAuth2.0 provider themselves. + + +By using specialized identity providers, API providers can leverage the provider's expertise and infrastructure to manage access to APIs and ensure the security of the authentication process. This also allows API providers to focus on their core business logic and reduce the burden of managing user identities themselves. + +**How does Tyk help** + +Tyk offers a standard and reliable way to work with identity providers through the Dynamic Client Registration protocol (DCR), which is an [Internet Engineering Task Force](https://www.ietf.org/) protocol that establishes standards for dynamically registering clients with authorization servers. + +Tyk Enterprise Developer portal allows API providers to set up a connection with identity providers that support DCR so that API Consumers can use the OAuth2.0 credentials issued by the identity provider to access APIs exposed on the portal. + + +<br/> + +### Prerequisites +Before getting starting with configuring the portal, it's required to configure your Identity provider and the Dashboard beforehand. + +#### Create an initial access token +Before setting up Tyk Enterprise Developer Portal to work with DCR, you need to configure the identity provider. Please refer to the guides for popular providers to create the initial access token for DCR: +* [Gluu](https://gluu.org/docs/gluu-server/4.0/admin-guide/openid-connect/#dynamic-client-registration) +* [Curity](https://curity.io/docs/idsvr/latest/token-service-admin-guide/dcr.html) +* [Keycloak](https://github.com/keycloak/keycloak/blob/25.0.6/docs/documentation/securing_apps/topics/client-registration.adoc) +* [Okta](https://developer.okta.com/docs/reference/api/oauth-clients/) + + + + <Note> + Whilst many providers require initial access tokens, they are optional. Please refer to your provider documentation to confirm if required. + </Note> + + +#### Create OAuth2.0 scopes to enforce access control and rate limit + +Tyk uses OAuth2.0 scope to enforce access control and rate limit for API Products. Therefore, creating at least two scopes for an API Product and plan is required. + +The below example demonstrates how to achieve that with Keycloak and Okta in the tabs below. + +<Tabs> +<Tab title="Keycloak"> + +1. **Navigate to the Client scopes menu item** + + <img src="/img/dashboard/portal-management/enterprise-portal/step-1-navigate-to-the-client-scopes-menu.png" alt="Navigate to the Client scopes menu item" /> + +2. **Create a scope for an API Product** + + <img src="/img/dashboard/portal-management/enterprise-portal/step-2-create-a-scope-for-an-api-product.png" alt="Create a scope for an API Product" /> + +3. **Create a scope for a plan** + + <img src="/img/dashboard/portal-management/enterprise-portal/step-3-create-a-scope-for-a-plan.png" alt="Create a scope for a plan" /> + + + + <Note> + When using Keycloak, ensure that you set the type of the scope to be `Optional`. Default scopes are applied automatically, while optional scopes can be requested by clients on a case-by-case basis to extend the permissions granted by the user. In recent versions of Keycloak this should appear as a dropdown menu option, as shown in the images above. In older releases of Keycloak this may need to be set explicitly in a separate tab, as show on the image below. + </Note> + + +<img src="/img/dashboard/portal-management/enterprise-portal/old-keycloak-version-client-scope.png" alt="Client Scope Assigned Type" /> + +</Tab> + +<Tab title="Okta"> + +1. **Create an auth server or use the `Default` authorization server** + + Go to Security β†’ API, Edit one of the auth servers and navigate to `Scopes` + + <img src="/img/dashboard/portal-management/enterprise-portal/okta-api-page.png" alt="Add or Edit oauth servers in okta" /> + +2. **Create a scope for an API Product** + + <img src="/img/dashboard/portal-management/enterprise-portal/okta-product-payment.png" alt="Create a scope for an API Product" /> + +3. **Create a scope for a plan** + + <img src="/img/dashboard/portal-management/enterprise-portal/okta-free-plan-scope.png" alt="Create a scope for a plan" /> + +</Tab> + +</Tabs> + +### Create Tyk policies for an API Product and plan + + +<Note> +You can skip this step if you are using Tyk Developer Portal version 1.13.0 or later. +Go directly to [Configure Tyk Enterprise Developer Portal to work with an identity provider](#configure-tyk-enterprise-developer-portal-to-work-with-an-identity-provider). +</Note> + + +Navigate to the Tyk Dashboard and create two policies: one for a plan and one for an API Product. Both policies should include only the APIs with JWT authentication that you want to bundle as an API Product. + +1. **Create a policy for an API product.** + + <img src="/img/dashboard/portal-management/enterprise-portal/create-jwt-policy-for-product.png" alt="Create a policy for a product" /> + +2. **Create a policy for a plan.** + + <img src="/img/dashboard/portal-management/enterprise-portal/create-jwt-policy-for-plan.png" alt="Create a policy for a plan" /> + + +#### Create the No Operation policy and API + + +<Note> +You can skip this step if you are using Tyk Developer Portal version 1.13.0 or later. +Go directly to [Configure Tyk Enterprise Developer Portal to work with an identity provider](#configure-tyk-enterprise-developer-portal-to-work-with-an-identity-provider). +</Note> + + +Tyk requires any API that uses the scope to policy mapping to have [a default policy](/basic-config-and-security/security/authentication-authorization/json-web-tokens ). Access rights and rate limits defined in the default policy take priority over other policies, including policies for the API Product and plan. + +To avoid that, you need to create the No Operation API and policy that won't grant access to the APIs included in the API Product but will satisfy the requirement for a default policy. + +1. **Create the No Operation API.** + + Navigate to the `APIs` menu in the Tyk Dashboard: + <img src="/img/dashboard/portal-management/enterprise-portal/navigate-to-the-api-menu-in-the-tyk-dashboard.png" alt="Navigate to the API menu in the Tyk Dashboard" /> + + + Create a new HTTP API: + <img src="/img/dashboard/portal-management/enterprise-portal/create-noop-api.png" alt="Create the No Operation API" /> + + + Save it: + <img src="/img/dashboard/portal-management/enterprise-portal/save-the-noop-api.png" alt="Save the No Operation API" /> + +<br/> + +2. **Create the No Operation policy.** + + Navigate to the `Policies` menu in the Tyk Dashboard: + <img src="/img/dashboard/portal-management/enterprise-portal/navigate-to-the-policies-menu.png" alt="Navigate to the policies menu" /> + + Create a new policy and select the No Operation API in the `Add API Access Rights` section: + <img src="/img/dashboard/portal-management/enterprise-portal/create-noop-policy.png" alt="Create the No Operation policy" /> + + Configure the No Operation policy and save it: + <img src="/img/dashboard/portal-management/enterprise-portal/save-the-noop-policy.png" alt="Save the No Operation policy" /> + +#### Configure scope to policy mapping + + +<Note> +You can skip this step if you are using Tyk Developer Portal version 1.13.0 or later. +Go directly to [Configure Tyk Enterprise Developer Portal to work with an identity provider](#configure-tyk-enterprise-developer-portal-to-work-with-an-identity-provider). +</Note> + + +To enforce policies for the API Product and plan, you need to configure the scope to policy mapping for each API included in the API Product. +To achieve that, perform the following steps for each API included in the API Product. + +1. Navigate to the API. + + <img src="/img/dashboard/portal-management/enterprise-portal/navigate-to-the-api.png" alt="Navigate to the API" /> + +2. Select the required JWT signing method. In this example, we use RSA. Leave the `Public key` and `pol` fields blank, they will be filled automatically by the Enterprise portal. + + <img src="/img/dashboard/portal-management/enterprise-portal/select-signing-method.png" alt="Select signing method for the API" /> + +3. Select the No Operation policy as the default policy for this API. + + <img src="/img/dashboard/portal-management/enterprise-portal/select-the-default-policy.png" alt="Select the default policy for the API" /> + +4. Enable scope to policy mapping and specify the value of the JWT claim used to extract scopes in the `Scope name` field (the default value is "scope"). + + <img src="/img/dashboard/portal-management/enterprise-portal/enable-scope-to-policy-mapping.png" alt="Enable scope to policy mapping" /> + + +5. Add a scope to policy mapping for the product scope. Type the product scope in the `Claim field` and select the product policy. + + <img src="/img/dashboard/portal-management/enterprise-portal/add-a-scope-to-policy-mapping-for-the-product-scope.png" alt="Add scope to policy mapping for the product scope" /> + +6. Add a scope to policy mapping for the plan scope. Type the plan scope in the `Claim field` and select the plan policy, then save the API. + + <img src="/img/dashboard/portal-management/enterprise-portal/add-a-scope-to-policy-mapping-for-the-plan-scope.png" alt="Add scope to policy mapping for the plan scope" /> + +### Configure Tyk Enterprise Developer Portal to work with an identity provider + +Set up the portal to work with your IdP. + + +#### Configure the App registration settings + +In the portal, navigate to the `OAuth2.0 Providers` menu section. In that section, you need to configure the connection settings to the IdP and define one or more types (configurations) of OAuth 2.0 clients. For instance, you can define two types of OAuth 2.0 clients: +* A confidential client that supports the Client credential grant type for backend integrations; +* A web client that supports the Authorization code grant type for integration with web applications that can't keep the client secret confidential. + +Each configuration of OAuth 2.0 client could be associated with one or multiple API Products so that when an API Consumer requests access to an API Product, they can select a client type that is more suitable for their use case. + + +##### Specify connection setting to your IdP + +To connect the portal to the IdP, you need to specify the following settings: +* OIDC well-known configuration URL. +* Initial access token. + +First of all, select your IdP from the `Identity provider` dropdown list. Different IdPs have slightly different approaches to DCR implementation, so the portal will use a driver that is specific to your IdP. If your IdP is not present in the dropdown list, select the `Other` option. In that case, the portal will use the most standard implementation of the DCR driver, which implements the DCR flow as defined in the RFC. + +Then you need to specify the connection settings: [the initial access token and the well-known endpoint](#create-an-initial-access-token). If your Identity Provider uses certificates that are not trusted, the portal will not work with it by default. To bypass certificate verification, you can select the `SSL secure skip verify` checkbox. + +The below example demonstrates how to achieve that with Keycloak and Okta in the tabs below. + +<Tabs> + +<Tab title="Keycloak"> + +<img src="/img/dashboard/portal-management/enterprise-portal/specify-connection-setting-to-your-idp.png" alt="Specify connection setting to the IdP" /> + +</Tab> + +<Tab title="Okta"> + +<img src="/img/dashboard/portal-management/enterprise-portal/specify-connection-setting-to-your-idp-okta.png" alt="Specify connection setting to the IdP" /> + +**OIDC URL**: `{your-domain.com}/oauth2/default/.well-known/openid-configuration` + +**Registration Access Token**: To obtain token, Go to Okta Admin Console β†’ Security β†’ API β†’ Tokens β†’ Create New Token + +</Tab> + +</Tabs> + +##### Create client configurations +Once the connection settings are specified, you need to create one or multiple types of clients. You might have multiple types of clients that are suitable for different use cases, such as backend integration or web applications. + +You need at least one type of client for the DCR flow to work. To add the first client type, scroll down to the `Client Types` section and click on the `Add client type` button. + +To configure a client type, you need to specify the following settings: +* **Client type display name.** This name will be displayed to API consumers when they check out API products. Try to make it descriptive and short, so it's easier for API consumers to understand. +* **Description.** A more verbose description of a client type can be provided in this field. By default, we do not display this on the checkout page, but you can customize the respective template and make the description visible to API consumers. Please refer to [the customization section](/portal/customization#) for guidance. +* **Allowed response_types.** Response types associated with this type of client as per [the OIDC spec](https://openid.net/specs/openid-connect-core-1_0-17.html). +* **Allowed grant_types.** Grant types that this type of client will support as per [the OIDC spec](https://openid.net/specs/openid-connect-core-1_0-17.html). +* **Token endpoint auth methods.** The token endpoint that will be used by this type of client as per [the OIDC spec](https://openid.net/specs/openid-connect-core-1_0-17.html). +* Additionally, there’s an additional field for Okta: **Okta application type** which defines which type of Okta client should be created. Ignored for all other IdPs. + +Please note that your IdP might override some of these settings based on its configuration. + +The below example demonstrates how to achieve that with Keycloak and Okta in the tabs below. After configuring a client type, scroll to the top of the page to save it by clicking on the `SAVE CHANGES` button. + +<Tabs> + +<Tab title="Keycloak"> + +<img src="/img/dashboard/portal-management/enterprise-portal/configure-type-of-client.png" alt="Configure a client type" /> + +</Tab> + +<Tab title="Okta"> + +<img src="/img/dashboard/portal-management/enterprise-portal/configure-type-of-client-okta.png" alt="Configure a client type" /> + +**For Okta Client Credentials**: allowed response types MUST be token only + +</Tab> + +</Tabs> + +### Configure API Products and plans for the DCR flow +Once the App registration settings are configured, it is time for the final step: to configure the API Products and plans to work with the DCR flow. + +#### Configure API Products for the DCR flow + +To configure API Products to work with the DCR flow, you need to: +* Enable the DCR flow for the products you want to work with the DCR flow. +* Associate each product with one or multiple types of clients that were created in the previous step. +* Specify scopes for this API Product. Note the portal uses the scope to policy mapping to enforce access control to API Products, so there should be at least one scope. + +For achieving this, navigate to the `API Products` menu and select the particular API product you want to use for the DCR flow. Next, go to the β€˜App registration configs’ section and enable the β€˜Enable dynamic client registration’ checkbox. + +After that, specify the scope for this API product. You should have at least one scope that was created in [the Prerequisites for getting started](/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/dynamic-client-registration#prerequisites). If you need to specify more than one scope, you can separate them with spaces. + +Finally, select one or multiple types of clients that were created in [the Create client configurations](/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/dynamic-client-registration#create-client-configurations) section of this guide to associate them with that product. + +<Tabs> + +<Tab title="Keycloak"> + +<img src="/img/dashboard/portal-management/enterprise-portal/configure-api-products-for-the-dcr-flow.png" alt="Configure an API Product to work with the DCR flow" /> + +</Tab> + +<Tab title="Okta"> + +<img src="/img/dashboard/portal-management/enterprise-portal/configure-api-products-for-the-dcr-flow-okta.png" alt="Configure an API Product to work with the DCR flow" /> + +</Tab> + +</Tabs> + +<br /> + + +<Note> +From version 1.13.0, you can complete the DCR configuration for a product under the `Dynamic Client Registration` tab in the product's view. Scope to policy mapping for the selected API/s will be automatically configured using the scope defined in the `Scopes` field. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-product-dcr.png" alt="Add DCR settings" /> +</Note> + + +##### Configure plans for the DCR flow + +The last step is to configure the plans you want to use with the DCR flow. To do this, go to the portal's `Plans` menu section and specify the OAuth2.0 scope to use with each plan. You should have at least one scope that was created in [the Prerequisites for getting started](/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/dynamic-client-registration#prerequisites). If you need to specify more than one scope, you can separate them with spaces. +<img src="/img/dashboard/portal-management/enterprise-portal/configure-plan-for-the-dcr-flow.png" alt="Configure a plan to work with the DCR flow" /> + +<br /> + + +<Note> +From version 1.13.0, you can complete the DCR configuration for a plan under the `Advanced settings (optional)` colapsible section in the plan's view. Scope to policy mapping for the plan will be automatically configured using the scope defined in the `Scopes` field. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-plan-advanced-settings.png" alt="Add Plan Advanced Settings" /> +</Note> + + +### Test the DCR flow +To test the DCR flow, you need to perform the following actions: +- Request access to the API product and plan you have selected for the DCR flow as a developer. +- Approve the access request as an admin. +- As a developer, copy the access credentials and obtain an access token. +- As a developer, make an API call to verify the flow's functionality. + +#### Request access to the API Product +To request access to the DCR enabled API Product: +- Log in as a developer and navigate to the catalog page. +- Select the DCR enabled API Product and add it to the shopping cart. +- Navigate to the checkout page. +- On the checkout page, select a plan to use with that product, select an existing application, or create a new one. If you plan to build an application that uses the Authorization code grant type, you also need to specify redirect URI of your application in the `Redirect URLs` field. If you have multiple redirect URI, you can separate them with commas. +- Select a client type which is more suitable for your use case in the `Select a client type` section. +- Finally, select the applicable type of client and click on the `Submit request` button. +<img src="/img/dashboard/portal-management/enterprise-portal/request-access-to-the-dcr-enabled-product.png" alt="Request access to the DCR enabled product" width="600" /> + +#### Approve the access request +To approve the access request, navigate to the `Access requests` menu in the portal, select the access request and approve it by clicking on the `Approve` button. +<img src="/img/dashboard/portal-management/enterprise-portal/approve-dcr-access-request.png" alt="Approve DCR access request" /> + +#### Obtain an access token +Once the access request is approved, the developer should receive an email informing them of the approval. Please refer to [the email customization section](/portal/customization/email-notifications) if you wish to change the email template. + +As a developer, navigate to the `My Dashboard` section in the developer portal, select the application, and copy the OAuth 2.0 credentials. + +<img src="/img/dashboard/portal-management/enterprise-portal/copy-oauth-credentials.png" alt="Copy the OAuth2.0 credentials" /> + +Then use the credentials you have copied to obtain an access token. Make sure to include the scopes that are used to enforce access to the API product and plan. Otherwise, the gateway will not authorize the request. Here's an example of to achieve that with `curl`: +```curl +curl --location --request POST 'http://localhost:9999/realms/DCR/protocol/openid-connect/token' \ +--header 'Authorization: Basic N2M2NGM2ZTQtM2I0Ny00NTMyLWFlMWEtODM1ZTMyMWY2ZjlkOjNwZGlJSXVxd004Ykp0M0toV0tLZHFIRkZMWkN3THQ0' \ +--header 'Content-Type: application/x-www-form-urlencoded' \ +--data-urlencode 'scope=product_payments free_plan' \ +--data-urlencode 'grant_type=client_credentials' +``` +Since in this example we use the client_secret_basic token endpoint authentication method, the credentials must be supplied as a Base64-encoded string: `{client_id}:{client_secret}`. + +As a result, you should receive a JWT access token containing the required scopes: +<img src="/img/dashboard/portal-management/enterprise-portal/jwt.png" alt="An example of a JWT" width="600" /> + +#### Make an API Call +Finally, use the access token to make an API call and test the flow functionality: +```curl +curl --location --request GET 'http://localhost:8080/payment-api/get' \ +--header 'Authorization: Bearer eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJUR1ZQd25MWlduaWpNc2taU3lHeHFtYnFDNVlIcW9QUUJYZE4xTmJCRDZjIn0.eyJleHAiOjE2Nzg0NDA2ODksImlhdCI6MTY3ODQ0MDM4OSwianRpIjoiMGYwNTdlYjItODQ5My00ZmM2LTllMzQtZTk0OWUzYWQ2MmI2IiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo5OTk5L3JlYWxtcy9EQ1IiLCJzdWIiOiJlNGE3YmFkNy04ZDA4LTQxOTAtODc1Ni1mNTU1ZWQ3Y2JhZjciLCJ0eXAiOiJCZWFyZXIiLCJhenAiOiI3YzY0YzZlNC0zYjQ3LTQ1MzItYWUxYS04MzVlMzIxZjZmOWQiLCJzY29wZSI6ImZyZWVfcGxhbiBwcm9kdWN0X3BheW1lbnRzIiwiY2xpZW50SWQiOiI3YzY0YzZlNC0zYjQ3LTQ1MzItYWUxYS04MzVlMzIxZjZmOWQiLCJjbGllbnRIb3N0IjoiMTcyLjE3LjAuMSIsImNsaWVudEFkZHJlc3MiOiIxNzIuMTcuMC4xIn0.WGp9UIqE7CjFhHdaM64b0G2HGP4adaDg3dgc0YVCV9rTDYmri32Djku7PcLiDKyNLCvlQXUm_O2YmwMCLLUHKPGlRmBMG2y-79-T8z5V-qBATbE6uzwPh38p-SYIIDBUZtlMEhnVp049ZqNolUW-n2uB4CTRb0kDosdRnqhiMUFpe-ORwnZB-4BHGRlwWKyjc5Da6CvVczM1a_c5akqurGMFaX9DC81SS-zMXXpQPDpAkvUJBfLYDHEvXWH8JISqYv7ZQSAbOyE4b-EkVAesyHIMDCQ_pzf5Yp2ivM0dOufN9kdG2w_9ToMqJieVyQILJPowEakmEealbNUFQvc5FA' +``` + +You should receive the following response: +```{.json} expandable +{ + "args": {}, + "headers": { + "Accept": "*/*", + "Accept-Encoding": "gzip", + "Authorization": "Bearer eyJhbGciOiJSUzI1NiIsInR5cCIgOiAiSldUIiwia2lkIiA6ICJUR1ZQd25MWlduaWpNc2taU3lHeHFtYnFDNVlIcW9QUUJYZE4xTmJCRDZjIn0.eyJleHAiOjE2Nzg0NDA2ODksImlhdCI6MTY3ODQ0MDM4OSwianRpIjoiMGYwNTdlYjItODQ5My00ZmM2LTllMzQtZTk0OWUzYWQ2MmI2IiwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo5OTk5L3JlYWxtcy9EQ1IiLCJzdWIiOiJlNGE3YmFkNy04ZDA4LTQxOTAtODc1Ni1mNTU1ZWQ3Y2JhZjciLCJ0eXAiOiJCZWFyZXIiLCJhenAiOiI3YzY0YzZlNC0zYjQ3LTQ1MzItYWUxYS04MzVlMzIxZjZmOWQiLCJzY29wZSI6ImZyZWVfcGxhbiBwcm9kdWN0X3BheW1lbnRzIiwiY2xpZW50SWQiOiI3YzY0YzZlNC0zYjQ3LTQ1MzItYWUxYS04MzVlMzIxZjZmOWQiLCJjbGllbnRIb3N0IjoiMTcyLjE3LjAuMSIsImNsaWVudEFkZHJlc3MiOiIxNzIuMTcuMC4xIn0.WGp9UIqE7CjFhHdaM64b0G2HGP4adaDg3dgc0YVCV9rTDYmri32Djku7PcLiDKyNLCvlQXUm_O2YmwMCLLUHKPGlRmBMG2y-79-T8z5V-qBATbE6uzwPh38p-SYIIDBUZtlMEhnVp049ZqNolUW-n2uB4CTRb0kDosdRnqhiMUFpe-ORwnZB-4BHGRlwWKyjc5Da6CvVczM1a_c5akqurGMFaX9DC81SS-zMXXpQPDpAkvUJBfLYDHEvXWH8JISqYv7ZQSAbOyE4b-EkVAesyHIMDCQ_pzf5Yp2ivM0dOufN9kdG2w_9ToMqJieVyQILJPowEakmEealbNUFQvc5FA", + "Host": "httpbin.org", + "User-Agent": "curl/7.85.0", + }, + "origin": "XXX.XXX.XXX.XXX, XXX.XXX.XXX.XXX", + "url": "http://httpbin.org/get" +} +``` + diff --git a/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/manage-apps-credentials.mdx b/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/manage-apps-credentials.mdx new file mode 100644 index 00000000..ef263fd2 --- /dev/null +++ b/tyk-stack/tyk-developer-portal/enterprise-developer-portal/api-access/manage-apps-credentials.mdx @@ -0,0 +1,28 @@ +--- +title: "Manage Applications in Developer Portal" +'og:description': "How to configure applications in Tyk developer portal" +tags: ['Developer Portal', 'Tyk', 'Rate Limit'] +sidebarTitle: "Manage Applications" +--- + +## Introduction + +**Prerequisites** + +- A Tyk Enterprise portal installation +- A portal admin app login +- A login to the already created app by an external API consumer + +### To view existing apps + +1. In the portal admin app, go to **Apps**. If there are some apps already created by external API-consumers, you’ll see them in the overview list. +<img src="/img/dashboard/portal-management/enterprise-portal/list-of-app-admin-app.png" alt="List of applications" /> + +2. Click on an app from the overview page. +3. On the right hand slideout, you will see the information about the created app. + +### Revoke app credentials + +1. In the portal admin app, go to **Apps** and open the app. +2. Click **credentials**. This means the developer will no longer be able to access the API Product via the selected plan. These actions will have an immediate effect. + diff --git a/tyk-stack/tyk-developer-portal/enterprise-developer-portal/getting-started-with-enterprise-portal/manage-get-started-guides-for-api-products.mdx b/tyk-stack/tyk-developer-portal/enterprise-developer-portal/getting-started-with-enterprise-portal/manage-get-started-guides-for-api-products.mdx new file mode 100644 index 00000000..7bd1ce83 --- /dev/null +++ b/tyk-stack/tyk-developer-portal/enterprise-developer-portal/getting-started-with-enterprise-portal/manage-get-started-guides-for-api-products.mdx @@ -0,0 +1,105 @@ +--- +title: "Create Get started guides for your API Products" +'og:description': "How to configure API Providers in Tyk developer portal" +tags: ['Developer Portal', 'Tyk', 'Managing Access', 'Catalogs', 'Rate Limit', 'Dynamic Client Registration', 'Documenting APIs'] +sidebarTitle: "Create Documentation" +--- + +## Introduction + +As an API Provider, you can package APIs into API Products and supply them with documentation to deliver value to your internal and external API Consumers.<br/> +API documentation is essential for API Products. Good API documentation goes beyond just Open API Specification. To make your API Products easy to use and engaging, you can add technical and business guides to the documentation of your API Products, for instance: +* Get started guides explaining how to start with your API Product; +* Use-cases; +* Business documentation that explains the business value and helps to convince decision-makers. + +Tyk Enterprise Developer portal provides two ways of documenting APIs: +* Open API Specifications for APIs; +* The Get started documentation feature enables API Providers to add as many technical and business guides to API Products as needed. + +This section explains how to add the Get started documentation to API Products. + +### Create Get started guides for your API Product + + +1. **Create and publish an API Product** + + To start with, create and publish an API Product. Please refer to the [Publish API Products and Plans](/portal/overview/getting-started#publish-an-api-product) page for further guidance. + +2. **Add API Documentation to API Products** + + + + <Note> + If you are using Tyk Developer Portal version 1.13.0 or later, you can add API Documentation under the `"Getting Started" guides` tab of the API Product's view. + <img src="/img/dashboard/portal-management/enterprise-portal/portal-product-guides.png" alt="Add Product Guides" /> + </Note> + + + To add API Documentation, select an API Product and navigate to the API Documentation section. + Click the β€˜Add API Documentation’ button to add a new API Documentation page. + <img src="/img/dashboard/portal-management/enterprise-portal/api-docs-navigate-to-the-api-docs-section.png" alt="Navigate to the API Documentation section" /> + + <br/> + + Tyk developer portal stores two versions of each API Documentation page: in Markdown and HTML formats. + You can edit both versions, but only one version of the page will be displayed. + To edit the Markdown version, click on the β€˜Change to Markdown’ button. + <img src="/img/dashboard/portal-management/enterprise-portal/api-documentation-page-change-to-md.png" alt="Toggle the Markdown editor" /> + + <br/> + + To edit the HTML version, click the 'Change to HTML' button. + <img src="/img/dashboard/portal-management/enterprise-portal/api-documentation-page-change-to-html.png" alt="Toggle the HTML editor" /> + + <br/> + + To select which version of the documentation to display, check the β€˜Display Markdown’ checkbox. + When it’s checked, the portal will display the Markdown version of the page. + Otherwise, portal will display the HTML version. + <img src="/img/dashboard/portal-management/enterprise-portal/api-documentation-page-display-md-version.png" alt="Display the MD version of the page" /> + +3. **Add more documentation pages and set the display order** + + With the Tyk developer portal, you can add multiple Markdown and HTML pages to your API Products and select their display order to form a table of contents suitable for your use case. + Please refer to the previous step to add one more documentation page to the API Product. + + To reorder API Documentation items, scroll down to the API Documentation section and click on the β€˜Reorder items’ button. + <img src="/img/dashboard/portal-management/enterprise-portal/api-documentation-page-start-reordering-docs.png" alt="Reorder API Docs" /> + + <br/> + + Then click on the β€˜Move up’ button to move an API Documentation item one position up, and the β€˜Move down’ button to bring it one position down. Click on the 'Done' button and then on the 'Save' button to save your changes. + <img src="/img/dashboard/portal-management/enterprise-portal/api-documentation-page-move-doc-up.png" alt="Reorder API Docs" /> + + <br/> + + Once you save the changes, the new order of items will be reflected in the β€˜Get started’ tab. + <img src="/img/dashboard/portal-management/enterprise-portal/api-documentation-page-get-started-live.png" alt="Reorder API Docs" /> + + <br/> + + All documentation pages are listed in the β€˜Product docs’ section. You can access and edit the API docs from that section. + <img src="/img/dashboard/portal-management/enterprise-portal/product-docs-section.png" alt="Product docs section" /> + +4. **Add tags to blogs and API Products** + + Finally, you can add blog posts to your API Products using the Tags feature. + You can specify tags for API Product and blog posts, then the blog posts that match tags with an API Product will be displayed in the β€˜Related blog content’ section. + <img src="/img/dashboard/portal-management/enterprise-portal/tags-diagram.png" alt="Tags diagram" width="600" height="314" /> + + <br/> + + To specify tags for an API Product, select an API Product, scroll down to the β€˜Tags’ section, type in a tag, and press the Enter key. + <img src="/img/dashboard/portal-management/enterprise-portal/tags-section-in-api-product.png" alt="Specify tags for an API Product" /> + + <br/> + + To specify tags for a blog post, select a blog post, scroll down to the β€˜Tags’ section, type in a tag, and press the Enter key. + <img src="/img/dashboard/portal-management/enterprise-portal/tags-section-in-blog-post.png" alt="Specify tags for a blog post" /> + + <br/> + + The blog posts that match tags with an API Product will be displayed in the β€˜Related blog content’ section of the respective API Product page. + <img src="/img/dashboard/portal-management/enterprise-portal/related-blog-content.png" alt="Related blog content" width="800" height="925" /> + diff --git a/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/enable-sso.mdx b/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/enable-sso.mdx new file mode 100644 index 00000000..6e3152f6 --- /dev/null +++ b/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/enable-sso.mdx @@ -0,0 +1,647 @@ +--- +title: "Enable single sign on for admin users and developers" +'og:description': "Learn how to enable single sign on for admin users and developers in the Developer Portal." +tags: ['Tyk Developer Portal', 'Enterprise Portal', 'Email', 'Notifications'] +sidebarTitle: "Single Sign On" +--- + +## Introduction + +Single sign-on (SSO) enables users to access multiple applications using one set of login credentials, +reducing the burden of password management and improving security. SSO is relevant for businesses of all sizes, +streamlining access control and improving user experience. Regardless of your organization's size, implementing SSO can enhance security, +simplify access to enterprise resources, and strengthen user satisfaction. + +In this section, you'll learn how to enable single sign-on for admin users and developers in the Tyk Enterprise Developer portal with 3rd party identity providers (IDPs). + +**Prerequisites** +- A Tyk Enterprise portal installation +- [Supported](https://github.com/TykTechnologies/tyk-identity-broker#using-identity-providers) 3rd party identity provider up and running + +## Portal SSO Configuration Options + +Tyk Enterprise Developer Portal uses the [Tyk Identity Broker (TIB)](/api-management/external-service-integration#what-is-tyk-identity-broker-tib) to integrate Tyk authentication with 3rd party identity providers (IDPs). + +From portal version 1.12.0, TIB is embedded in the portal. With this, you have two options to configure SSO in the portal: + +1. **[Using Embedded TIB](#configuring-sso-with-embedded-tib)**: No need to install it separately. +2. **[Using External TIB](#configuring-sso-with-external-tib)**: If you are using a previous version of the portal, you can still use SSO with TIB installed as a separate application. + +## Configuring SSO with Embedded TIB + +Configuring SSO with Embedded TIB is a four-step process: + +1. **[Enabling Embedded TIB](#enabling-embedded-tib)** +2. **[Understanding UserGroup Mapping](#understanding-usergroup-mapping)** +3. **[Creating TIB Profile](#creating-tib-profile)** +4. **[Testing SSO](#testing-sso)** + +### Enabling Embedded TIB + +To enable Embedded TIB in the portal, add the `PORTAL_TIB_ENABLED` variable to [the portal .env file](/product-stack/tyk-enterprise-developer-portal/deploy/configuration#sample-env-file): +```.ini +PORTAL_TIB_ENABLED=true +``` + + +<Note> +The Tyk Enterprise Developer Portal embedded TIB only supports OIDC, LDAP or Social SSO providers. +</Note> + + +### Understanding UserGroup Mapping + +The Tyk Enterprise Developer portal has two audiences: + +1. **Developers**: + + Developers created by the sso flow are portal users that belong to an organization and team/s, if a user group mapping is not specified, they are assigned to the default organization and default team. Developers created by the sso flow are always assinged the **Consumer Super Admin** role. If part of an organization and a team, this means that the developer is a super admin for that organization. Read more about managing api consumer organizations [here](/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/manage-api-consumer-organisations). + +2. **Admins**: + + Admins created by the SSO flow are portal users who do not belong to any organization (OrgID is 0) and are assigned the **Provider Admin** role. + +TIB uses **user group mapping** to map the user groups from the identity provider to the portal teams within an organization. +<img src="/img/dashboard/portal-management/enterprise-portal/user-group-mapping.png" alt="User group mapping" width="600" /> + +To define the user group mapping for your developer audience, you need to add the UserGroupMapping object to the corresponding [TIB profile](/api-management/external-service-integration#exploring-tib-profiles): +```yaml + "UserGroupMapping": { + "{IDP groupA ID}": "{portal teamA ID}", + "{IDP groupB ID}": "{portal teamB ID}", + ... + } +``` + +#### Default behaviour of UserGroup Mapping + +The `UserGroupMapping` object contains keys that refer to group IDs in your IDP, and the corresponding values are team IDs in the portal. +When the Tyk Identity Broker authorizes a user, it searches for a key that matches the user's group ID in the IDP. +If TIB can't find a matching group ID, it logs the user in to the team with an ID equal to `DefaultUserGroupID` in the portal (if `DefaultUserGroupID` is defined). +We recommend always defining `DefaultUserGroupID` and ensuring it refers to a valid team ID in your portal instance. The portal will refuse login attempts if `DefaultUserGroupID` is defined but refers to an invalid team ID. + +If no matching group ID is found in the `UserGroupMapping` object and `DefaultUserGroupID` isn't defined, the portal logs in the user to the "Default organization | All users" team with an ID of 1. + +#### Login Evaluation Algorithm + +To determine whether a developer should be allowed to log in and which team they should be logged into, the portal uses the following algorithm: +<img src="/img/dashboard/portal-management/enterprise-portal/user-group-mapping-algorithm.png" alt="User group mapping algorithm" width="1000" /> + +### Creating TIB Profile + +In the following sections you will learn how to configure the SSO profiles for admins and developers and map developers to the teams. + +You can configure the SSO profiles for admins in the Tyk Developer Portal application. Under **Settings** > **SSO Profiles** > **Add new SSO Profile**. + +There are two ways of creating SSO profiles: +1. **[Wizard Form](#using-the-wizard-form)**: Create a profile using the wizard guided form. +2. **[Raw JSON Editor](#using-the-json-raw-editor)**: Create a profile using JSON editor where you can specify your tib raw JSON profile. + +#### Using the Wizard Form + +You can access the wizard form by switching to the **Wizard** view. +<Tabs> +<Tab title="Profile for admins"> +Create a profile for admins: +1. Complete the **Profile action** step. Here you can choose a name (this name will generate the profile ID), the profile type (select **Profile for admin users**) and the redirect url on failure. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-sso-wizard-admin-1.png" alt="SSO Profiles Wizard" /> +2. Select a supported **Provider type**. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-sso-wizard-provider.png" alt="SSO Profiles Wizard" /> +3. Complete the **Profile configuration** step. Here you can specify the access to your idp. And Advanced settings if needed. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-sso-wizard-profile-config.png" alt="SSO Profiles Wizard" /> +4. Don't add any group mapping since we are creating a profile for admins and will be ignored if added. Click on **Continue** to create the profile. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-sso-wizard-admin-4.png" alt="SSO Profiles Wizard" /> +</Tab> +<Tab title="Profile for developers"> + +Create a profile for developers: + +1. Complete the **Profile action** step. Here, you can choose a name (this name will generate the profile ID), the profile type (select **Profile for developers**), and the redirect URL on failure. + + <img src="/img/dashboard/portal-management/enterprise-portal/portal-sso-wizard-dev-1.png" alt="SSO Profiles Wizard" /> + +2. Select a supported **Provider type**. + + <img src="/img/dashboard/portal-management/enterprise-portal/portal-sso-wizard-dev-2.png" alt="SSO Profiles Wizard" /> + +3. Complete the **Profile configuration** step. Here you can specify the access to your idp. And Advanced settings if needed. + + <img src="/img/dashboard/portal-management/enterprise-portal/portal-sso-wizard-dev-3.png" alt="SSO Profiles Wizard" /> + +4. Add the group mapping for the developers. **Custom user group claim name** must equal the JWT claim name that refers to the user group in your IDP. + + <img src="/img/dashboard/portal-management/enterprise-portal/portal-sso-wizard-dev-4.png" alt="SSO Profiles Wizard" /> + +5. Click on **Continue** to create the profile. +</Tab> +</Tabs> + +#### Using the JSON Raw Editor + +The Tyk Identity Broker (TIB) uses [profiles](/api-management/external-service-integration#exploring-tib-profiles) to define details related to the identity provider such as its type and access credentials, and instructs TIB on how to treat users that try log in with that provider. +You can access the raw editor by switching to the **Raw editor** view, which displays a JSON editor with an empty TIB profile for guidance. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-sso-raw-editor.png" alt="SSO Profiles Raw Editor" /> + +<Tabs> +<Tab title="Profile for admins"> +Create a profile for admins. Make sure the ActionType is equal to `GenerateOrLoginUserProfile` and the OrgID is equal to β€œ0.” +<img src="/img/dashboard/portal-management/enterprise-portal/portal-sso-raw-editor-admin.png" alt="SSO Profiles Raw Editor" /> + +In the above example, you need to specify the following parameters: +- `OrgID` must be `"0"` for being accepted as a provider-admin +- `ActionType` must be equal to `"GenerateOrLoginUserProfile"` +- Replace the `host` and `port` in the fields `CallbackBaseURL`, `FailureRedirect` and `ReturnURL` with the actual host and port on which your portal instance is running. Also, replace `http` with `https` for the respective fields if you use https for your portal instance +- Replace the `host` and `port` in the field `DiscoverURL` with the actual host and port on which your IDP instance is running. Also, replace `http` with `https` accordingly +- In the `"ID"` field, specify an ID of this TIB profile. You can select any value for this field that consists of digits, letters, and special signs, no spaces allowed. It is better to pick a human-readable ID for your profile for better maintainability of the configuration + +</Tab> + +<Tab title="Profile for developers"> +Create a developer profile. Ensure the ActionType is equal to β€œGenerateOrLoginDeveloperProfile,” and if you define a user group mapping, the team/s exist in the portal. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-sso-raw-editor-dev.png" alt="SSO Profiles Raw Editor" /> + +In the above example, you need to specify the following parameters: +- `OrgID` could be anything as its value is ignored; +- `ActionType` must be equal to `"GenerateOrLoginDeveloperProfile"` +- Replace the `host` and `port` in the fields `CallbackBaseURL`, `FailureRedirect` and `ReturnURL` with the actual host and port on which your portal instance is running. Also, replace `http` with `https` for the respective fields if you use HTTPS for your portal instance +- Replace the `host` and `port` in the field `DiscoverURL` with the actual host and port on which your IDP instance is running. Also, replace `http` with `https` accordingly +- In the `"ID"` field, specify an ID of this TIB profile. You can select any value for this field that consists of digits, letters, and special signs; no spaces are allowed. It is better to pick a human-readable ID for your profile for better maintainability of the configuration +- `CustomUserGroupField` must be equal to the JWT claim name that refers to the user group in your IDP +- `UserGroupMapping` is an object that defines the relationship between user groups in the IDP and teams in the portal. If not specified, the optional parameter will cause the portal to rely on the `DefaultUserGroupID` field to determine which team a developer should log in to. Please refer to the [User group mapping section](#user-group-mapping ) for guidance +- `DefaultUserGroupID` is the default organization that the portal will use to determine which team a developer should be logged in to if it is not able to find a UserGroupMapping for that developer +</Tab> + +<Note> +**Nuances of OIDC configuration** + +To ensure that the portal can log in a user with your OIDC Identity provider, you may need to either explicitly specify the email scopes in a profile +configuration or configure your IDP to include the email claim in the JWT. Failure to include the email scope in the JWT +would result in the portal not having access to the user's email. + +As an example, for Okta, you can use the following configuration: +```json expandable +"UseProviders": [ + { + "Name": "openid-connect", + "Key": "{oAuth2.0 key}", + "Secret": "{oAuth2.0 secret}", + "Scopes": ["openid", "email"], + "DiscoverURL": "{OIDC well-known endpoint}" + } +] +``` +</Note> + +Please refer to the [TIB configuration section](/api-management/external-service-integration#single-sign-on-sso) for step-by-step instructions for setting up the UseProviders section. + +</Tabs> + +### Testing SSO + +You can access the login URL in your SSO Profile details **Provider configuration** section. +<img src="/img/dashboard/portal-management/enterprise-portal/portal-sso-login.png" alt="SSO Profile Details" /> + +Tyk Enterprise Developer Portal doesn't supply a login page for Single Sign-On out of the box, so you might need to create one. +Here is an example of such a page that works with a profile for the LDAP identity management system: +```.html expandable +<html> + <head> + <title>Tyk Developer portal login + + + Login to the Developer portal +
    + username:
    + password:
    + +
    + + +``` + + +Configuration on the portal side is quite straightforward. You need to specify the portal SSO API secret that acts as a credential for the APIs that are used by TIB for communication with the portal within Single Sign-On flow. +You can use any value for the portal SSO API secret, but it should be consistent with [TIB configuration](#configure-tyk-identity-broker-to-work-with-tyk-enterprise-developer-portal). + +To specify the portal SSO API secret, add the `PORTAL_API_SECRET` variable to [the portal .env file](/product-stack/tyk-enterprise-developer-portal/deploy/configuration#sample-env-file): +```.ini +PORTAL_API_SECRET=your-portal-api-secret +``` + +If you use [the Tyk helm chart](/tyk-self-managed/install#install-more-tyk-components), it is required to add the `PORTAL_API_SECRET` to extraEnvs: +```.yaml +extraEnvs: +- name: PORTAL_API_SECRET + value: "your-portal-api-secret" +``` + +## Configuring SSO with External TIB + +### Configure Tyk Identity Broker to work with Tyk Enterprise Developer Portal +The Tyk Enterprise Developer portal uses the [Tyk Identity Broker](/api-management/external-service-integration#what-is-tyk-identity-broker-tib) to work with various Identity Management Systems, such as LDAP, +Social OAuth (e.g., GPlus, Twitter, GitHub), or Basic Authentication providers. Therefore, to configure Single Sign-On for the portal, +you need to install and configure Tyk Identity Broker first. Follow these steps to achieve this: + +#### Install Tyk Identity Broker +Please refer to [the TIB installation guide documentation](/api-management/external-service-integration#install-standalone-tib) for different installation options: +- [Docker](https://hub.docker.com/r/tykio/tyk-identity-broker/#the-tibconf-file) +- [packages](https://packagecloud.io/tyk/tyk-identity-broker/install#bash-deb) +- [Tyk helm chart](/api-management/external-service-integration#install-standalone-tib) + +#### Specify TIB settings to work with the Tyk Enterprise Developer portal + +##### Docker or packages + +Create tib.conf file for [the Docker installation](https://hub.docker.com/r/tykio/tyk-identity-broker/#the-tibconf-file) or if you use [packages](https://packagecloud.io/tyk/tyk-identity-broker/install#bash-deb) to deploy TIB: +```.json expandable +{ + "Secret":"test-secret", + "HttpServerOptions":{ + "UseSSL":false, + "CertFile":"./certs/server.pem", + "KeyFile":"./certs/server.key" + }, + "SSLInsecureSkipVerify": true, + "BackEnd": { + "Name": "in_memory", + "IdentityBackendSettings": { + "Hosts" : { + "localhost": "6379" + }, + "Password": "", + "Database": 0, + "EnableCluster": false, + "MaxIdle": 1000, + "MaxActive": 2000 + } + }, + "TykAPISettings":{ + "DashboardConfig":{ + "Endpoint":"https://{your portal host}", + "Port":"{your portal port}", + "AdminSecret":"{portal-api-secret}" + } + } +} +``` +Setting reference: +- **TykAPISettings.DashboardConfig.Endpoint** is the Developer portal url. Pay attention if any of the elements (TIB or Portal) is running on containers. +- **TykAPISettings.DashboardConfig.Port** is the Developer portal port. +- **TykAPISettings.DashboardConfig.AdminSecret** is `PortalAPISecret` in the configuration file of the Developer portal. + +The full reference for the configuration file is in [the TIB section of the documentation](/tyk-configuration-reference/tyk-identity-broker-configuration). +##### Helm charts +If you wish ot deploy TIB in Kubernetes via [Tyk helm chart](/api-management/external-service-integration#install-standalone-tib), you need to specify TIB config as extraVars: +```.yaml expandable +extraEnvs: + - name: TYK_IB_HTTPSERVEROPTIONS_CERTFILE + value: "./certs/server.pem" + - name: TYK_IB_HTTPSERVEROPTIONS_KEYFILE + value: "./certs/server.key" + - name: TYK_IB_SSLINSECURESKIPVERIFY + value: "true" + - name: TYK_IB_BACKEND_NAME + value: "in_memory" + - name: TYK_IB_BACKEND_IDENTITYBACKENDSETTINGS_HOSTS + value: "redis.tyk-cp:6379" + - name: TYK_IB_BACKEND_IDENTITYBACKENDSETTINGS_PASSWORD + value: "" + - name: TYK_IB_BACKEND_IDENTITYBACKENDSETTINGS_DATABASE + value: "0" + - name: TYK_IB_BACKEND_IDENTITYBACKENDSETTINGS_ENABLECLUSTER + value: "false" + - name: TYK_IB_BACKEND_IDENTITYBACKENDSETTINGS_MAXIDLE + value: "1000" + - name: TYK_IB_BACKEND_IDENTITYBACKENDSETTINGS_MAXACTIVE + value: "2000" + - name: TYK_IB_TYKAPISETTINGS_DASHBOARDCONFIG_ENDPOINT + value: "https://{your portal host}" + - name: TYK_IB_TYKAPISETTINGS_DASHBOARDCONFIG_PORT + value: "{your portal port}" + - name: TYK_IB_TYKAPISETTINGS_DASHBOARDCONFIG_ADMINSECRET + value: "{portal-api-secret}" +``` + +The full reference for the configuration file is in [the TIB section of the documentation](/tyk-configuration-reference/tyk-identity-broker-configuration). + +### Configure Single Sign-On for admin users and developers + +#### What is the Tyk Identity Broker profile +The Tyk Identity Broker (TIB) uses [profiles](/api-management/external-service-integration#exploring-tib-profiles) to define details related to the identity provider such as its type and access credentials, and instructs TIB on how to treat users that try log in with that provider. +In this guide, you will create two TIB profiles for admins users and developers. This allows you to have different identity providers for admins and developers as well as for internal and external users. + +Depending on your installation options for TIB, you need to specify profiles via a json file (for Docker or packages) or via a ConfigMap (for Tyk Helm Chart). + +##### profiles.json for Docker or packages installation +Here is an example of profiles.json file for Docker or packages installation: +```.json expandable +[ + { + "ActionType": "GenerateOrLoginUserProfile", + "ID": "{ID of your TIB profile}", + "OrgID": "0", + "IdentityHandlerConfig": { + "DashboardCredential": "{portal API secret}" + }, + "ProviderConfig": { + "CallbackBaseURL": "http://{TIB host}:{TIB port}", + "FailureRedirect": "http://{portal host}:{portal port}/?fail=true", + "UseProviders": [ + { + "Name": "openid-connect", + "Key": "{oAuth2.0 key}", + "Secret": "{oAuth2.0 secret}", + "DiscoverURL": "OIDC well-known endpoint" + } + ] + }, + "ProviderName": "SocialProvider", + "ReturnURL": "http://{portal host}:{portal port}/sso", + "Type": "redirect" + }, + { + "ActionType": "GenerateOrLoginDeveloperProfile", + "ID": "{ID of your TIB profile}", + "OrgID": "0", + "IdentityHandlerConfig": { + "DashboardCredential": "{portal API secret}" + }, + "ProviderConfig": { + "CallbackBaseURL": "http://{TIB host}:{TIB port}", + "FailureRedirect": "http://{portal host}:{portal port}/?fail=true", + "UseProviders": [ + { + "Name": "openid-connect", + "Key": "{oAuth2.0 key}", + "Secret": "{oAuth2.0 secret}", + "DiscoverURL": "OIDC well-known endpoint" + } + ] + }, + "ProviderName": "SocialProvider", + "ReturnURL": "http://{portal host}:{portal port}/sso", + "Type": "redirect", + "DefaultUserGroupID": "1" + } +] +``` + +##### ConfigMap for Tyk Helm chart installation +Here is an example of ConfigMap for the Tyk Helm chart installation: +```.yaml expandable +apiVersion: v1 +kind: ConfigMap +metadata: + name: tyk-tib-profiles-conf +data: + profiles.json: | + [{ + "ActionType": "GenerateOrLoginUserProfile", + "ID": "{ID of your TIB profile}", + "OrgID": "0", + "IdentityHandlerConfig": { + "DashboardCredential": "{portal API secret}" + }, + "ProviderConfig": { + "CallbackBaseURL": "http://{TIB host}:{TIB port}", + "FailureRedirect": "http://{portal host}:{portal port}/?fail=true", + "UseProviders": [ + { + "Name": "openid-connect", + "Key": "{oAuth2.0 key}", + "Secret": "{oAuth2.0 secret}", + "DiscoverURL": "OIDC well-known endpoint" + } + ] + }, + "ProviderName": "SocialProvider", + "ReturnURL": "http://{portal host}:{portal port}/sso", + "Type": "redirect" + }, + { + "ActionType": "GenerateOrLoginDeveloperProfile", + "ID": "{ID of your TIB profile}", + "OrgID": "0", + "IdentityHandlerConfig": { + "DashboardCredential": "{portal API secret}" + }, + "ProviderConfig": { + "CallbackBaseURL": "http://{TIB host}:{TIB port}", + "FailureRedirect": "http://{portal host}:{portal port}/?fail=true", + "UseProviders": [ + { + "Name": "openid-connect", + "Key": "{oAuth2.0 key}", + "Secret": "{oAuth2.0 secret}", + "DiscoverURL": "OIDC well-known endpoint" + } + ] + }, + "ProviderName": "SocialProvider", + "ReturnURL": "http://{portal host}:{portal port}/sso", + "Type": "redirect", + "DefaultUserGroupID": "1" + }] +``` + +#### Configure Single Sign-On for admin users +The Tyk Enterprise Developer portal has two audiences: developers and admins. This section provides guidance on implementing +Single Sign-On for admin users. The configuration is rather straightforward, and you need to take these three steps +to enable Single Sign-On for admin users in your portal instance: +1. Create a profile for the Tyk Identity Broker (TIB) to work on your identity provider. Make sure the ActionType is equal to "GenerateOrLoginUserProfile", and OrgID is equal to "0": +```.json expandable +[{ + "ActionType": "GenerateOrLoginUserProfile", + "ID": "{ID of your TIB profile}", + "OrgID": "0", + "IdentityHandlerConfig": { + "DashboardCredential": "{portal API secret}" + }, + "ProviderConfig": { + "CallbackBaseURL": "http://{TIB host}:{TIB port}", + "FailureRedirect": "http://{portal host}:{portal port}/?fail=true", + "UseProviders": [ + { + "Name": "openid-connect", + "Key": "{oAuth2.0 key}", + "Secret": "{oAuth2.0 secret}", + "DiscoverURL": "{OIDC well-known endpoint}" + } + ] + }, + "ProviderName": "SocialProvider", + "ReturnURL": "http://{portal host}:{portal port}/sso", + "Type": "redirect" +}] +``` +In the above example, you need to specify the following parameters: +- `OrgID` must be `"0"` for being accepted as a provider-admin or super-admin +- `ActionType` must be equal to `"GenerateOrLoginUserProfile"` +- `IdentityHandlerConfig.DashboardCredential` must be equal to the `PortalAPISecret` field in the configuration file of the portal +- Replace `{portal host}` and `{portal port}` with the actual host and port on which your portal instance is running. Also, replace `http` with `https` for the respective fields if you use https for your portal instance +- Replace `{TIB host}` and `{TIB port}` with the actual host and port on which your TIB instance is running. Also, replace `http` with `https` for the respective fields if you use https for your TIB instance +- In the `"ID"` field, specify an ID of this TIB profile. You can select any value for this field that consists of digits, letters, and special signs, no spaces allowed. It is better to pick a human-readable ID for your profile for better maintainability of the configuration + + + + + **Nuances of OIDC configuration** + + To ensure that the portal can log in a user with your OIDC Identity provider, you may need to either explicitly specify the email scopes in a profile + configuration or configure your IDP to include the email claim in the JWT. Failure to include the email scope in the JWT + would result in the portal not having access to the user's email. + + As an example, for Okta, you can use the following configuration: + ```yaml expandable + "UseProviders": [ + { + "Name": "openid-connect", + "Key": "{oAuth2.0 key}", + "Secret": "{oAuth2.0 secret}", + "Scopes": ["openid", "email"], + "DiscoverURL": "{OIDC well-known endpoint}" + } + ] + ``` + + + +Please refer to the [TIB configuration section](/api-management/external-service-integration#single-sign-on-sso) for step-by-step instructions for setting up the UseProviders section. +Any changes to the TIB profile will be effective after restarting your TIB instance. + +2. Create a login page for admin users. We don't supply a login page for Single Sign-On out of the box, so you need to create one. +Here is an example of such page that works with a profile for LDAP identity management system: +```.html + + + Tyk Developer portal login + + + Login to the Developer portal +
    + username:
    + password:
    + +
    + + +``` expandable +3. Now you should be able to log in to the portal with your identity provider as an admin user + +#### Configure Single Sign-On for developers +This section relates to configuration and settings required to set up Single Sign-On for developers. Configuration for developers is also straight forward. +However, for developers there is one additional. + +##### User group mapping +In order to land a developer into the right API Consumer organization, it is necessary to configure the UserGroupMapping +in the TIB profile that creates a binding between user groups in your IDP and developer teams in the portal. + +User group mapping + +To define the user group mapping for your developer audience, you need to add the UserGroupMapping object to the corresponding TIB profile: +```yaml + "UserGroupMapping": { + "{IDP groupA ID}": "{portal teamA ID}", + "{IDP groupB ID}": "{portal teamB ID}", + ... + } +``` expandable + +The `UserGroupMapping` object contains keys that refer to group IDs in your IDP, and the corresponding values are team IDs in the portal. +When the Tyk Identity Broker authorizes a user, it searches for a key that matches the user's group ID in the IDP. +If TIB can't find a matching group ID, it logs the user in to the team with an ID equal to `DefaultUserGroupID` in the portal (if `DefaultUserGroupID` is defined). +We recommend always defining `DefaultUserGroupID` and ensuring that it refers to a valid team ID in your portal instance. If `DefaultUserGroupID` is defined but refers to an invalid team ID, the portal will refuse login attempts. + +If no matching group ID is found in the `UserGroupMapping` object and `DefaultUserGroupID` isn't defined, the portal logs in the user to the "Default organization | All users" team with an ID of 1. + +To determine whether a developer should be allowed to log in and which team they should be logged into, the portal uses the following algorithm: +User group mapping algorithm + + +##### Configure profile to enable Single Sign-On for developers +Follow these steps to enable Single Sign-On for developers: +1. Create a profile for the Tyk Identity Broker (TIB) to work on your identity provider. Make sure the ActionType is equal to "GenerateOrLoginUserProfile", and OrgID is equal to "0": +```.json +[{ + "ActionType": "GenerateOrLoginDeveloperProfile", + "ID": "{ID of your TIB profile}", + "OrgID": "0", + "IdentityHandlerConfig": { + "DashboardCredential": "{ID of your TIB profile}" + }, + "ProviderConfig": { + "CallbackBaseURL": "http://{TIB host}:{TIB port}", + "FailureRedirect": "http://{portal host}:{portal port}/?fail=true", + "UseProviders": [ + { + "Name": "openid-connect", + "Key": "{oAuth2.0 key}", + "Secret": "{oAuth2.0 secret}", + "DiscoverURL": "{OIDC well-known endpoint}" + } + ] + }, + "ProviderName": "SocialProvider", + "ReturnURL": "http://{portal host}:{portal port}/sso", + "Type": "redirect", + "CustomUserGroupField": "{your group ID field}", + "UserGroupMapping": { + "{IDP group ID}": "{portal team ID}" + }, + "DefaultUserGroupID": "{portal team ID}" +}] +``` expandable +In the above example, you need to specify the following parameters: +- `OrgID` could be anything as its value is ignored; +- `ActionType` must be equal to `"GenerateOrLoginDeveloperProfile"` +- `IdentityHandlerConfig.DashboardCredential` must be equal to the `PortalAPISecret` field in the configuration file of the portal +- Replace `{portal host}` and `{portal port}` with the actual host and port on which your portal instance is running. Also, replace `http` with `https` for the respective fields if you use https for your portal instance +- Replace `{TIB host}` and `{TIB port}` with the actual host and port on which your TIB instance is running. Also, replace `http` with `https` for the respective fields if you use https for your TIB instance +- In the `"ID"` field, specify an ID of this TIB profile. You can select any value for this field that consists of digits, letters, and special signs, no spaces allowed. It is better to pick a human-readable ID for your profile for better maintainability of the configuration +- `CustomUserGroupField` must be equal to the JWT claim name that refers to the user group in your IDP +- `UserGroupMapping` an object that defines relationship between user groups in the IDP and teams in the portal. The optional parameter, if not specified, will cause the portal to rely on the `DefaultUserGroupID` field to determine which team a developer should log in to. Please refer to the [User group mapping section](#user-group-mapping ) for guidance +- `DefaultUserGroupID` is the default organization that the portal will use to determine which team a developer should be logged in to if it is not able to find a UserGroupMapping for that developer + + + + + **Nuances of OIDC configuration** + + To ensure that the portal can log in a user with your OIDC Identity provider, you may need to either explicitly specify the email scopes in a profile + configuration or configure your IDP to include the email claim in the JWT. Failure to include the email scope in the JWT + would result in the portal not having access to the user's email. + + As an example, for Okta, you can use the following configuration: + ```json + "UseProviders": [ + { + "Name": "openid-connect", + "Key": "{oAuth2.0 key}", + "Secret": "{oAuth2.0 secret}", + "Scopes": ["openid", "email"], + "DiscoverURL": "{OIDC well-known endpoint}" + } + ] + ``` + + + +2. Create a login page for developers. We don't supply a login page for Single Sign-On out of the box, so you need to create one. +Here is an example of such page that works with a profile for LDAP identity management system: +```.html + + + Tyk Developer portal login + + + Login to the Developer portal +
    + username:
    + password:
    + +
    + + +``` +3. Now you should be able to log in to the portal with your identity provider as a developer + diff --git a/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/manage-api-consumer-organisations.mdx b/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/manage-api-consumer-organisations.mdx new file mode 100644 index 00000000..8170504e --- /dev/null +++ b/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/manage-api-consumer-organisations.mdx @@ -0,0 +1,144 @@ +--- +title: "Manage Organizations" +'og:description': "How to manage organizations in Tyk developer portal" +tags: ['Developer Portal', 'Tyk', 'API Consumer', 'Organization'] +sidebarTitle: "Organizations" +--- + +## Introduction + +Quite often, API Providers have to provide API Products to other companies. In fact, 90% of our customers say that their primary audience is other companies. In this case, they are dealing with not just individual developers but with teams of developers. +Unlike individual developers, companies require more sophisticated machinery to access API credentials: +* Usually, a company is represented by a team of developers, not just an individual. Communication between API Providers and API Consumers mustn’t rely on a single individual that may leave a company or be fired; +* API Consumers need to share access credentials securely within their team. Without that capability, they have to share credentials with internal communication tools, which is a horrible practice. Credentials may be stolen, exposed to an incorrect audience, or not appropriately updated; +* Those teams have an internal hierarchy: some users have admin responsibilities with broader permissions, while other teammates’ permissions are restricted to only accessing API Credentials; +* API Consumers should be able to maintain their teams by themselves: invite new members or remove ones that left the team. + +So, simply put, there are two main challenges that the new API Consumer organization management capability solves: +* How to share securely share access credentials between team members; +* How to manage user permissions on the API consumer side. + +**Prerequisites** + +Before starting, you need to set up an email server because it’s used to send invitations to API Consumer team members. +Please refer to the email notifications documentation to set up the email server. + +Please refer to the [email notification section](/product-stack/tyk-enterprise-developer-portal/getting-started/setup-email-notifications) for further instructions for setting up the email server. + +## Admin Settings and Governance + +You can control if API Consumers can register an organization and if such registration requires approval from the portal admins. +To enable API Consumer organization registration, navigate to the Settings/General menu and scroll to the API Consumer access section. In that section, there are two settings that control API Consumer registration: +* **Enable API consumers to register organizations**: when this setting is enabled, API Consumers can register organizations, and the respective button appears in the navigation menu; +* **Auto-approve API consumers registering organization**: When this setting is enabled, no approval is required from the portal admins for an API Consumer to register an organization. If this setting is disabled, API Consumer can register organizations, but they won’t be able to invite team members. + +
    This is how it looks in the portal's UI: +Organization registration settings + +
    To proceed with the following steps, enable the Enable API consumers setting to register organizations. + +## Self Registration + +**Steps for Configuration** + +1. **Request org registration** + + Register a developer account or use an existing one and log in to the developer portal as a developer. + To start the organization registration flow, click on the **Create an organization** button in the top right corner of the screen. + Become an organization button + +

    You will be navigated to the screen where you can specify the name of your future organization. + Specify name of the organization + +

    If the **Auto-approve API consumers registering organization** setting is enabled, the new organization will instantly be provisioned. + Organization registration is approved + +

    Otherwise, the developer will have to wait for approval from admin users. + Organization registration is pending + +2. **Approve or reject organization registration requests** + + If the **Auto-approve API consumers registering organization** setting is disabled and the email settings are configured correctly, the admin users will be notified about the new organization registration request via email. + New organization registration request notification + +

    If the **Auto-approve API consumers registering organization** setting is disabled, the new API Consumer organizations won’t be immediately provisioned. + As an admin user, you can approve or reject organization registration requests from the Organization menu. + New organization registration request view + + When admin users approve or reject organization registration requests, the respective email notification is sent to API Consumers. + + Notification when organization request is approved: + Organization registration request is approved + +

    Notification when organization request is rejected: + Organization registration request is rejected + +

    Both emails are customizable. Refer to [the email customization documentation](/portal/customization/email-notifications) for further information on the email customization. + +3. **Invite or remove teammates** + + Once admin users approve the organization registration request, API Consumers can invite teammates. + As an API Consumer, navigate to the Dashboard to invite new teammates. + Navigate to the dashboard + +

    Then select the Users tab in the side menu. + Navigate to the Users tab + +

    You can add a new team member to your API Consumer organization in the Users tab. To invite a new team member, specify their first and last name, email address, and role. + Invite new team member + +

    There are two possible roles for API Consumers: + * Super admin; + * Team member. + + The difference between these two roles is that the Super admins can invite or remove users from their organization and manage applications, while the Team members can only manage applications. + +

    Once the invitation is sent, the invited team member should receive the following email: + Invite new team member email + +

    The invited team member can use the link from the email to register in the portal and join the organization. + Invite new team member email + +4. **Manage API Consumers' role** + + API Consumer Super admins can manage users in their organizations. To do so, navigate to the Users menu in the Dashboard and select a user to edit. + Edit API Consumer profile + +

    As a Super admin, you can change users’ first and last names and roles. The changes will take effect immediately. + Manage API Consumer profile + +5. **Sharing assets between teammates** + + Now, when any team member creates an application, all other team members can access it and use the credentials. + Share credentials between API Consumers + +## Manually Create Organizations + +In this section, you’ll learn how to create a new organization for your external API Consumers. + +**Prerequisites** + +- A Tyk Enterprise portal installation +- A portal admin app login + +**Step by step instructions** + +1. From the **API Consumers > Organizations** menu, click **Add**. + + Portal Organizations menu + Add a new Organization + +2. Enter the name of your new organization + + Add a new Organization + +3. Click **Save** to create your new organization. A new default-team will also automatically created that is tied to your new organization. + + + + + If you want to edit the default team name you can do so by navigating to **Teams**, open up the team associated with the organization you created and edit the name as required. + + + + diff --git a/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/manage-api-consumers.mdx b/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/manage-api-consumers.mdx new file mode 100644 index 00000000..a23fc730 --- /dev/null +++ b/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/manage-api-consumers.mdx @@ -0,0 +1,95 @@ +--- +title: "Manage Users" +'og:description': "How to manage users in Tyk developer portal" +tags: ['Developer Portal', 'Tyk', 'API Consumer', 'Users', 'Self Registration', 'Invite Users', 'Approve Requests'] +sidebarTitle: "Users" +--- + +## Register a New API User + +Developers need to register with the portal to access API Products and manage their applications. This section outlines how developers can register with the portal, access API Products, and reset their passwords if needed. + +There are two ways for registering a new account +1. Self signup via the form. +2. Receive an invitation from the portal admin. + +Here you’ll learn about how to add and invite a new external user to the developer portal. + +**Prerequisites** + +- A Tyk portal installation +- Log in to the portal admin app + +## Self Registration/Signup + +To use the self sign up flow, you’ll need to: +1. Access the Portal and click **REGISTER**. + + Portal login and Register menu + +2. Complete the **Create an Account** form. + + Form to create a developer portal account + +3. Click **Register to developer portal**. +4. If the portal allows signup without approval, you'll get a message that allows you to log in straight away. + + Account registered to allow immediate access to the portal + +5. If the portal requires an admin to approve a registration request, after submitting the **Create an Account** form, you will get the following message. + + Registration account submitted for admin approval + +## Invite a New User + +1. From the **API Consumers > Users** menu Click **Add new user**. + + Portal API Users menu + +2. In the **Add user** dialog, enter **First** and **Last** names, and **Email**. +3. Select an organization to which to register your user. +4. You can also set a password for a user by typing it in the **Set password** field. Check the **User must change password at the next login** if you wish your developer to change their password at next login. + + Please note, that you can either send the invite email or set the password yourself, but you cannot use both methods. + + Add API Users dialog + +5. Click **Save** to add your user. +6. To generate the invite email, click **More Options** in the Overview section and then **Send invite**. + + The user will receive an email with a link to the registration form. This option is only available if you didn't set the password before. + To customize the invite email, please refer to the [Email customization section](/portal/customization/email-notifications) for guidance. + + Users Send invite dialog + +## Approve Self Registering Requests + +## Manual Approval + +This section explains how to approve/reject external users self-registering requests to the developer portal. Follow the step-by-step guide. + +**Prerequisites** + +A Tyk Enterprise portal installation + +**Step by step instructions** + +1. Click *Users* from the **API Consumers** menu + +Portal API Users menu + +2. When a new user has self-registered to access the developer portal, their user profile will be added to the overview in the **Users** section. + +List of Users for your portal app + +3. To approve a user, click on an **inactive** user. Select **Activate developer** from the dialog. + +Select Activate developer + +## Automatically Approve User Registrations + +If you want all Users to be automatically approved this setting can be changed under **Settings > General**. Select **Auto approve developer regestering requests**. + +Setting to automatically approve user registrations + + diff --git a/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/manage-catalogues.mdx b/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/manage-catalogues.mdx new file mode 100644 index 00000000..860094e2 --- /dev/null +++ b/tyk-stack/tyk-developer-portal/enterprise-developer-portal/managing-access/manage-catalogues.mdx @@ -0,0 +1,50 @@ +--- +title: "Manage Catalogs" +'og:description': "How to configure Catalogs in Tyk developer portal" +tags: ['Developer Portal', 'Tyk', 'Managing Access', 'Catalogs'] +sidebarTitle: "Manage Catalogs" +--- + +## Introduction + +Catalogs are a way for you to tailor the audience for API products and Plans. You can, for example create a Partner Catalog, with a specific API product tailored to them and a preferential plan not available in your public portal. + +In this section, you will learn about how catalogs work and how to create a new catalog to expose your API products and plans. + +**Prerequisites** + +- Connect to a provider [Tyk Self-Managed](/portal/overview/getting-started#connect-to-a-provider) +- Create [policies with enforced access rights](/portal/overview/getting-started#create-api-products-and-plans) (API Product in the Portal) +- Create one or more [policies with enforced rate limit and quotas](/portal/overview/getting-started#create-api-products-and-plans) (Plan in the Portal) + +## Create a New Catalog + +1. Navigate to the **Catalog section** section + + Catalogue menu + +2. Click Create a new catalog + + Create a new catalogue + +3. Enter Name and Path URL + + Name the new catalogue + +4. Set the access required for the catalog e.g. Public, Private or Custom + + - Public: External developers can access the catalog + - Private: The catalog is only visible to developers that are logged in + - Custom: Only selected teams can access this catalog + +5. [If creating a custom catalog] Under Audience, select one or multiple teams that you want to have access to this catalog. + + + + + For this audience to apply, the visibility needs to be set to custom. + + + +6. Select the catalog content in terms of which API Products and plans this catalog should contain. + diff --git a/tyk-stack/tyk-gateway/important-prerequisites.mdx b/tyk-stack/tyk-gateway/important-prerequisites.mdx new file mode 100644 index 00000000..7648a7f8 --- /dev/null +++ b/tyk-stack/tyk-gateway/important-prerequisites.mdx @@ -0,0 +1,54 @@ +--- +title: "Useful Configurations when Getting started" +sidebarTitle: "Useful Configurations" +--- + +These are some common settings that you need before proceeding with other parts of our tutorials. + +## Tyk Config + +### Path to Tyk API Definitions configurations directory + +You may need to explicitly define the path in your Tyk config to the directory where you will add +the API definitions for Tyk to serve. + +```yaml +... +"app_path": "/opt/tyk-gateway/apps", +... +``` + +### Path to Policies file + +You need to explicitly set the path to your Policies JSON file in your Tyk config. + +```yaml expandable +... + "policies": { + "policy_source": "file", + "policy_record_name": "policies/policies.json" + }, +... +``` + +### Remove Tyk Dashboard related config options + +Some config options for the Community Edition are not compatible with the Dashboard +version, which requires a license. So, **remove** any section in your Tyk config which +starts with: + +```yaml +... +"db_app_conf_options" { + ... +}, +... +``` + +## Hot reload is critical in Tyk CE + +Each time you add an API definition in Tyk CE, you need to make a hot reload API call as follows: + +```curl +curl -H "x-tyk-authorization: {your-secret}" -s https://{your-tyk-host}:{port}/tyk/reload/group | python -mjson.tool +``` diff --git a/ui-examples/feature-cards.mdx b/ui-examples/feature-cards.mdx new file mode 100644 index 00000000..f8d6d54e --- /dev/null +++ b/ui-examples/feature-cards.mdx @@ -0,0 +1,155 @@ +--- +title: "Feature Cards" +'og:description': "Documentation for the feature-cards shortcode in Tyk. Learn how to display responsive feature cards with icons, titles, and descriptions." +tags: ['UI Examples', 'Shortcodes', 'Feature Cards'] +sidebarTitle: "UI Example: Feature Cards shortcode" +--- + +import UIExampleCards from '/snippets/UIExampleCards.mdx'; + +The `feature-cards` shortcode creates a responsive grid of cards with icons, titles, and descriptions. This is ideal for presenting features or capabilities in a visually appealing way. + +## When to Use + +Use the feature-cards shortcode when you need to display a set of features or capabilities in a visually appealing grid format. It's particularly useful for: + +- Product feature highlights +- Service capabilities +- Key benefits lists +- Team member profiles +- Pricing plan comparisons + +The responsive design ensures your content looks great on all devices, from mobile phones to desktop screens. + +## Usage + +To use the feature cards shortcode in your markdown file, you need to specify a data file: + +```md +{{}} +``` + +## Configuration + +The feature cards are configured through JSON files located in the `data` directory. You must specify which data file to use with the `dataFile` parameter. Here's an example of how to structure a data file: + +```json expandable +{ + "components": [ + { + "title": "Feature Title 1", + "icon": "πŸš€", + "description": "Description of the first feature." + }, + { + "title": "Feature Title 2", + "icon": "βš™οΈ", + "description": "Description of the second feature." + }, + { + "title": "Feature Title 3", + "icon": "πŸ“Š", + "description": "Description of the third feature." + }, + { + "title": "Feature Title 4", + "icon": "πŸ”’", + "description": "Description of the fourth feature." + } + ] +} +``` + +### Data Structure + +Each feature card requires the following fields: + +- `title`: The title of the feature (displayed as a heading) +- `icon`: An emoji or character to use as the feature icon +- `description`: A short description of the feature + +Optionally, you can include: + +- `path`: A URL path for the "Learn More" button +- `button_text`: The text to display on the button (defaults to "Learn More") + +## Responsive Behavior + +The feature cards are fully responsive and will adjust based on screen size: + +- On mobile devices (width < 576px): 1 card per row +- On tablet devices (width 576px - 992px): 2 cards per row +- On desktop devices (width β‰₯ 992px): 4 cards per row + +## Styling + +The cards feature large, centered emoji icons with consistent styling: + +- White background +- Subtle shadow effect +- Rounded corners +- Centered content +- Consistent spacing +- Blue action buttons (optional) + +## Complete Example + +Here's a complete example of how to implement feature cards: + +1. Create or modify a JSON file in the data directory (e.g., `data/my-features.json`): + +```json expandable +{ + "components": [ + { + "title": "Security", + "icon": "πŸ”’", + "description": "Enterprise-grade security with robust authentication options." + }, + { + "title": "Analytics", + "icon": "πŸ“Š", + "description": "Real-time insights into API usage and performance." + }, + { + "title": "Automation", + "icon": "βš™οΈ", + "description": "Automate workflows and reduce manual intervention." + }, + { + "title": "Scalability", + "icon": "πŸš€", + "description": "Scale seamlessly to handle growth and demand spikes." + } + ] +} +``` + +2. Reference this data in the shortcode template (you'll need to modify the shortcode to use your custom data file). + +3. Use the shortcode in your markdown file with the custom data file: + +```md +{{}} +``` + +## Using Multiple Feature Card Sets + +You can use multiple different sets of feature cards in your documentation by: + +1. Creating additional JSON files in the data directory with different names +2. Specifying the appropriate data file when using the shortcode + +This approach allows you to have multiple independent sets of feature cards throughout your documentation. + +## Live Example + +The following example uses a custom data file called `ui-example-features.json`: + +```md +{{}} +``` + +Which renders as: + + diff --git a/ui-examples/test-pill-label-security.mdx b/ui-examples/test-pill-label-security.mdx new file mode 100644 index 00000000..675bb22e --- /dev/null +++ b/ui-examples/test-pill-label-security.mdx @@ -0,0 +1,49 @@ +--- +title: "UI Examples: Pill Label Security Test" +--- + +## Security Test Cases for Pill Label Shortcode + +### Normal Usage +NORMAL + +### HTML Injection Attempt + + +### Invalid Class - Generates "Warning: Unsupported class" Message +INVALID CLASS + +### Malicious Style Attempt - Generates "Warning: Potentially unsafe style attribute" Message +BAD STYLE + +### Additional Unsafe Style Test - Generates "Warning: Potentially unsafe style attribute" Message +SCRIPT IN STYLE + +### Expression Style Test - Generates "Warning: Potentially unsafe style attribute" Message +EXPRESSION + +### Safe Complex Style Attempt +COMPLEX + +## Warning Messages Generated During Build + +The Hugo build log should show these warnings for the malicious test cases: + +1. For invalid class: + ``` + Warning: Unsupported class 'not-allowed-class' used in pill-label shortcode. Using default class instead. + ``` + +2. For unsafe style attributes: + ``` + Warning: Potentially unsafe style attribute in pill-label shortcode was sanitized + ``` + +## IMPORTANT + +This page is set to `draft: true` to prevent it from being built in production, +avoiding warning messages in deployment logs. To test the security features: + +1. Run Hugo locally with the draft flag: `hugo serve -D` +2. Observe the warning messages in the terminal +3. Verify the rendered output in the browser diff --git a/ui-examples/test-pill-label.mdx b/ui-examples/test-pill-label.mdx new file mode 100644 index 00000000..0f4b7047 --- /dev/null +++ b/ui-examples/test-pill-label.mdx @@ -0,0 +1,193 @@ +--- +title: "UI Examples: Pill label shortcode" +sidebarTitle: "UI Example: Pill Label Shortcode" +--- + +This page demonstrates all available styles and usage examples of the pill-label shortcode. + +**Feel free to contribute by adding more classes or adjusting the colors via PR.** + +Click on any example to see details about its usage and styling. + + +--- + +

    Default style - Enterprise edition

    +### Enterprise security features EE + + +Type: Default Style + +Usage: Used for Enterprise Edition features + +Class name: `pill-outline-brandpurple-light` (default) + +This creates a pill with: +- Transparent background +- Dark purple (#8438fa) border +- Dark purple (#8438fa) text + +Example appearance: `[ EE ]` (outline style with purple text) + +Code example: +``` +### Enterprise security features {{}} +``` + + +--- + +

    Dark purple style - Cloud features

    +### Governance CLOUD + + +Type: Dark Purple Style + +Usage: Used for Cloud Features + +Class name: `pill-brandpurple-dark` + +This creates a pill with: +- Dark purple (#8438fa) background +- White text + +Example appearance: `[CLOUD]` (solid purple background with white text) + +Code example: +``` +### Cloud-only feature {{}} +``` + + +--- + +

    Yellow style - Lab releases

    +### AI future feature LAB RELEASE + + +Type: Yellow Style + +Usage: Used for Lab Releases + +Class name: `pill-yellow` + +This creates a pill with: +- Yellow (#d6b218) background +- Black text + +Example appearance: `[LAB RELEASE]` (solid yellow background with black text) + +Code example: +``` +### Lab feature {{}} +``` + + +--- + +

    Red style - Deprecated features

    +### Tyk classic API definition DEPRECATED + + +Type: Red Style + +Usage: Used for Deprecated Features + +Class name: `pill-red` + +This creates a pill with: +- Red (#ff6c7d) background +- White text + +Example appearance: `[DEPRECATED]` (solid red background with white text) + +Code example: +``` +### Tyk classic API definition {{}} +``` + + +--- + +

    Light purple style - Tutorials

    +### Tyk streams getting started TUTORIAL + + +Type: Light Purple Style + +Usage: Used for Tutorials + +Class name: `pill-default` + +This creates a pill with: +- Light purple (#ededf9) background +- Medium purple (#505071) text + +Example appearance: `[TUTORIAL]` (light purple background with dark purple text) + +Code example: +``` +### Tyk streams getting started {{}} +``` + + +--- + +

    Green style - New features

    +### New feature NEW + + +Type: Green Style + +Usage: Used for New Features + +Class name: `pill-brandgreen` + +This creates a pill with: +- Green (#00cdb0) background +- Black text + +Example appearance: `[NEW]` (solid green background with black text) + +Code example: +``` +### New feature {{}} +``` + + +--- + +

    Custom styling

    +### Custom label CUSTOM + + +Type: Custom Styling + +Usage: Used for special cases requiring custom styling + +Uses inline `style` attribute instead of class + +This creates a pill with custom inline styling - in this case: +- Light gray background +- Dark gray text +- Light gray border + +Example appearance: `[CUSTOM]` (custom styling as specified) + +Code example: +``` +### Custom label {{}} +``` + + +--- + +

    Reference table

    +| Class Name | Best For | Background | Text Color | Border | +| :------------ | :---------- | :------------ | :------------ | :-------- | +| pill-outline-brandpurple-light (default) | Enterprise Edition | Transparent | Dark Purple | Dark Purple | +| pill-brandpurple-dark | Cloud Features | Dark Purple | White | None | +| pill-yellow | Lab Releases | Yellow | Black | None | +| pill-red | Deprecated Features | Red | White | None | +| pill-brandgreen | New Features | Green | Black | None | +| pill-default | Tutorials | Light Purple | Medium Purple | None | diff --git a/ui-examples/youtube-video-embed.mdx b/ui-examples/youtube-video-embed.mdx new file mode 100644 index 00000000..d4e0125d --- /dev/null +++ b/ui-examples/youtube-video-embed.mdx @@ -0,0 +1,71 @@ +--- +title: "YouTube Video Embed" +'og:description': "Documentation for the youtube-seo shortcode in Tyk. Learn how to embed YouTube videos in your documentation with proper styling and responsive behavior." +tags: ['UI Examples', 'Shortcodes', 'Video Embed'] +sidebarTitle: "UI Example: YouTube Video Embed shortcode" +--- + +The `youtube-seo` shortcode allows you to easily embed YouTube videos in your documentation with proper styling and responsive behavior. + +## When to Use + +Use the `youtube-seo` shortcode whenever you need to embed a YouTube video in your documentation, such as: + +- Product demonstrations +- Tutorial walkthroughs +- Conference presentations +- Webinar recordings +- Feature explanations + +The shortcode ensures consistent styling and proper SEO attributes for all embedded videos across your documentation. + +## Basic Usage + +To embed a YouTube video, simply use the shortcode with the video ID: + +```md +{{}} +``` + +Replace `VIDEO_ID` with the actual YouTube video ID (the part after `v=` in a YouTube URL) and `Video Title` with a descriptive title for the video. + +## Example + +```md + +``` + +## Parameters + +The shortcode supports several parameters: + +| Parameter | Required | Default | Description | +| :----------- | :---------- | :--------- | :------------- | +| id | Yes | - | The YouTube video ID | +| title | Yes | - | The title of the video (for SEO and accessibility) | +| width | No | 640 | The width of the video in pixels | +| height | No | 360 | The height of the video in pixels | +| allow | No | accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share | Permissions for the embedded iframe | +| frameborder | No | 0 | Border around the iframe | + +## Responsive Behavior + +The embedded video will: +- Be centered on the page +- Maintain its aspect ratio on all screen sizes +- Have appropriate margin spacing above and below +- Appear at a comfortable size (640x360 by default) + +## Advanced Usage + +If you need to customize the video dimensions: + +```md +{{< youtube-seo id="VIDEO_ID" title="Video Title" width="800" height="450" >}} +``` + +If you need to restrict certain iframe permissions: + +```md +{{< youtube-seo id="VIDEO_ID" title="Video Title" allow="accelerometer; encrypted-media" >}} +``` diff --git a/use-cases/open-banking.mdx b/use-cases/open-banking.mdx new file mode 100644 index 00000000..2327191f --- /dev/null +++ b/use-cases/open-banking.mdx @@ -0,0 +1,339 @@ +--- +title: "Tyk FAPI Accelerator for Open Banking" +'og:description': "Learn how Tyk API Gateway enables financial institutions to implement secure, standards-compliant Open Banking APIs with advanced features like DPoP authentication, event notifications, and idempotency support." +sidebarTitle: "Open Banking" +tags: ['open banking', 'financial services', 'FAPI', 'OAuth 2.0', 'DPoP', 'event notifications', 'idempotency', 'JWS', 'security'] +--- + +## Introduction + +Open Banking is a financial services framework that enables secure data sharing between banks and third-party providers (TPPs) through standardized APIs. It fosters innovation, enhances customer experiences, and ensures compliance with regulatory requirements such as: +- **Payment Services Directive 2 (PSD2)** ([PSD2](https://www.ecb.europa.eu/press/intro/mip-online/2018/html/1803_revisedpsd.en.html)) in the European Union, mandating secure access to account information and payment services. +- **UK Open Banking** ([UK Open Banking](https://www.openbanking.org.uk/)), a UK initiative for standardized financial APIs. +- **Consumer Data Right (CDR)** ([CDR](https://www.cdr.gov.au/)) in Australia, enabling consumer data sharing. +- **Financial Data Exchange (FDX)** ([FDX](https://financialdataexchange.org/)) in the United States, an industry-led standard for data sharing. + +Open Banking is enabling the banking industry to move from closed ecosystems to open standards and collaborative platforms where customer data can be securely shared with authorized third parties. However, financial institutions face challenges implementing secure, compliant, Open Banking APIs. This document explores how [Tyk API Gateway](/tyk-oss-gateway), through its [FAPI Accelerator](https://github.com/TykTechnologies/tyk-fapi), enables financial institutions to implement robust Open Banking APIs. + +## Challenges in Financial Services API Implementation + +Implementing Open Banking APIs presents several challenges: + +1. **Stringent Security Requirements**: + - Compliance with **Financial-grade API (FAPI)** standards, such as FAPI 2.0, which ensure high-security API interactions. + - Ensuring data integrity, authenticity, and non-repudiation for sensitive financial transactions. + +2. **Complex Authentication**: + - Implementing **OAuth 2.0** with features like **Pushed Authorization Requests (PAR)** ([PAR](https://datatracker.ietf.org/doc/html/rfc9126)) and **Demonstrating Proof of Possession (DPoP)** ([DPoP](https://datatracker.ietf.org/doc/html/rfc9449)). + - Managing secure user consent and authorization flows. + +3. **Event Notifications**: + - Enabling TPPs to subscribe to real-time updates about account and payment events. + - Signing notifications with cryptographic signatures for security. + +4. **Idempotency**: + - Ensuring that duplicate requests (especially for payments) don't result in duplicate transactions. + +5. **Developer Experience**: + - Providing a seamless experience for TPP developers while maintaining high-security standards. + +## What is Tyk FAPI Accelerator? + +The **[Tyk FAPI Accelerator](https://github.com/TykTechnologies/tyk-fapi)** is a reference implementation provided by Tyk Technologies to help financial institutions build secure, standards-compliant Open Banking APIs. Built on the [Tyk API Gateway](/tyk-oss-gateway). + +### Architecture Overview + +The diagram below shows the Tyk FAPI Accelerator system in its environment, including users, external systems, and key actors. + +```mermaid +flowchart TB + %% People/Actors + psu(["PSU (Payment Services User) + A customer of the bank who accesses their accounts and initiates payments through third-party providers"]) + tpp(["TPP (Third Party Provider) + Companies providing financial services like account aggregators, credit checkers, and savings apps that integrate with banks"]) + aspsp(["ASPSP (Account Servicing Payment Service Provider) + Banks and financial institutions that provide account and payment APIs to authorized third parties"]) + + %% Systems + subgraph tykFAPI ["Tyk FAPI Accelerator"] + tykFAPISystem["Provides FAPI-compliant APIs for account information and payment initiation"] + end + + %% Relationships + psu -->|"Views accounts, + initiates payments"| tykFAPI + tpp -->|"Integrates with, + consumes APIs from"| tykFAPI + aspsp -->|"Configures, monitors, + provides services through"| tykFAPI + + %% Styling + classDef person fill:#335FFD,color:#F7F7FF,stroke:#9393AA + classDef system fill:#00A3A0,color:#F7F7FF,stroke:#03031C + class psu,tpp,aspsp person + class tykFAPISystem system +``` + +### Tyk FAPI Accelerator + +The diagram below shows all major components of the Tyk FAPI Accelerator and their interactions. + +```mermaid +flowchart TB + %% People/Actors + psu(["PSU (Payment Services User) + A customer of the bank who accesses their accounts and initiates payments through third-party providers"]) + tpp(["TPP (Third Party Provider) + Companies providing financial services like account aggregators, credit checkers, and savings apps that integrate with banks"]) + + %% Tyk FAPI Accelerator System with Containers + subgraph tykFAPI ["Tyk FAPI Accelerator"] + tppApp["TPP Application + (NextJS) + Demonstrates how a TPP would interact with a bank's API"] + apiGateway["API Gateway + (Tyk Gateway) + Secures and routes API requests, enforces FAPI compliance, and handles event notifications"] + authServer["Authorization Server + (Keycloak) + Handles authentication and authorization"] + tykBank["Tyk Bank + (Node.js) + Mock bank implementation providing backend services"] + database[(Database)] + databaseLabel["PostgreSQL + Stores account information, payment data, and event subscriptions"] + kafka[(Message Broker)] + kafkaLabel["Kafka + Handles event notifications"] + end + + %% Connect labels to database and kafka + database --- databaseLabel + kafka --- kafkaLabel + + %% Relationships + psu -->|"Uses + (HTTPS)"| tppApp + tpp -->|"Develops + (IDE)"| tppApp + tppApp -->|"Makes API calls to + (HTTPS)"| apiGateway + tppApp -->|"Authenticates with + (OAuth 2.0/OIDC)"| authServer + apiGateway -->|"Routes requests to + (HTTPS)"| tykBank + authServer -->|"Verifies consents with + (HTTPS)"| tykBank + tykBank -->|"Reads from and writes to + (SQL)"| database + tykBank -->|"Publishes events to + (Kafka Protocol)"| kafka + + %% Event notification flow + kafka -->|"Subscribes to events"| apiGateway + apiGateway -->|"Sends signed notifications + (JWS/HTTPS Webhooks)"| tppApp + + %% Styling + classDef person fill:#335FFD,color:#F7F7FF,stroke:#9393AA + classDef tppStyle fill:#335FFD,color:#F7F7FF,stroke:#9393AA + classDef component fill:#00A3A0,color:#F7F7FF,stroke:#03031C + classDef authStyle fill:#00A3A0,color:#F7F7FF,stroke:#03031C + classDef bankStyle fill:#C01FB8,color:#F7F7FF,stroke:#03031C + classDef kafkaStyle fill:#E09D00,color:#F7F7FF,stroke:#03031C + classDef database fill:#5900CB,color:#F7F7FF,stroke:#03031C + classDef label fill:none,stroke:none + + class psu,tpp person + class tppApp tppStyle + class apiGateway component + class authServer authStyle + class tykBank bankStyle + class database database + class kafka kafkaStyle + class databaseLabel,kafkaLabel label +``` + +### Key Components + +1. **API Gateway (Tyk Gateway)**: + - Routes API requests to appropriate backend services + - Implements DPoP authentication via gRPC plugin + - Handles idempotency for payment requests + - Signs and delivers event notifications to TPPs + +2. **Authorization Server (Keycloak)**: + - Provides FAPI 2.0 compliant OAuth 2.0 and OpenID Connect + - Supports Pushed Authorization Requests (PAR) + - Manages user authentication and consent + +3. **Mock Bank Implementation**: + - Implements UK Open Banking Account Information API + - Implements UK Open Banking Payment Initiation API + - Implements UK Open Banking Event Subscriptions API + - Provides realistic testing environment + +4. **TPP Application**: + - Demonstrates how third parties integrate with the bank's APIs + - Implements FAPI 2.0 security profile + - Shows account information retrieval and payment initiation flows + +### Security Features + +The Tyk FAPI Accelerator implements several security features required for financial-grade APIs: + +1. **DPoP (Demonstrating Proof of Possession)**: + - Ensures the client possesses the private key corresponding to the public key in the token + - Prevents token theft and replay attacks + - Implemented as a gRPC plugin for Tyk Gateway + +2. **JWS Signing for Event Notifications**: + - Signs webhook notifications with JSON Web Signatures (JWS) + - Ensures authenticity and integrity of notifications + - Allows TPPs to verify the source of notifications + +3. **Idempotency Support**: + - Prevents duplicate transactions from repeated API calls + - Caches responses for idempotent requests + - Includes automatic garbage collection of expired entries + +4. **OAuth 2.0 with PAR**: + - Implements Pushed Authorization Requests for enhanced security + - Supports both automatic and manual authorization flows + - Complies with FAPI 2.0 security profile + + +## Getting Started + +For detailed setup instructions, code examples, and deployment guides, please refer to the [Tyk FAPI Accelerator GitHub repository](https://github.com/TykTechnologies/tyk-fapi/tree/main?tab=readme-ov-file#getting-started). + + +## Implementation Examples + +### Payment Flow Example + +The following sequence diagram illustrates a typical payment flow in the Tyk FAPI Accelerator: + +```mermaid +sequenceDiagram + actor User as End User + participant TPP as TPP Application + participant Gateway as API Gateway + participant Auth as Authorization Server + participant Bank as Tyk Bank + participant DB as Database + participant Kafka as Message Broker + + %% Payment Initiation + User->>TPP: 1. Initiate payment (amount, recipient) + + %% Payment Consent Creation + TPP->>Gateway: 2. Create payment consent + Gateway->>Bank: 3. Forward consent request + Bank->>DB: 4. Store consent + DB-->>Bank: 5. Return consent ID + Bank-->>Gateway: 6. Consent response with ConsentId + Gateway-->>TPP: 7. Return ConsentId + + %% Pushed Authorization Request (PAR) + TPP->>Auth: 8. Push Authorization Request (PAR) + Note right of TPP: Direct connection to Auth Server + Auth-->>TPP: 9. Return request_uri + + %% Authorization Options + TPP->>User: 10. Display authorization options + + %% Two possible authorization flows + alt Automatic Authorization + User->>TPP: 11a. Select automatic authorization + TPP->>Bank: 12a. Direct authorize consent request + Note right of TPP: Server-side authorization + Bank->>DB: 13a. Update consent status + DB-->>Bank: 14a. Confirm update + Bank-->>TPP: 15a. Authorization confirmation + else Manual Authorization + User->>TPP: 11b. Select manual authorization + TPP->>User: 12b. Redirect to authorization URL + User->>Auth: 13b. Authorization request with request_uri + Auth->>Bank: 14b. Verify consent + Bank->>DB: 15b. Get consent details + DB-->>Bank: 16b. Return consent details + Bank-->>Auth: 17b. Consent details + Auth->>User: 18b. Display authorization UI + User->>Auth: 19b. Approve authorization + Auth->>DB: 20b. Update consent status + DB-->>Auth: 21b. Confirm update + Auth->>User: 22b. Redirect to callback URL with code + User->>TPP: 23b. Callback with authorization code + end + + %% Payment Creation + TPP->>Gateway: 24. Create payment with authorized consent + Gateway->>Bank: 25. Forward payment request + Bank->>DB: 26. Store payment + DB-->>Bank: 27. Return payment ID + Bank-->>Gateway: 28. Payment response with PaymentId + Gateway-->>TPP: 29. Return PaymentId + + %% Payment Confirmation + TPP->>User: 30. Display payment confirmation + + %% Event Notification + Bank->>Kafka: 31. Publish payment event + Kafka->>Bank: 32. Stream processor consumes event + Bank->>DB: 33. Query subscriptions + DB-->>Bank: 34. Return matching subscriptions + Bank->>TPP: 35. Send payment notification + TPP-->>Bank: 36. Acknowledge notification +``` + +### Event Notification Example + +The event notification system allows TPPs to receive updates about payment status changes: + +```mermaid +sequenceDiagram + actor TPP as TPP Application + participant Gateway as API Gateway + participant EventAPI as Event Subscriptions API + participant PaymentAPI as Payment Initiation API + participant DB as Database + participant Kafka as Message Broker + + %% Subscription Registration + TPP->>Gateway: 1. Register callback URL + Gateway->>EventAPI: 2. Forward registration request + EventAPI->>DB: 3. Store subscription + DB-->>EventAPI: 4. Return subscription ID + EventAPI-->>Gateway: 5. Registration response with SubscriptionId + Gateway-->>TPP: 6. Return SubscriptionId + + %% Event Generation + Note over PaymentAPI: Payment status change or other event occurs + PaymentAPI->>Kafka: 7. Publish event + Note right of PaymentAPI: Event includes type, subject, timestamp + + %% Event Processing + Kafka-->>Gateway: 8. Consume event + Gateway->>DB: 9. Query subscriptions for event type + DB-->>Gateway: 10. Return matching subscriptions + Gateway->>Gateway: 11. Determine target TPPs and sign with JWS + + %% Notification Delivery + Gateway->>TPP: 12. Send signed notification + Note right of Gateway: Notification includes event details, links, and JWS signature + TPP->>TPP: 13. Verify JWS signature + TPP-->>Gateway: 14. Acknowledge (HTTP 200 OK) + + %% Error Handling (Alternative Flow) + alt Delivery Failure + Gateway->>TPP: 12. Send signed notification + TPP--xGateway: 13. Failed delivery (timeout/error) + Gateway->>Gateway: 14. Retry with exponential backoff + Gateway->>TPP: 15. Retry notification + TPP->>TPP: 16. Verify JWS signature + TPP-->>Gateway: 17. Acknowledge (HTTP 200 OK) + end +```