package Amazon::API;

# Generic interface to Amazon APIs (powered by botocore)

use strict;
use warnings;

use 5.010;

our %LOG4PERL_LOG_LEVELS;

BEGIN {
  %LOG4PERL_LOG_LEVELS = eval {
    require Log::Log4perl::Level;

    Log::Log4perl::Level->import();

    use vars qw($ERROR $WARN $INFO $DEBUG $TRACE $FATAL);

    return (
      error => $ERROR,
      warn  => $WARN,
      info  => $INFO,
      debug => $DEBUG,
      trace => $TRACE,
      fatal => $FATAL,
    );
  };
}

use parent qw( Exporter Class::Accessor::Fast);

use Amazon::API::Error;
use Amazon::API::Signature4;
use Amazon::API::Constants qw( :all );
use Amazon::Credentials;
use Amazon::API::Botocore qw(paginator);
use Amazon::API::Botocore::Shape::Serializer;
use Amazon::API::Botocore::Shape::Utils qw(
  require_class
  require_shape
  create_module_name
  get_service_from_class
);

use Carp;
use Carp::Always;
use Data::Dumper;
use Date::Format;
use English qw( -no_match_vars);
use HTTP::Request;
use JSON qw( encode_json decode_json );
use LWP::UserAgent;
use List::Util qw(any all pairs none);
use Readonly;
use Scalar::Util qw( reftype blessed );
use Time::Local;
use Time::HiRes qw( gettimeofday tv_interval );
use URL::Encode qw(url_encode);
use XML::LibXML;
use XML::Simple;

Readonly::Scalar my $DEFAULT_REGION   => 'us-east-1';
Readonly::Scalar my $REGIONAL_URL_FMT => '%s://%s.%s.amazonaws.com';
Readonly::Scalar my $GLOBAL_URL_FMT   => '%s://%s.amazonaws.com';

Readonly::Scalar my $DEFAULT_LAYOUT_PATTERN => '[%d] %p (%R/%r) %M:%L - %m%n';

local $Data::Dumper::Pair  = $COLON;
local $Data::Dumper::Useqq = $TRUE;
local $Data::Dumper::Terse = $TRUE;

__PACKAGE__->follow_best_practice;

__PACKAGE__->mk_accessors(
  qw(
    action
    api
    api_methods
    botocore_metadata
    botocore_operations
    botocore_shapes
    content_type
    credentials
    debug
    decode_always
    endpoint_prefix
    error
    force_array
    http_method
    last_action
    _log_level
    logger
    log_file
    log_layout
    namespace
    paginators
    print_error
    protocol
    raise_error
    raise_serialization_errors
    region
    request_uri
    response
    service
    serializer
    service_url_base
    target
    target_prefix
    url
    use_botocore
    user_agent
    use_paginator
    version
  )
);

our $VERSION = '2.1.4';  ## no critic (RequireInterpolationOfMetachars)

our @EXPORT_OK = qw(
  create_urlencoded_content
  get_api_service
  param_n
  paginator
  service
  generate_xml
);

our $START_TIME    = [gettimeofday];
our $LAST_LOG_TIME = [gettimeofday];

=begin 'ignore'

The service categories were extracted from Botocore project:

grep -ri '"protocol":"' botocore/botocore/data/* | grep 'xml' | \
  cut -f 8 -d '/' | sort -u > xml.services

grep -ri '"protocol":"' botocore/botocore/data/* | grep 'json' | \
  cut -f 8 -d '/' | sort -u > json.services

grep -ri '"protocol":"' botocore/botocore/data/* | grep 'query' | \
  cut -f 8 -d '/' | sort -u > query.services

These service categories are used to deduce content type for
parameters sent to methods for these APIs when Botocore metadata was
not used to create an API class. Content-Type can however be
overridden when invoking APIs if we guess wrong.

 rest-json => application/x-amz-1.1
 rest-xml  => application/xml
 query     => application/x-www-form-urlencoded

=end 'ignore'

=cut

our %SERVICE_CONTENT_TYPES = (
  ec2         => 'application/x-www-form-urlencoded',
  query       => 'application/x-www-form-urlencoded',
  'rest-json' => 'application/json',
  json        => 'application/x-amz-json',
  'rest-xml'  => 'application/xml',
);

our %API_TYPES = (
  query => [
    qw(
      autoscaling
      cloudformation
      cloudsearch
      cloudwatch
      docdb
      ec2
      elasticache
      elasticbeanstalk
      elb
      elbv2
      iam
      importexport
      neptune
      rds
      redshift
      sdb
      ses
      sns
      sts
    )
  ],
  xml => [
    qw(
      cloudfront
      route53
      s3
      s3control
    )
  ],
  json => [
    qw(
      accessanalyzer
      account
      acm
      acm-pca
      amp
      amplify
      amplifybackend
      amplifyuibuilder
      apigateway
      apigatewaymanagementapi
      apigatewayv2
      appconfig
      appconfigdata
      appfabric
      appflow
      appintegrations
      application-autoscaling
      application-insights
      application-signals
      applicationcostprofiler
      appmesh
      apprunner
      appstream
      appsync
      apptest
      arc-zonal-shift
      artifact
      athena
      auditmanager
      autoscaling-plans
      b2bi
      backup
      backup-gateway
      backupsearch
      batch
      bcm-data-exports
      bcm-pricing-calculator
      bedrock
      bedrock-agent
      bedrock-agent-runtime
      bedrock-data-automation
      bedrock-data-automation-runtime
      bedrock-runtime
      billing
      billingconductor
      braket
      budgets
      ce
      chatbot
      chime
      chime-sdk-identity
      chime-sdk-media-pipelines
      chime-sdk-meetings
      chime-sdk-messaging
      chime-sdk-voice
      cleanrooms
      cleanroomsml
      cloud9
      cloudcontrol
      clouddirectory
      cloudfront-keyvaluestore
      cloudhsm
      cloudhsmv2
      cloudsearchdomain
      cloudtrail
      cloudtrail-data
      codeartifact
      codebuild
      codecatalyst
      codecommit
      codeconnections
      codedeploy
      codeguru-reviewer
      codeguru-security
      codeguruprofiler
      codepipeline
      codestar-connections
      codestar-notifications
      cognito-identity
      cognito-idp
      cognito-sync
      comprehend
      comprehendmedical
      compute-optimizer
      config
      connect
      connect-contact-lens
      connectcampaigns
      connectcampaignsv2
      connectcases
      connectparticipant
      controlcatalog
      controltower
      cost-optimization-hub
      cur
      customer-profiles
      databrew
      dataexchange
      datapipeline
      datasync
      datazone
      dax
      deadline
      detective
      devicefarm
      devops-guru
      directconnect
      discovery
      dlm
      dms
      docdb-elastic
      drs
      ds
      ds-data
      dsql
      dynamodb
      dynamodbstreams
      ebs
      ec2-instance-connect
      ecr
      ecr-public
      ecs
      efs
      eks
      eks-auth
      elastictranscoder
      emr
      emr-containers
      emr-serverless
      entityresolution
      es
      events
      evidently
      finspace
      finspace-data
      firehose
      fis
      fms
      forecast
      forecastquery
      frauddetector
      freetier
      fsx
      gamelift
      gameliftstreams
      geo-maps
      geo-places
      geo-routes
      glacier
      globalaccelerator
      glue
      grafana
      greengrass
      greengrassv2
      groundstation
      guardduty
      health
      healthlake
      identitystore
      imagebuilder
      inspector
      inspector-scan
      inspector2
      internetmonitor
      invoicing
      iot
      iot-data
      iot-jobs-data
      iot-managed-integrations
      iotanalytics
      iotdeviceadvisor
      iotevents
      iotevents-data
      iotfleethub
      iotfleetwise
      iotsecuretunneling
      iotsitewise
      iotthingsgraph
      iottwinmaker
      iotwireless
      ivs
      ivs-realtime
      ivschat
      kafka
      kafkaconnect
      kendra
      kendra-ranking
      keyspaces
      kinesis
      kinesis-video-archived-media
      kinesis-video-media
      kinesis-video-signaling
      kinesis-video-webrtc-storage
      kinesisanalytics
      kinesisanalyticsv2
      kinesisvideo
      kms
      lakeformation
      lambda
      launch-wizard
      lex-models
      lex-runtime
      lexv2-models
      lexv2-runtime
      license-manager
      license-manager-linux-subscriptions
      license-manager-user-subscriptions
      lightsail
      location
      logs
      lookoutequipment
      lookoutmetrics
      lookoutvision
      m2
      machinelearning
      macie2
      mailmanager
      managedblockchain
      managedblockchain-query
      marketplace-agreement
      marketplace-catalog
      marketplace-deployment
      marketplace-entitlement
      marketplace-reporting
      marketplacecommerceanalytics
      mediaconnect
      mediaconvert
      medialive
      mediapackage
      mediapackage-vod
      mediapackagev2
      mediastore
      mediastore-data
      mediatailor
      medical-imaging
      memorydb
      meteringmarketplace
      mgh
      mgn
      migration-hub-refactor-spaces
      migrationhub-config
      migrationhuborchestrator
      migrationhubstrategy
      mq
      mturk
      mwaa
      neptune-graph
      neptunedata
      network-firewall
      networkflowmonitor
      networkmanager
      networkmonitor
      notifications
      notificationscontacts
      oam
      observabilityadmin
      omics
      opensearch
      opensearchserverless
      opsworks
      opsworkscm
      organizations
      osis
      outposts
      panorama
      partnercentral-selling
      payment-cryptography
      payment-cryptography-data
      pca-connector-ad
      pca-connector-scep
      pcs
      personalize
      personalize-events
      personalize-runtime
      pi
      pinpoint
      pinpoint-email
      pinpoint-sms-voice
      pinpoint-sms-voice-v2
      pipes
      polly
      pricing
      privatenetworks
      proton
      qapps
      qbusiness
      qconnect
      qldb
      qldb-session
      quicksight
      ram
      rbin
      rds-data
      redshift-data
      redshift-serverless
      rekognition
      repostspace
      resiliencehub
      resource-explorer-2
      resource-groups
      resourcegroupstaggingapi
      robomaker
      rolesanywhere
      route53-recovery-cluster
      route53-recovery-control-config
      route53-recovery-readiness
      route53domains
      route53profiles
      route53resolver
      rum
      s3outposts
      s3tables
      sagemaker
      sagemaker-a2i-runtime
      sagemaker-edge
      sagemaker-featurestore-runtime
      sagemaker-geospatial
      sagemaker-metrics
      sagemaker-runtime
      savingsplans
      scheduler
      schemas
      secretsmanager
      security-ir
      securityhub
      securitylake
      serverlessrepo
      service-quotas
      servicecatalog
      servicecatalog-appregistry
      servicediscovery
      sesv2
      shield
      signer
      simspaceweaver
      sms
      sms-voice
      snow-device-management
      snowball
      socialmessaging
      sqs
      ssm
      ssm-contacts
      ssm-incidents
      ssm-quicksetup
      ssm-sap
      sso
      sso-admin
      sso-oidc
      stepfunctions
      storagegateway
      supplychain
      support
      support-app
      swf
      synthetics
      taxsettings
      textract
      timestream-influxdb
      timestream-query
      timestream-write
      tnb
      transcribe
      transfer
      translate
      trustedadvisor
      verifiedpermissions
      voice-id
      vpc-lattice
      waf
      waf-regional
      wafv2
      wellarchitected
      wisdom
      workdocs
      workmail
      workmailmessageflow
      workspaces
      workspaces-thin-client
      workspaces-web
      xray
    )
  ],
);

our @GLOBAL_SERVICES = qw(
  cloudfront
  iam
  importexport
  route53
  s3
  savingsplans
  sts
);

our @REQUIRED_KEYS = qw( aws_access_key_id aws_secret_access_key );

########################################################################
sub new {
########################################################################
  my ( $class, @options ) = @_;

  $class = ref $class || $class;

  my %options = ref $options[0] ? %{ $options[0] } : @options;

  my $log_level = delete $options{log_level};
  $log_level //= 'error';

  my $self = $class->SUPER::new( \%options );

  if ( $self->get_service_url_base ) {
    $self->set_service( $self->get_service_url_base );
  }

  croak 'service is required'
    if !$self->get_service;

  $self->_set_default_logger($log_level);

  $self->_set_defaults(%options);

  $self->init_log_level( log_level => $log_level, debug => $options{debug} );

  $self->_create_methods;

  if ( $self->is_botocore_api ) {
    $self->set_serializer(
      Amazon::API::Botocore::Shape::Serializer->new( service => get_service_from_class( ref $self ) ) );
  }

  return $self;
}

########################################################################
sub set_log_level {
########################################################################
  my ( $self, $log_level ) = @_;

  $self->set__log_level($log_level);

  if ( Log::Log4perl->initialized() && $self->get_logger ) {
    $self->get_logger->level( $LOG4PERL_LOG_LEVELS{$log_level} );
  }
  else {
    my $logger = $self->get_logger;

    if ( $logger->can('level') ) {
      $logger->level($log_level);
    }
  }

  return $self;
}

########################################################################
sub get_log_level {
########################################################################
  my ($self) = @_;

  return $self->get__log_level();
}

########################################################################
sub service { goto &get_api_service; }
########################################################################

########################################################################
sub get_api_service {
########################################################################
  my ( $service_name, @args ) = @_;

  $service_name = create_module_name $service_name;

  my $class = sprintf 'Amazon::API::%s', $service_name;

  require_class $class;

  return $class->new(@args);
}

########################################################################
sub decode_response {
########################################################################
  my ( $self, $response, $serialize ) = @_;

  $serialize //= $TRUE;

  $response = $response || $self->get_response;

  # could be Furl or HTTP?
  if ( !ref $response && $response->can('content') ) {
    croak q{can't decode response - not a response object: } . ref $response;
  }

  my $content = $response->content;

  my $content_type = $response->content_type;

  # this is expected to fail if !$self->is_botocore_api
  my ( $protocol, $botocore_action )
    = eval { return ( $self->get_botocore_metadata->{protocol}, $self->get_botocore_action ); };

  TRACE(
    sub {
      return Dumper(
        [ protocol         => $protocol,
          response_content => $content
        ]
      );
    }
  );

  my $decoded_content;

  if ($content) {

    $decoded_content = eval {
      if ( $content_type =~ /json/xmsi ) {
        decode_json($content);
      }
      elsif ( $content_type =~ /xml/xmsi ) {
        ################################################################
        # Maddening interpretation of XML output from these ass hats (-:|3
        #
        # A little more explanation: some rest-xml and ec2 protocol
        # API responses have wrapper elements that are not part of the
        # shape description...some do. It probably would have been
        # best not to try to serialize this XML object here, but
        # rather downstream when trying to create shapes from
        # it. However, the serializer now tries to figure out whether
        # to disgard the wrapper or keep it. The botocore metadata
        # could have offered some clues, however I believe the
        # serializers in Botocore have been tuned to specific APIs in
        # some cases making it difficult to fix legacy mistakes in
        # these APIs.
        ################################################################
        XMLin(
          $content,
          KeepRoot      => $TRUE,
          SuppressEmpty => $FALSE,
          ForceContent  => $self->is_botocore_api && $botocore_action->{output},
          ForceArray    => ['item'],
          #          $self->get_force_array ? ( ForceArray => ['item'] ) : ()
        );
      }
    };

    # disregard content_type (it might be misleading?)
    # this is almost certainly going to be a problem somewhere...
    if ( !$decoded_content || $EVAL_ERROR ) {

      WARN("unable to decode content: $EVAL_ERROR");
      WARN('Trying again using JSON and XML decoders.');

      $decoded_content = eval { return decode_json($content); };

      if ( !$decoded_content || $EVAL_ERROR ) {
        $decoded_content = eval {
          return XMLin(
            $content,
            SuppressEmpty => 0,  # $protocol eq 'ec2' ? $FALSE : $TRUE,
            $self->get_force_array ? ( ForceArray => ['item'] ) : ()
          );
        };
      }
    }
  }

  $content = $decoded_content || $content;

  DEBUG( sub { return Dumper( [ 'content' => $content ] ) } );

  # we'll only have a "serializer" if this is a Botocore generated API
  my $serializer = $self->get_serializer;

  return $content
    if !ref $content || !$serializer || !$serialize;

  my $output = $botocore_action->{output};

  return $decoded_content
    if !$output;

  my $orig_content = $content;

  if ( $output->{resultWrapper} ) {
    $content = $content->{ $output->{resultWrapper} };
  }

  DEBUG(
    sub {
      return Dumper(
        [ content                => $content,
          botocore_action_output => $output
        ]
      );
    }
  );

  $serializer->set_logger( $self->get_logger );

  $content = eval {
    $serializer->serialize(
      service => get_service_from_class( ref $self ),
      shape   => $output->{shape},
      data    => $content
    );
  };

  # ...but this isn't necessarily where things STB
  if ( !$content || $EVAL_ERROR ) {
    if ( $self->get_raise_serialization_errors ) {
      die $EVAL_ERROR;
    }
    elsif ($EVAL_ERROR) {
      carp "error serializing content: please report this error\n$EVAL_ERROR";
      $content = $orig_content;
    }
  }

  return $content;
}

########################################################################
sub get_botocore_action {
########################################################################
  my ($self) = @_;

  return $self->get_botocore_operations->{ $self->get_action };
}

########################################################################
sub is_botocore_shape {
########################################################################
  my ($request) = @_;

  my $shape_name = ref $request;

  if ( $shape_name =~ /Botocore::Shape::([^:]+)::([^:]+)$/xsm ) {
    $shape_name = $2;
  }
  else {
    $shape_name = undef;
  }

  return $shape_name;
}

# returns the 'locationName' of the element or '' locationName for a
# given parameter type (uri, querystring, etc)
#
# determines where to find the parameter in the input payload

########################################################################
sub is_param_type {
########################################################################
  my ( $self, $shape_name, $param, $type ) = @_;

  my $members = $self->get_botocore_shapes->{$shape_name}->{members};
  my $member  = $members->{$param};

  my $location = $member->{location};

  TRACE(
    sub {
      return Dumper(
        [ members    => $members,
          member     => $member,
          location   => $location,
          param      => $param,
          type       => $type,
          shape_name => $shape_name
        ]
      );
    }
  );

  return ( $location && $location eq $type ) ? $member->{locationName} : $EMPTY;
}

########################################################################
sub is_query_param {
########################################################################
  my ( $self, $shape_name, $param ) = @_;

  return $self->is_param_type( $shape_name, $param, 'querystring' );
}

########################################################################
sub is_uri_param {
########################################################################
  my ( $self, $shape_name, $param ) = @_;

  return $self->is_param_type( $shape_name, $param, 'uri' );
}

########################################################################
sub create_botocore_request {
########################################################################
  my ( $self, %args ) = @_;

  my ( $parameters, $action ) = @args{qw(parameters action)};

  $action //= $self->get_action;

  croak 'no action'
    if !$action;

  croak 'no parameters'
    if !$parameters;

  croak 'not a botocore API'
    if !$self->is_botocore_api($action);

  my $botocore_operations = $self->get_botocore_operations->{$action};

  my $input = $botocore_operations->{input};
  my $shape = $input->{shape};

  my $class = require_shape( $shape, get_service_from_class($self) );

  croak "could not create request shape: $shape\n$EVAL_ERROR"
    if !$class;

  my $request = $class->new($parameters);  # request class

  return $request;
}

########################################################################
# init_botocore_request( $self, $request)
########################################################################

# This function will accept either an object which is a sub-class of
# Amazon::API::Botocore::Shape, or a hash if the parameters have been
# constructed "by-hand",
#
# The parameters are used to populate both the URL if some parameters
# are passed in the URL and either a JSON or XML payload depending on
# the API type (rest-json, rest-xml).

########################################################################
sub find_content_type {
########################################################################
  my ($self) = @_;
  my $protocol = $self->get_botocore_metadata->{protocol};

  my $content_type = $SERVICE_CONTENT_TYPES{$protocol};

  if ( $protocol eq 'json' ) {
    $content_type = sprintf '%s-%s', $content_type, $self->get_botocore_metadata->{jsonVersion};
  }
  return $content_type;
}

########################################################################
sub init_botocore_request {
########################################################################
  my ( $self, $request ) = @_;

  my $metadata = $self->get_botocore_metadata;

  my $protocol = $metadata->{protocol};

  my $content_type = $self->find_content_type() // $EMPTY;
  $self->set_content_type($content_type);

  $request //= {};

  my $action = $self->get_action;

  my $botocore_operations = $self->get_botocore_operations->{$action};

  my $http = $botocore_operations->{http};

  my $method = $http->{method};

  TRACE(
    sub {
      return Dumper(
        [ request             => $request,
          protocol            => $protocol,
          action              => $action,
          method              => $method,
          content_type        => $content_type,
          botocore_metadata   => $metadata,
          botocore_operations => $botocore_operations,
        ]
      );
    }
  );

  my $input = $botocore_operations->{input};

  my $shape = $input->{shape};

  my $request_shape_name = is_botocore_shape($request);

  # if a shape object is passed, it must be the correct type
  croak "$action requires a $shape object, not a $request_shape_name object"
    if $request_shape_name && $request_shape_name ne $shape;

  # try to create a Botocore request shape
  my $boto_request;

  if ( !$request_shape_name && $self->is_botocore_api ) {

    $boto_request = $self->create_botocore_request( parameters => $request );

    if ( !$boto_request ) {
      croak "could not create a botocore request object\n$EVAL_ERROR\n";
    }
    else {
      $request_shape_name = is_botocore_shape($boto_request);

      $request = $boto_request;
    }
  }

  # some XML requests require an xmlns attribute
  $self->find_namespace( $request, $input );

  my %parameters;

  # is the request a Botocore::Shape object? if so we can use metadata
  # to create URI and payload, otherwise it's up to the caller to make
  # sure the URI and the payload are correct...good luck!

  if ( !$request_shape_name ) {
    %parameters = %{$request};
  }
  else {
    my $finalized_request = $request->finalize;

    DEBUG(
      sub {
        return Dumper(
          [ request           => $request,
            finalized_request => $finalized_request,
          ]
        );
      }
    );

    if ( $protocol =~ /rest\-(xml|json)/xsm ) {
      $finalized_request = { $request_shape_name => $finalized_request };
    }
    elsif ( $protocol eq 'ec2' ) {
      $finalized_request = [ param_n($finalized_request) ];

      $self->set_http_method($method);

      return $finalized_request;
    }

    DEBUG(
      sub {
        return Dumper( [ finalized_request => $finalized_request, ] );
      }
    );

    if ( ref($finalized_request) eq 'HASH' ) {
      %parameters = %{$finalized_request};
    }
  }

  $self->set_http_method($method);

  my $uri;

  if ( $protocol =~ /^rest\-(json|xml)/xsm ) {
    my @args = @{ $http->{parsed_request_uri}->{parameters} };

    my $request_uri_tpl = $http->{parsed_request_uri}->{request_uri_tpl};

    DEBUG(
      sub {
        return Dumper(
          [ args               => \@args,
            request_uri_tpl    => $request_uri_tpl,
            request_shape_name => $request_shape_name,
            input              => $input,
          ]
        );
      }
    );

    # if the request is a shape, we've already checked for required
    # parameters but some may be buried in the payload!
    if ( !$request_shape_name ) {
      foreach my $p (@args) {
        croak 'required parameter ' . $p . ' not found.'
          if !exists $parameters{$p};
      }

      $uri = sprintf $request_uri_tpl, @parameters{@args};
      $self->set_request_uri($uri);

      delete @parameters{@args};
    }
    else {
      $uri = $http->{requestUri};  # use the Botocore template

      my $shape_parameters = $parameters{$shape};

      DEBUG(
        sub {
          return Dumper(
            [ requestUri       => $uri,
              shape_parameters => $shape_parameters,
              shape            => $shape,
              parameters       => \%parameters,
            ]
          );
        }
      );

      foreach my $p ( keys %{$shape_parameters} ) {
        TRACE( Dumper( [ parameter => $p ] ) );

        if ( my $var = $self->is_uri_param( $request_shape_name, $p ) ) {

          my $val = $shape_parameters->{$p};

          TRACE(
            Dumper(
              [ var => $var,
                val => $val,
              ]
            )
          );

          $uri =~ s/[{]$var[}]/$val/xsm;

          delete $shape_parameters->{$p};
        }
      }

      # we're not done yet...just to make things interesting, some
      # APIs embed request parameters in the uri, payload AND query
      # string!

      if ($request_shape_name) {
        my %query_parameters;

        my $shape_parameters = $parameters{$shape};

        foreach my $p ( keys %{$shape_parameters} ) {
          if ( my $var = $self->is_query_param( $request_shape_name, $p ) ) {
            $query_parameters{$var} = $shape_parameters->{$p};
            delete $shape_parameters->{$p};
          }
        }

        if ( keys %query_parameters ) {
          # $self->set_content_type(undef);

          $uri = sprintf '%s?%s', $uri, create_urlencoded_content( \%query_parameters );
        }

        if ( !keys %{$shape_parameters} ) {
          %parameters = ();
        }
      }
    }

    $self->set_request_uri($uri);

    # payload (I think) tells us where the actual object to pass will
    # be found in the request object. Since we may be formatting an
    # XML request we need to insert the namespace so our formatter can
    # properly serialize the request
    if ( $request->{payload} && $parameters{$request_shape_name}->{ $request->{payload} } ) {
      %parameters = %{ $parameters{$request_shape_name} };
      if ( $self->get_namespace ) {
        $parameters{ $request->{payload} }->{_attr} = { xmlns => $self->get_namespace };
      }
    }
    elsif ( $self->get_namespace ) {
      my $locationName = $input->{locationName};
      $parameters{$locationName}->{_attr} = { xmlns => $self->get_namespace };
    }

  }

  my $content = \%parameters;

  if ( $method ne 'POST' && !keys %parameters ) {
    $content = undef;
  }

  DEBUG(
    sub {
      return Dumper [
        namespace   => $self->get_namespace,
        parameters  => \%parameters,
        request_uri => $uri,
        content     => $content,
      ];
    }
  );

  return $content;
}

########################################################################
# namespaces may be buried a bit in the members specfication of
# request objects or can be part of the specification for the request
# method itself (cloudfront.CreateInvalidationBatch)
########################################################################
sub find_namespace {
########################################################################
  my ( $self, $request, $input ) = @_;

  my ($has_namespace) = grep { defined $_ }
    map { $request->{members}->{$_}->{xmlNamespace}->{uri} } keys %{ $request->{members} // {} };

  if ( !$has_namespace && $input->{xmlNamespace} ) {
    $has_namespace = $input->{xmlNamespace}->{uri};
  }

  $self->set_namespace($has_namespace);

  return;
}

########################################################################
sub is_botocore_api {
########################################################################
  my ($self) = @_;

  return defined $self->get_botocore_metadata;
}

########################################################################
sub serialize_content {
########################################################################
  my ( $self, $parameters ) = @_;

  my $content      = $parameters;
  my $action       = $self->get_action;
  my $version      = $self->get_version;
  my $content_type = $self->get_content_type;

  TRACE(
    sub {
      return Dumper(
        [ content_type => $content_type,
          parameters   => $parameters,
          service      => $self->get_service,
        ]
      );
    }
  );

  # if the API is a query API, url encode parameters
  if ( any { $_ eq lc $self->get_service } @{ $API_TYPES{query} } ) {
    $parameters //= [];
    $content = create_urlencoded_content( $parameters, $action, $version );
  }
  elsif ( $parameters && ref $parameters && reftype($parameters) eq 'HASH' ) {
    if ( $content_type =~ /json/xsm ) {
      delete $parameters->{_attr};
      $content = encode_json($parameters);
    }
    elsif ( $content_type =~ /xml/xms ) {
      return
        if !ref $content || !keys %{$content};

      $content = generate_xml($parameters);
    }
  }

  return $content;
}

########################################################################
# invoke_api( action, parameters, content-type, headers)
########################################################################
sub invoke_api {
########################################################################
  my ( $self, @args ) = @_;

  DEBUG(
    sub {
      return Dumper( [ args => \@args ] );
    }
  );

  my ( $action, $parameters, $content_type, $headers );

  # use_botocore overrides the creation of a Botocore request
  # object. In that case the arguments are expected to conform to the
  # format specified for this particular AWS API.

  if ( ref $args[0] && reftype( $args[0] ) eq 'HASH' ) {
    ( $action, $parameters, $content_type, $headers )
      = @{ $args[0] }{qw( action parameters content_type headers)};
  }
  else {
    ( $action, $parameters, $content_type, $headers ) = @args;
  }

  my $use_botocore;

  if ( $parameters && !ref $parameters ) {
    $use_botocore = $FALSE;
  }
  elsif ( $parameters && reftype($parameters) ne 'HASH' ) {
    $use_botocore = $FALSE;
  }
  else {
    $use_botocore = $self->is_botocore_api;
  }

  my $protocol = eval { $self->get_botocore_metadata->{protocol}; };

  DEBUG(
    sub {
      return Dumper(
        [ parameters      => $parameters,
          'content-type'  => $content_type,
          protocol        => $protocol,
          use_botocore    => $use_botocore,
          is_botocore_api => $self->is_botocore_api,
        ]
      );
    }
  );

  $self->set_action($action);
  $self->set_last_action($action);
  $self->set_error(undef);

  my $decode_response = $self->get_decode_always;

  # someone is trying to send a blessed object, possibly a Botocore
  # request object using a class that does not have the Botocore
  # metadata
  croak sprintf qq{"%s" was not generated with Botocore support.\n}
    . qq{Parameters should be simple objects, not blessed.\n}, ref $self
    if blessed($parameters) && !$self->is_botocore_api;

  my @paged_results;

  # auto pagination is only available when using the Botocore metadata
  my ( $paginator, $use_paginator, $limit );

  if ( $use_botocore && $self->get_use_paginator ) {
    $paginator     = $self->get_paginators && $self->get_paginators->{$action};
    $use_paginator = $paginator;
  }

  # if we're using the Botocore metadata, create a request object
  if ($use_botocore) {
    if ($paginator) {

      $paginator->{more_results} //= $paginator->{output_token};

      if ( $paginator->{limit_key} ) {
        $limit = $parameters->{ $paginator->{limit_key} };
      }

      DEBUG(
        sub {
          return Dumper(
            [ paginator  => $paginator,
              limit      => $limit,
              parameters => $parameters,
            ]
          );
        }
      );
    }

    $parameters = $self->init_botocore_request($parameters);

    $content_type = $self->get_content_type;

    DEBUG(
      sub {
        return Dumper(
          [ parameters   => $parameters,
            content_type => $content_type
          ]
        );
      }
    );

  }
  elsif ( !$content_type ) {
    $content_type = $self->set_content_type( $self->_set_content_type );
  }

  if ( !$parameters && !$use_botocore && $content_type =~ /json/xsm ) {
    $parameters = {};
  }

  my $serialized_content = $self->serialize_content($parameters);

  DEBUG(
    sub {
      Dumper [
        content_type       => $content_type,
        parameters         => $parameters,
        serialized_content => $serialized_content
      ];
    }
  );

  my $page_count = 0;

  PAGINATE:

  ++$page_count;

  DEBUG( sub { return Dumper( [ page => $page_count ] ) } );

  my $rsp = $self->submit(
    content      => $serialized_content,
    content_type => $content_type,
    headers      => $headers,
  );

  $self->set_response($rsp);

  if ( !$rsp->is_success ) {

    $self->set_error(
      Amazon::API::Error->new(
        { error        => $rsp->code,
          message_raw  => $rsp->content,
          content_type => scalar $rsp->content_type,
          api          => ref $self,
          response     => $rsp,
          action       => $self->get_last_action,
        }
      )
    );

    if ( $self->get_print_error && !$self->get_raise_error ) {
      print {*STDERR} $self->get_error;
    }
    elsif ( $self->get_raise_error ) {
      die $self->get_error;
    }
  }

  DEBUG(
    sub {
      return Dumper(
        [ decode_always => $self->get_decode_always,
          content       => $rsp->content
        ]
      );
    }
  );

  return $rsp->content
    if !$self->get_decode_always;

  eval {
    if ($use_paginator) {
      my $result = $self->decode_response;

      DEBUG(
        sub {
          return Dumper(
            [ paginator     => $paginator,
              paged_results => @paged_results,
              result        => $result
            ]
          );
        }
      );

      my $actual_result = dig( $result, $paginator, 'result_key' );

      return \@paged_results
        if !$actual_result;

      DEBUG(
        sub {
          return Dumper( [ result => $result, ] );
        }
      );

      push @paged_results, @{$actual_result};

      DEBUG(
        sub {
          return Dumper(
            [ paged_results => \@paged_results,
              more_results  => $paginator->{more_results},
              paginator     => $paginator,
            ]
          );
        }
      );

      if ( !dig( $result, $paginator, 'more_results' ) ) {
        # might have to dig here too, but generally I don't think so
        if ( $paginator->{limit_key} ) {
          delete $result->{ $paginator->{limit_key} };
        }

        return \@paged_results;
      }

      $limit = $limit // $result->{ $paginator->{limit_key} };

      my $parameters = $self->init_botocore_request(
        { $limit ? ( $paginator->{limit_key} => $limit ) : (),
          $paginator->{input_token} => dig( $result, $paginator, 'output_token' ),
        }
      );

      $serialized_content = $self->serialize_content($parameters);

      goto PAGINATE;
    }
  };

  if ($EVAL_ERROR) {
    if ( $self->get_raise_serialization_errors ) {
      die $EVAL_ERROR;
    }
    else {
      warn "error serializing content: please report this error\n$EVAL_ERROR";
      return $rsp->content;
    }
  }

  return bury( \@paged_results, $paginator, 'result_key' )
    if $use_paginator;

  my $results = $self->decode_response;

  # some APIs return no results, but we probably do not want to return
  # undef
  if ( !$results && $rsp->is_success ) {
    $results = {};
  }

  return $results
    if !$paginator || !$use_paginator;

  # you must have a paginator, but told me not to use it, delete
  # blank/null markers

  # TODO: figure out the comment above because it does not match the code
  # 'use_paginator' is tested above and if it is false (you told me not to use the
  # paginator) we return the results...

  if ( !dig( $results, $paginator, 'more_results' ) ) {
    delete $results->{ $paginator->{input_token} };
    delete $results->{ $paginator->{limit_key} };
  }

  return $results;
}

########################################################################
# the analog to dig, we need to set the result to a hash element specified
# by a dot encoded string. Example: DistributionList.Item
########################################################################
sub bury {
########################################################################
  my ( $result, $paginator, $key ) = @_;

  return $result
    if !$paginator;

  $key = $paginator->{$key};

  return { $key => $result }
    if $key !~ /[.]/xsm;

  my $parent = {};
  my $child  = $parent;

  my @keys       = split /[.]/xsm, $key;
  my $result_key = pop @keys;  # remove last element

  foreach (@keys) {
    $child->{$_} = {};
    $child = $child->{$_};
  }

  $child->{$result_key} = $result;

  return $parent;
}

########################################################################
# follow the . encoded key to find the hash element
# example: DistributionList.Items
########################################################################
sub dig {
########################################################################
  my ( $result, $paginator, $key, $delete ) = @_;

  return $result
    if !$paginator || !$paginator->{$key};

  $key = $paginator->{$key};

  return $result->{$key}
    if $key !~ /[.]/xsm;

  foreach ( split /[.]/xsm, $key ) {
    $result = $result->{$_};
    $key    = $_;
  }

  if ($delete) {
    delete $result->{$key};
  }

  return $result;
}

########################################################################
sub print_error {
########################################################################
  my ($self) = @_;

  my $error = eval {
    my $last_error = $self->get_error;

    return "$last_error"
      if $last_error && ref($last_error) =~ /Amazon::API::Error/xms;

    return $last_error;
  };

  $error //= $EVAL_ERROR;

  if ($error) {
    print {*STDERR} $error;
  }

  return $error;
}

########################################################################
sub get_valid_token {
########################################################################
  my ($self) = @_;

  my $credentials = $self->get_credentials;

  return
    if !$credentials->get_token;

  if ( $credentials->can('is_token_expired') ) {
    if ( $credentials->is_token_expired ) {

      if ( !$credentials->can('refresh_token') || !$credentials->refresh_token ) {
        croak 'token expired';
      }
    }
  }

  TRACE( sub { return Dumper( [ 'valid token:', $credentials->get_token ] ) } );

  return $credentials->get_token;
}

########################################################################
sub submit {
########################################################################
  my ( $self, %options ) = @_;

  DEBUG( sub { return Dumper [ submit => \%options ] } );

  my $method  = $self->get_http_method || 'POST';
  my $headers = $options{'headers'}    || [];

  my $url = $self->get_url;

  my $botocore_protocol = eval { $self->get_botocore_metadata->{'protocol'} } // $EMPTY;

  if ( $botocore_protocol =~ /^rest\-(json|xml)/xsm ) {
    croak 'no request URI provided for rest-json call'
      if !$self->get_request_uri;

    $url .= $self->get_request_uri;
  }

  DEBUG(
    sub {
      return Dumper [
        method  => $method,
        url     => $url,
        headers => $headers
      ];
    }
  );

  my $request = HTTP::Request->new( $method, $url, $headers );

  # 1. set the header
  # 2. set the content
  # 3. sign the request
  # 4. send the request & return result

  # see IMPLEMENTATION NOTES for an explanation
  if ( $self->get_api || $self->get_target_prefix ) {
    $request = $self->_set_x_amz_target($request);
  }

  $self->_set_request_content( request => $request, %options );
  my $credentials = $self->get_credentials;

  if ( my $token = $self->get_valid_token ) {
    $request->header( 'X-Amz-Security-Token' => $token );
  }

  # TODO: global end-points
  my $region = $self->get_region;

  # sign the request
  Amazon::API::Signature4->new(
    -access_key     => $credentials->get_aws_access_key_id,
    -secret_key     => $credentials->get_aws_secret_access_key,
    -security_token => $credentials->get_token || undef,
    service         => $self->get_service,
    region          => $region,
  )->sign( $request, $self->get_region );

  DEBUG( sub { return Dumper( [ request => $request ] ) } );

  # make the request, return response object
  my $ua  = $self->get_user_agent;
  my $rsp = $ua->request($request);

  DEBUG(
    sub {
      return Dumper [ response => $rsp ];
    }
  );

  return $rsp;
}

# +------------------+
# | EXPORTED METHODS |
# +------------------+

########################################################################
sub generate_xml {
########################################################################
  my ($data) = @_;

  my $doc = XML::LibXML::Document->new( '1.0', 'UTF-8' );

  my $fragment = XML::LibXML::DocumentFragment->new();

  for my $key ( keys %{$data} ) {
    _to_xml( $doc, $fragment, $key, $data->{$key} );
  }

  my $xml = $doc->toString(1) . $fragment->toString(1);

  TRACE(
    sub {
      return Dumper(
        [ data => $data,
          xml  => $xml
        ]
      );
    }
  );

  return $xml;
}

########################################################################
sub param_n {
########################################################################
  my (@args) = @_;

  return Amazon::API::Botocore::Shape::Utils::param_n(@args);
}

########################################################################
# create_urlencoded_content(parameters, action, version)
# input:
#       parameters:
#                  SCALAR - query string to encode (x=y&w=z...)
#                  ARRAY  - either an array of hashes or...
#                           key/value pairs of the form x=y or...
#                           key/values pairs
#                  HASH   - key/value pairs, if the value is an array then
#                           it is assumed to be a list of hashes
#       action:    API method
#       version:   wsdl version for API
#
# output:
#       URL encodode query string
#
########################################################################
sub create_urlencoded_content {
########################################################################
  my ( $parameters, $action, $version ) = @_;

  my @args;

  if ( $parameters && !ref $parameters ) {
    @args = map { split /=/xsm } split /&/xsm, $parameters;
  }
  elsif ( $parameters && reftype($parameters) eq 'HASH' ) {
    foreach my $key ( keys %{$parameters} ) {
      if ( ref $parameters->{$key}
        && reftype( $parameters->{$key} ) eq 'ARRAY' ) {
        push @args, map { %{$_} } @{ $parameters->{$key} };
      }
      else {
        push @args, $key, $parameters->{$key};
      }
    }
  }
  elsif ( $parameters && reftype($parameters) eq 'ARRAY' ) {

    # if any are refs then they should be hashes...
    if ( any {ref} @{$parameters} ) {

      @args = map { %{$_} } @{$parameters};  # list of hashes
    }
    elsif ( any {/=/xsm} @{$parameters} ) {
      @args = map { split /=/xsm } @{$parameters};  # formatted list
    }
    else {
      @args = @{$parameters};                       # simple list
    }
  }

  my $content;

  if ( $action && !any {/Action/xsm} @args ) {
    push @args, 'Action', $action;
  }

  if ( $version && !any {/Version/xsm} @args ) {
    push @args, 'Version', $version;
  }

  return join $AMPERSAND, map { sprintf '%s=%s', $_->[0], url_encode( $_->[1] ) } pairs @args;
}

########################################################################
sub has_keys {
########################################################################
  my ( $self, %options ) = @_;

  # note that self should NOT really have keys! Not sure why I added this test
  my %creds = keys %options ? %options : map { $_ => $self->{$_} } @REQUIRED_KEYS;

  return $creds{aws_secret_access_key} && $creds{aws_access_key_id};
}

# +-----------------+
# | PRIVATE METHODS |
# +-----------------+

# should not be called if we have botocore definition
########################################################################
sub _set_content_type {
########################################################################
  my ($self) = @_;

  my $service = $self->get_service;

  # default content-type
  my $content_type = $self->get_content_type;

  return 'application/x-www-form-urlencoded'
    if any { $_ eq $service } @{ $API_TYPES{query} };

  return 'application/x-amz-json-1.1'
    if any { $_ eq $service } @{ $API_TYPES{json} };

  return 'application/xml'
    if any { $_ eq $service } @{ $API_TYPES{xml} };

  return $content_type;
}

########################################################################
sub _create_methods {
########################################################################
  my ($self) = @_;

  my $class = ref $self || $self;

  if ( $self->get_api_methods ) {
    no strict 'refs';        ## no critic (TestingAndDebugging::ProhibitNoStrict)
    no warnings 'redefine';  ## no critic (TestingAndDebugging::ProhibitNoWarnings)

    my $stash = \%{ __PACKAGE__ . $DOUBLE_COLON };

    foreach my $api ( @{ $self->get_api_methods } ) {

      my $method = lcfirst $api;

      $method =~ s/([[:lower:]])([[:upper:]])/$1_$2/xmsg;
      $method = lc $method;

      my $snake_case_method = $class . $DOUBLE_COLON . $method;
      my $camel_case_method = $class . $DOUBLE_COLON . $api;

      # snake case rules the day

      if ( !$stash->{$method} ) {
        *{$snake_case_method} = sub {
          my $self = shift;

          $self->invoke_api( $api, @_ );
        };
      }

      # ...but some prefer camels
      if ( !$stash->{$api} ) {
        *{$camel_case_method} = sub {
          my $self = shift;

          $self->$method(@_);
        };
      }
    }

  }

  return $self;
}

########################################################################
sub _create_stealth_logger {
########################################################################
  my ( $self, $name, $level ) = @_;

  my $sub_name = sprintf '%s::%s', $name, uc $level;

  no strict 'refs';  ## no critic ProhibitNoStrict

  return
    if defined *{$sub_name}{CODE};

  *{$sub_name} = sub {
    if ( $self->get_logger && $self->get_logger->can($level) ) {
      $self->get_logger->$level(@_);
    }
    else {
      return;
    }
  };

  return;
}

########################################################################
sub _set_default_logger {
########################################################################
  my ( $self, $log_level ) = @_;

  if ( !$self->get_logger && !Log::Log4perl->initialized ) {

    require Log::Log4perl;

    Log::Log4perl->import(':easy');

    $log_level = $LOG4PERL_LOG_LEVELS{ lc $log_level } // $LOG4PERL_LOG_LEVELS{info};

    Log::Log4perl->easy_init(
      { level  => $log_level,
        layout => $self->get_log_layout // $DEFAULT_LAYOUT_PATTERN,
        ( $self->get_log_file ? ( file => $self->get_log_file ) : () ),

      }
    );
  }
  else {
    for my $level (qw(debug trace info warn error )) {
      $self->_create_stealth_logger( __PACKAGE__, $level );
      $self->_create_stealth_logger( ref($self),  $level );
    }
  }

  return $self;
}

########################################################################
sub init_log_level {
########################################################################
  my ( $self, %options ) = @_;

  my $debug //= ( $options{debug} || $ENV{DEBUG} );

  $options{log_level} = $debug ? 'debug' : $options{log_level} // 'info';

  return $self->set_log_level( $options{log_level} );
}

########################################################################
sub _set_defaults {
########################################################################
  my ( $self, %options ) = @_;

  $self->set_raise_error( $self->get_raise_error // $TRUE );
  $self->set_print_error( $self->get_print_error // $TRUE );

  $self->set_use_paginator( $self->get_use_paginator // $TRUE );

  if ( !$self->get_user_agent ) {
    $self->set_user_agent( LWP::UserAgent->new );
  }

  if ( !defined $self->get_decode_always ) {
    $self->set_decode_always($TRUE);

    if ( !$self->get_decode_always && !defined $self->get_force_array ) {
      $self->set_force_array($FALSE);
    }
  }

  # most API services are POST, but using the Botocore metadata is best
  $self->set_http_method( $self->get_http_method // 'POST' );

  $self->set_protocol( $self->get_protocol() // 'https' );

  # note some APIs are global, hence an API may send '' to indicate global
  if ( !defined $self->get_region ) {
    $self->set_region( $self->get_region
        || $ENV{'AWS_REGION'}
        || $ENV{'AWS_DEFAULT_REGION'}
        || $DEFAULT_REGION );
  }

  my $debug //= ( $options{debug} || $ENV{DEBUG} );

  $options{log_level} = $debug ? 'debug' : $options{log_level} // 'info';

  $self->set_log_level( $options{log_level} );

  if ( !$self->get_credentials ) {
    if ( $self->has_keys(%options) ) {
      $self->set_credentials(
        Amazon::Credentials->new(
          { aws_secret_access_key => $options{aws_secret_access_key},
            aws_access_key_id     => $options{aws_access_key_id},
            token                 => $options{token},
            region                => $self->get_region,
            no_passkey_warning    => $options{no_passkey_warning},
          }
        )
      );
    }
    else {
      $self->set_credentials(
        Amazon::Credentials->new(
          order              => $options{order},
          region             => $options{region},
          no_passkey_warning => $options{no_passkey_warning},
        )
      );

      if ( !defined $self->get_region ) {
        $self->set_region( $self->get_credentials->get_region );
      }
    }
  }

  # set URL last since it contains region
  $self->_set_url;

  return $self;
}

########################################################################
sub _create_service_url {
########################################################################
  my ($self) = @_;

  my $url;

  my $botocore_metadata = $self->get_botocore_metadata;
  my $service           = $self->get_service;

  if ( $botocore_metadata && $botocore_metadata->{globalEndpoint} ) {
    $url = sprintf '%s://%s', $self->get_protocol, $botocore_metadata->{globalEndpoint};
  }
  else {
    my $endpoint = $self->get_endpoint_prefix || $service;

    if ( any { $_ eq $service } @GLOBAL_SERVICES ) {
      $url = sprintf $GLOBAL_URL_FMT, $self->get_protocol, $endpoint;
    }
    elsif ( $self->get_region ) {
      $url = sprintf $REGIONAL_URL_FMT, $self->get_protocol, $endpoint, $self->get_region;
    }
    else {
      $url = sprintf $REGIONAL_URL_FMT, $self->get_protocol, $endpoint, 'us-east-1';
    }
  }

  return $url;
}

########################################################################
sub _set_url {
########################################################################
  my ($self) = @_;

  my $url = $self->get_url;

  if ( !$url ) {
    $url = $self->_create_service_url;
  }
  else {
    if ( $url !~ /^https?/xmsi ) {
      $url =~ s/^\///xms;  # just remove leading slash...
      $url = $self->get_protocol . '://' . $url;
    }

  }

  $self->set_url($url);

  return $self;
}

########################################################################
sub _set_x_amz_target {
########################################################################
  my ( $self, $request ) = @_;

  my $target  = $self->get_target_prefix;
  my $version = $self->get_version;
  my $api     = $self->get_api;
  my $action  = $self->get_action;

  if ( !$target ) {
    $target = $version ? $api . $UNDERSCORE . $version : $api;
  }

  $target = $target . $DOT . $action;

  $self->set_target($target);

  $request->header( 'X-Amz-Target' => $target );

  return $request;
}

########################################################################
sub _set_request_content {
########################################################################
  my ( $self, %args ) = @_;

  my $request      = $args{request};
  my $content      = $args{content};
  my $content_type = $args{content_type} || $self->get_content_type;

  TRACE(
    sub {
      return Dumper(
        [ method       => $self->get_http_method,
          args         => \%args,
          request      => $request,
          content      => $content,
          content_type => $content_type
        ]
      );
    }
  );

  if ( $self->get_http_method ne 'GET' || !defined $content ) {
    if ($content_type) {
      $request->content_type( $content_type . '; charset=utf-8' );
    }
    $request->content($content);
  }
  else {
    $request->uri( $request->uri . $QUESTION_MARK . $content );
  }

  DEBUG( sub { return Dumper( [ request => $request ] ); } );

  return $request;
}

########################################################################
# Convert a Perl data object to XML
########################################################################
sub _to_xml {
########################################################################
  my ( $doc, $parent, $key, $value ) = @_;

  # Handle array references (multiple child elements)
  if ( ref $value eq 'ARRAY' ) {
    for my $item ( @{$value} ) {
      _to_xml( $doc, $parent, $key, $item );
    }
  }
  # Handle hash references (nested structures)
  elsif ( ref $value eq 'HASH' ) {
    my $element = $doc->createElement($key);

    # Handle attributes (if _attr key exists)
    if ( exists $value->{_attr} ) {
      my $attrs = delete $value->{_attr};
      for ( keys %{$attrs} ) {
        $element->setAttribute( $_, $attrs->{$_} );
      }
    }

    # Process nested elements
    for my $subkey ( keys %{$value} ) {
      _to_xml( $doc, $element, $subkey, $value->{$subkey} );
    }

    $parent->appendChild($element);
  }
  # Handle scalar values (text nodes)
  else {
    my $element = $doc->createElement($key);
    $element->appendTextNode($value);
    $parent->appendChild($element);
  }

  return;
}

1;

__END__

=pod

=head1 NAME

Amazon::API - A generic base class for AWS Services

=head1 SYNOPSIS

  use Amazon::API;

  my $service = Amazon::API->new( service => 'events', api => 'AWSEvents');
  my $rules = $service->invoke_api('ListRules');

=head1 DESCRIPTION

https://github.com/rlauer6/perl-Amazon-API/actions/workflows/build.yml/badge.svg

=begin markdown

[![amazon-api](https://github.com/rlauer6/perl-Amazon-API/actions/workflows/build.yml/badge.svg)](https://github.com/rlauer6/perl-Amazon-API/actions/workflows/build.yml)

=end markdown

=pod

Generic class for constructing AWS API interfaces. Typically used as a
parent class, but can be used directly. This package can also
generates stubs for Amazon APIs using the Botocore project
metadata. (See L</BOTOCORE SUPPORT>).

I<The typical use of this is API is through the classes you build with
the included tool (F<amazon-api>). The tool leverages the Botocore
project's metadata to build classes that are specific to each API (and
are documented in the perlish way). Using C<Amazon::API> directly may
not work in all circumstances unless you are very familiar with the
API you are calling. If you decide to take the L<Luddite approaches|/Take the Luddite approach>, read the documentation carefully before using C<Amazon::API>.>

=over 5

=item * See L</IMPLEMENTATION NOTES> for using C<Amazon::API>
directly to call AWS services.

=item * See
L<Amazon::CloudWatchEvents|https://github.com/rlauer6/perl-Amazon-CloudWatchEvents/blob/master/src/main/perl/lib/Amazon/CloudWatchEvents.pm.in>
for an example of how to use this module as a parent class.

=item * See C<amazon-api -h> for information regarding
how to automatically create Perl classes for AWS services using
Botocore metadata.

=back

=head1 BACKGROUND AND MOTIVATION

A comprehensive Perl interface to AWS services similar to the
I<Botocore> library for Python has been a long time in coming. The
Paws project has been creating an always up-to-date AWS interface with
community support. If you are looking for an extensible method of
installing and invoking a subset of services you might want to
consider C<Amazon::API>.

Think of this class as a DIY kit for installing B<only> the APIs and methods you
need for your AWS project.  Using the included C<amazon-api> utility
you can also roll your own complete Amazon API classes that
include support for serializing requests and responses based on
metadata provided by the Botocore project. The classes you create with
C<amazon-api> include full documentation as pod. (See L</BOTOCORE
SUPPORT> for more details).

=over 5

I<NOTE:> The original L<Amazon::API> was written in 2017 as a I<very>
lightweight way to call a handfull of APIs. The evolution of the
module was based on discovering, without much documentation or help,
the nature of Amazon APIs. In retrospect, even back then, it would
have been easier to consult the Botocore project and decipher how that
project managed to create a library from the metadata. Fast forward
to 2022 and L<Amazon::API> began using the Botocore
metadata in order to, in most cases, correctly call any AWS service.
The L<Amazon::API> module can still be used without the assistance of
Botocore metadata, but it works a heckuva lot better with it.

=back

You can use L<Amazon::API> in 3 different ways:

=head2 Take the Luddite approach

  my $queues = Amazon::API->new(
   {
    service     => 'sqs',
    http_method => 'GET'
   })->invoke_api('ListQueues');

=head2 Build your own API classes with just what you need

 package Amazon::API::SQS;
 
 use strict;
 use warnings;
 
 use parent qw( Amazon::API );
 
 our @API_METHODS = qw(
   ListQueues
   PurgeQueue
   ReceiveMessage
   SendMessage
 );
 
 sub new {
   my ( $class, @options ) = @_;
   $class = ref($class) || $class;
 
   my %options = ref( $options[0] ) ? %{ $options[0] } : @options;
 
   return $class->SUPER::new(
     { service       => 'sqs',
       http_method   => 'GET',
       api_methods   => \@API_METHODS,
       decode_always => 1,
       %options
     }
   );
 }
 
 1;

 use Amazon::API::SQS;
 use Data::Dumper;

 my $sqs = Amazon::API::SQS->new;

 print {*STDERR} Dumper($sqs->ListQueues);

=head2 Use the Botocore metadata to build classes for you

 amazon-api -s sqs create-stubs
 amazon-api -s sqs create-shapes

 perl -I . -MData::Dumper -MAmazon::API:SQS -e 'print Dumper(Amazon::API::SQS->new->ListQueues);'

=over 5

I<NOTE:> In order to use Botocore metadata you must clone the Botocore
repository and point the utility to the repo.

Clone the Botocore project from GitHub:

 mkdir ~/git
 cd git
 git clone https://github.com/boto/botocore.git

Generate stub classes for the API and shapes:

 amazon-api -b ~/git/botocore -s sqs -o ~/lib/perl5 create-stubs
 amazon-api -b ~/git/botocore -s sqs -o ~/lib/perl5 create-shapes

 perldoc Amazon::API::SQS

See L<Amazon::API::Botocore::Pod> for more details regarding building
stubs and shapes.

=back

=head1 THE APPROACH

Essentially, most AWS APIs are RESTful services that adhere to a
common protocol, but differences in services make a single solution
difficult. All services more or less adhere to this framework:

=over 5

=item 1. Set HTTP headers (or query string) to indicate the API and
method to be invoked

=item 2. Set credentials in the header

=item 3. Set API specific headers

=item 4. Sign the request and set the signature in the header

=item 5. Optionally send a payload of parameters for the method being invoked

=back

Specific details of the more recent AWS services are well documented,
however early services were usually implemented as simple HTTP
services that accepted a query string. This module attempts to account
for most of the nuances involved in invoking AWS services and
provide a fairly generic way of invoking these APIs in the most
lightweight way possible.

Using L<Amazon::API> as a generic, lightweight module, naturally does
not provide nuanced support for individual AWS services. To use this
class in that manner for invoking the AWS APIs, you need to be very
familiar with the specific API requirements and responses and be
willng to invest time reading the documentation on Amazon's website.
The payoff is that you can probably use this class to call I<any> AWS
API without installing a large number of dependencies.

If you don't mind a few extra dependencies and overhead, you should
generate the stub APIs and support classes using the C<amazon-api>
utility. The stubs and shapes produced by the utility will serialize
and deserialize requests and responses correctly by using the Botocore
metadata. Botocore metadata provides the necessary information to
create classes that can successfully invoke all of the Amazon APIs.

A good example of creating a quick and dirty interface to CloudWatch
Events can be found here:

L<Amazon::CloudWatchEvents|https://github.com/rlauer6/perl-Amazon-CloudWatchEvents/blob/master/src/main/perl/lib/Amazon/CloudWatchEvents.pm.in>

And invoking some of the APIs can be as easy as:

  Amazon::API->new(
    service     => 'sqs',
    http_method => 'GET'
  }
  )->invoke_api('ListQueues');

=head1 BOTOCORE SUPPORT

Using Botocore metadata and the utilities in this project, you can
create Perl classes that simplify calling AWS services.  After
creating service classes and shape objects from the Botocore metadata
calling AWS APIs will look something like this:

 use Amazon::API::SQS;

 my $sqs = Amazon::API::SQS->new;
 my $rsp = $sqs->ListQueues();

The L<Amazon::API::Botocore> module augments L<Amazon::API> by using
Botocore metadata for determining how to call individual services and
serialize parameters passed to its API methods. A utility (C<amazon-api>)
is provided that can generate Perl classes for all AWS services using
the Botocore metadata.

Perl classes that represent AWS data structures (aka shapes) that are
passed to or returned from services can also be generated. These
classes allow you to call all of the API methods for a given service
using simple Perl objects that are serialized correctly for a specific
method.

Service classes are subclassed from C<Amazon::API> so their C<new()>
constructor takes the same arguments as C<Amazon::API::new()>.

 my $credentials = Amazon::Credential->new();

 my $sqs = Amazon::API::SQS->new( credentials => $credentials );

If you are going to use the Botocore support and automatically
generate API classes you I<must also> create the data structure classes
that are used by each service. The Botocore based APIs will use these
classes to serialize requests and responses.

For more information on generating API classes, see
L<Amazon::API::Botocore::Pod>.

=head2 Response Serialization

With little documentation to go on, interpretting the Botocore
metadata and deducing how to serialize Botocore shapes (using a single
serializer) from Perl objects has been a difficult task. It's likely
that there are still some edge cases and bugs lurking in the
serialization methods. Accordingly, starting with version 1.4.5,
serialization exceptions or exceptions that occur while attempting to
decode a response, will result in the raw response being returned to
the caller. The idea being that getting something back that allows you
figure out what to do with the response might be better than receiving
an error.

OTOH, you might want to see the error, report it, or possibly
contribute to its resolution.  You can prevent errors from being
surpressed by setting the C<raise_serializtion_errors> to a true
value. The default is I<false>.

I<Throughout the rest of this documentation a request made using one
of the classes created by the Botocore support scripts will be
referred to as a B<Botocore request> or B<Botocore API>.>

Starting with version 2.0.12 serialization has become B<much more reliable>,
but there are still some differences in the way the Python Botocore
library serialize responses. For example, some serializers may include
or exclude members that are not present in the response payload. If
you are testing a response element, the best approach is to first test
the truthiness and then test the presence of content.

  if ( $result->{$key} && @{$result->{$key}} ) 

  if ( $result->{$key} && %{result->{$key}} ) 

=head1 ERRORS

When an error is returned from an API request, an exception class
(C<Amazon::API::Error>) will be raised if C<raise_error> has been set
to a true value (the default). If you set C<print_error> to true AND
C<raise_error> is false, then errors will be printed to STDERR.

See L<Amazon::API::Error> for more details.

=head1 METHODS AND SUBROUTINES

I<Reminder: You can mostly ignore this part of the documentation when
you are leveraging Botocore to generate your API classes.>

=head2 new

 new(options)

All options are described below. C<options> can be a list of
key/values or hash reference.

=over 5

=item action

The API method. Normally, you would not set C<action> when you
construct your object. It is set when you call the C<invoke_api>
method or automatically set when you call one of the API stubs created
for you.

Example: 'PutEvents'

=item api

The name of the AWS service. See L</IMPLEMENTATION NOTES> for a
detailed explanation of when to set this value.

Example: 'AWSEvents'

=item api_methods

A reference to an array of method names for the API.  The new
constructor will automatically create methods for each of the method
names listed in the array.

The methods that are created for you are nothing more than stubs that
call C<invoke_api>. The stub is a convenience for calling the
C<invoke_api> method as shown below.

  
 my $api = Amazon::CloudWatch->new;

 $api->PutEvents($events);

...is equivalent to:

 $api->invoke_api->('PutEvents', $events);

Consult the Amazon API documentation for the service to determine what
parameters each action requires.

=item aws_access_key_id

Your AWS access key. Both the access key and secret access key are
required if either is passed. If no credentials are passed, an attempt
will be made to find credentials using L<Amazon::Credentials>. Note
that you may need to pass C<token> as well if you are using temporary
credentials.

=item aws_secret_access_key

Your AWS secret access key.

=item content_type

Default content for parameters passed to the C<invoke_api()>
method. If you do not provide this value, a default content type will
be selected based on the service's protocol.

 query     => application/x-www-form-urlencoded
 rest-json => application/x-amz-json-1.1
 json      => application/json
 rest-xml  => application/xml

=item credentials (optional)

Accessing AWS services requires credentials with sufficient privileges
to make programmatic calls to the APIs that support a service.  This
module supports three ways that you can provide those credentials.

=over 10

=item 1. Pass the credentials directly.

Pass the values for the credentials (C<aws_access_key_id>,
C<aws_secaret_access_key>, C<token>) when you call the C<new> method.
A session token is typically required when you have assumed
a role, you are using the EC2's instance role or a container's role.

=item 2. Pass a class that will provide the credential keys.

Pass a reference to a class that has I<getters> for the credential
keys. The class should supply I<getters> for all three credential keys.

Pass the reference to the class as C<credentials> in the constructor
as shown here:

 my $api = Amazon::API->new(credentials => $credentials_class, ... );

=item 3. Use the default C<Amazon::Credentials> class.

If you do not explicitly pass credentials or do not pass a class that
will supply credentials, the module will use the
C<Amazon::Credentials> class that attempts to find credentials in the
I<environment>, your I<credentials file(s)>, or the I<container or
instance role>.  See L<Amazon::Credentials> for more details.

I<NOTE: The latter method of obtaining credentials is probably the
easiest to use and provides the most succinct and secure way of
obtaining credentials.>

=back

=item debug

Set debug to a true value to enable debug messages. Debug mode will
dump the request and response from all API calls. You can also set the
environment variable DEBUG to enable debugging output. Set the debug
value to '2' to increase the logging level.

default: false

=item decode_always

Set C<decode_always> to a true value to return Perl objects from API
method calls. The default is to return the raw output from the call.
Typically, API calls will return either XML or JSON encoded objects.
Setting C<decode_always> will attempt to decode the content based on
the returned content type.

default: false

=item error

The most recent result of an API call. C<undef> indicates no error was
encountered the last time C<invoke_api> was called.

=item http_method

Sets the HTTP method used to invoke the API. Consult the AWS
documentation for each service to determine the method utilized. Most
of the more recent services utilize the POST method, however older
services like SQS or S3 utilize GET or a combination of methods
depending on the specific method being invoked.

default: POST

=item last_action

The last method call invoked.

=item no_passkey_warning

Prevent passkey warning. This is an option to C<Amazon::Credentials>.

=item print_error

Setting this value to a true value will print a detailed error message
containing the error code and any messages returned by the API to
STDERR when an error occurs. Errors will NOT be printed if
C<raise_error> is also true.

default: true

=item protocol

One of 'http' or 'https'.  Some Amazon services do not support https
(yet).

default: https

=item raise_error

Setting this value to a true value will raise an exception when errors
occur. If you set this value to false you can inspect the C<error>
attribute to determine the success or failure of the last method call.

 $api->invoke_api('ListQueues');

 if ( $api->get_error ) {
   ...
 }

default: true

=item region

The AWS region. Pass an empty string if the service is a global
service that does not require or want a region.

default: $ENV{'AWS_REGION'}, $ENV{'AWS_DEFAULT_REGION'}, 'us-east-1'

=item response

The HTTP response from the last API call.

=item service

The AWS service name. Example: C<sqs>. This value is used as a prefix
when constructing the the service URL (if not C<url> attribute is set).

=item service_url_base

Deprecated, use C<service>

=item token

Session token for assumed roles.

=item url

The service url.  Example: https://events.us-east-1.amazonaws.com

Typically this will be constructed for you based on the region and the
service being invoked. However, you may want to set this manually if
for example you are using a service like
<LocalStack|https://localstack.cloud/> that mocks AWS API calls.

 my $api = Amazon::API->new(service => 's3', url => 'http://localhost:4566/');

=item user_agent

Your own user agent object.  Using
C<Furl>, if you have it avaiable may result in faster response.

default: C<LWP::UserAgent>

=item version

Sets the API version.  Some APIs require a version. Consult the
documentation for individual services.

=back

=head2 invoke_api

 invoke_api(action, [parameters], [content-type], [headers]);

or using named parameters...

 invoke_api({ action => args, ... } )

Invokes the API with the provided parameters.

=over 5

=item action

API name.

=item parameters

Parameters to send to the API. C<parameters> can be a scalar, a hash
reference or an array reference. See the discussion below regarding
C<content-type> and how C<invoke_api()> formats parameters before
sending them as a payload to the API.

You can use the C<param_n()> method to format query string arguments
that are required to be in the I<param.n> notation. This is about the
best documentation I have seen for that format. From the AWS
documentation...

=over 10

Some actions take lists of parameters. These lists are specified using
the I<param.n> notation. Values of n are integers starting from 1. For
example, a parameter list with two elements looks like this:

&AttributeName.1=first

&AttributeName.2=second

=back

An example of using this notation is to set queue attributes when
creating an SQS queue.

 my $attributes = { Attributes => [ { Name => 'VisibilityTimeout', Value => '100' } ] };
 my @sqs_attributes= Amazon::API::param_n($attributes);

 eval {
   $sqs->CreateQueue([ 'QueueName=foo', @sqs_attributes ]);
 };

See L</param_n> for more details.

=item content-type

If you pass the C<content-type> parameter, it is assumed that the parameters are
the actual payload to be sent in the request (unless the parameter is a reference).

The C<parameters> will be converted to a JSON string if the
C<parameters> value is a hash reference.  If the C<parameters> value
is an array reference it will be converted to a query string (Name=Value&...).

To pass a query string, you should send an array of key/value
pairs, or an array of scalars of the form C<Name=Value>.

 [ { Action => 'DescribeInstances' } ]
 [ 'Action=DescribeInstances' ]

=item headers

Array reference of key/value pairs representing additional headers to
send with the request.

=back

=head2 decode_response

Boolean that indicates whether or not to deserialize the most recent
response from an invoked API based on the I<Content-Type> header
returned.  If there is no I<Content-Type> header, then the method will
try to decode it first as a JSON string and then as an XML string. If
both of those fail, the raw content is returned.

You can enable or disable deserializing responses globally by setting
the C<decode_always> attribute when you call the C<new> constructor.

default: true

By default, `Amazon::API` will retrieve all results for Botocore based
API calls that require pagination. To turn this behavior off, set
C<use_paginator> to a false value when you instantiate the API
service.

 my $ec2 = Amazon::API->new(use_paginator => 0);

You can also use the L</paginator> method to retrieve all results from Botocore requests that implement pagination.

=head2 print_error

Prints a formatted version of the last error encountered to STDERR.

=head2 submit

 submit(options)

I<This method is used internally by C<invoke_api> and normally should
not be called by your applications.>

C<options> is a reference to a hash of options:

=over 5

=item content

Payload to send.

=item content_type

Content types we have seen used to send values to AWS APIs:

 application/json
 application/x-amz-json-1.0
 application/x-amz-json-1.1
 application/x-www-form-urlencoded

Check the documentation for the individual APIs for the correct
content type.

=item headers

Array reference of key/value pairs that represent additional headers
to send with the request.

=back

=head1 EXPORTED METHODS

=head2 generate_xml

 generate_xml(object)

Generates XML from a Perl object (uses L<XML::LibXML>). This seems to
do a much better job than XMLout() in allowing a mix of attributes and
nested objects. With C<XMLout()> you need to choose between allowing
attributes (which we need to add the namespace for certain requests)
and nested elements (NoAttr => 1).

=head2 get_api_service

 get_api_service(api, options)

Convenience routine that will return an API instance.

 my $sqs = get_api_service 'sqs';

Equivalent to:

 require Amazon::API::SQS;

 my $sqs = Amazon::API::SQS->new(%options);

=over 5

=item api

The service name. Example: route53, sqs, sns

=item options

list of key/value pairs passed to the new constructor as options

=back

=head2 create_url_encoded_content

 create_urlencoded_content(parameters, action, version)

Returns a URL encoded query string. C<parameters> can be any of SCALAR, ARRAY, or HASH. See below.

=over 5

=item parameters

=over 5

=item SCALAR

Query string to encode (x=y&w=z..)

=item ARRAY

Can be one of:

=over 5

=item * Array of hashes where the keys are the query string variable and the value is the value of that variable

=item * Array of strings of the form "x=y"

=item * An array of key/value pairs - qw( x y w z )

=back

=item HASH

Key/value pairs. If value is an array it is assumed to be a list of hashes

=back

=item action

The method being called. For some query type APIs an Action query variable is required.

=item version

The WSDL version for the API. Some query type APIs require a Version query variable.

=back

=head2 paginator

 paginator(service, api, request)

Returns an array containing the results of an API call that requires
pagination,


 my $result = paginator($ec2, 'DescribeInstances', { MaxResults => 10 });

=head2 param_n

 param_n(parameters)

Format parameters in the "param.n" notation.

C<parameters> should be a hash or array reference.

A good example of a service that uses this notation is the
I<SendMessageBatch> SQS API call.

The sample request can be found here:

L<SendMessageBatch|https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessageBatch.html>


 https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue/
 ?Action=SendMessageBatch
 &SendMessageBatchRequestEntry.1.Id=test_msg_001
 &SendMessageBatchRequestEntry.1.MessageBody=test%20message%20body%201
 &SendMessageBatchRequestEntry.2.Id=test_msg_002
 &SendMessageBatchRequestEntry.2.MessageBody=test%20message%20body%202
 &SendMessageBatchRequestEntry.2.DelaySeconds=60
 &SendMessageBatchRequestEntry.2.MessageAttribute.1.Name=test_attribute_name_1
 &SendMessageBatchRequestEntry.2.MessageAttribute.1.Value.StringValue=test_attribute_value_1
 &SendMessageBatchRequestEntry.2.MessageAttribute.1.Value.DataType=String
 &Expires=2020-05-05T22%3A52%3A43PST
 &Version=2012-11-05
 &AUTHPARAMS

To produce this message you would pass the Perl object below to C<param_n()>:

 my $message = {
   SendMessageBatchRequestEntry => [
     { Id          => 'test_msg_001',
       MessageBody => 'test message body 1'
     },
     { Id               => 'test_msg_002',
       MessageBody      => 'test message body 2',
       DelaySeconds     => 60,
       MessageAttribute => [
         { Name  => 'test_attribute_name_1',
           Value =>
             { StringValue => 'test_attribute_value_1', DataType => 'String' }
         }
       ]
     }
   ]
 };

=head1 CAVEATS

=over 5

=item If you are calling an API that does not expect parameters (or all of
them are optional and you do not pass a parameter) the default is to
pass an empty hash..

  $cwe->ListRules();

would be equivalent to...

  $cwe->ListRules({});

I<CAUTION! This may not be what the API expects! Always consult
the AWS API for the service you are are calling.>

=back

=head1 IMPLEMENTATION NOTES

If you have taken the advice above and created classes using the
F<amazon-api> script you can probably ignore this section. This
section is intended to help those trying to create the lightest weight
possible AWS API class.

Just a reminder for those wanting to go lite...

=over 5

=item * Read the documentation AWS provides for the API. You need to understand the request parameters and headers required.

=item * Examine the Botocore data for the API. That might help you understand that structures required for the calling parameters.

=item * Use the C<aws> CLI script in debug mode to see the actual payloads and how they are formatted.

=back

=head2 Headers

=head3 X-Amz-Target

Most of the newer AWS APIs are invoked as HTTP POST operations and
accept a header C<X-Amz-Target> in lieu of the CGI parameter C<Action>
to specify the specific API action. Some APIs also want the version in
the target, some don't. There is sparse documentation about the
nuances of using the REST interface I<directly> to call AWS APIs, but
you kinda sorta figure it out by parsing the Botocore data for a
particular API.

When invoking an API, the class uses the C<api> value to indicate
that the action should be set in the C<X-Amz-Target> header.  We also
check to see if the version needs to be attached to the action value
as required by some APIs.

  if ( $self->get_api ) {
    if ( $self->get_version) {
      $self->set_target(sprintf('%s_%s.%s', $self->get_api, $self->get_version, $self->get_action));
    }
    else {
      $self->set_target(sprintf('%s.%s', $self->get_api, $self->get_action));
    }

    $request->header('X-Amz-Target', $self->get_target());
  }

DynamoDB and KMS seem to be able to use this in lieu of query
variables C<Action> and C<Version>, although again, there seems to be
a lot of inconsistency (and sometimes flexibility) in the APIs.
DynamoDB uses DynamoDB_YYYYMMDD.Action while KMS does not require the
version that way and prefers TrentService.Action (with no version).
There is no explanation in any of the documentations I have been able
to find as to what "TrentService" might actually mean.  Again, your
best approach is to read Amazon's documentation and look at their
sample requests for guidance.  You can also look to the L<Botocore
project|https://github.com/boto/botocore> for information regarding
the service.  Checkout the F<service-2.json> file within the
sub-directory F<botocore/botocore/data/{api-version}/{service-name}>
which contains details for each service.

In general, the AWS API ecosystem is very organic. Each service seems
to have its own rules and protocol regarding what the content of the
headers should be.

As noted, this generic API interface tries to make it possible to use
one class C<Amazon::API> as a sort of gateway to the APIs. The most
generic interface is simply sending query variables and not much else
in the header.  Services like EC2 conform to that protocol and can be
invoked with relatively little fanfare.

 use Amazon::API;
 use Data::Dumper;

 print Dumper(
   Amazon::API->new(
     service => 'ec2',
     version => '2016-11-15'
   )->invoke_api('DescribeInstances')
 );

Note that invoking the API in this fashion, C<version> is
required.

For more hints regarding how to call a particular service, you can use
the AWS CLI with the --debug option.  Invoke the service using the CLI
and examine the payloads sent by the Botocore library.

=head3 Rolling a New API

Once again, your best bet is to use the C<amazon-api> script to roll a
class from the Botocore metadata, but if you really want to create your own
class the lite way read on.

The L<Amazon::API> class will stub out methods for the API if you pass
an array of API method names.  The stub is equivalent to:

 sub some_api {
   my $self = shift;

   $self->invoke_api('SomeApi', @_);
 }

Some will also be happy to know that the class will create an
equivalent I<CamelCase> version of the method.

As an example, here is a possible implementation of
C<Amazon::CloudWatchEvents> that implements one of the API calls.

 package Amazon::CloudWatchEvents;

 use strict;
 use warnings;

 use parent qw(Amazon::API);

 sub new {
   my ($class, $options) = @_;

   my $self = $class->SUPER::new(
     { %{$options},
       api         => 'AWSEvents',
       service     => 'events',
       api_methods => [qw( ListRules )],
     }
   );

   return $self;
 }

Then...

 use Data::Dumper;

 print Dumper(Amazon::CloudWatchEvents->new->ListRules({}));

Of course, creating a class for the service is optional. It may be
desirable however to create higher level and more convenient methods
that aid the developer in utilizing a particular API.

=head3 Overriding Methods

Because the class does some symbol table munging, you cannot easily
override the methods in the usual way.

 sub ListRules {
   my $self = shift;
   ...
   $self->SUPER::ListRules(@_)
 }

Instead, you should re-implement the method as implemented by this
class.

 sub ListRules {
   my $self = shift;
   ...
   $self->invoke_api('ListRules', @_);
 }

=head2 Content-Type

Yet another piece of evidence that suggests the I<organic> nature of
the Amazon API ecosystem is their use of different C<Content-Type>
headers.  Some of the variations include:

 application/json
 application/x-amz-json-1.0
 application/x-amz-json-1.1
 application/x-www-form-urlencoded

Accordingly, the C<invoke_api()> method can be passed the
C<Content-Type> or will try to make its I<best guess> based on the
service protocol or the type of object being passed as
parameters. There is a hash of service names and service types that
this module uses to determine the content type required by the
service. If services are added that hash needs to be updated.

You can also set the default content type used for the calling service
by passing the C<content_type> option to the constructor.

  $class->SUPER::new(
    content_type => 'application/x-amz-json-1.1',
    api          => 'AWSEvents',
    service      => 'events'
  );

=head2 ADDITIONAL HINTS

=over 5

=item * Bad Request

If you send the wrong headers or payload you're liable to get a 400
Bad Request. You may also get other errors that can be misleading when
you send incorrect parameters. When in doubt compare your requests to
requests from the AWS CLI using the C<--debug> option.

=over 10 

=item 1. Set the C<debug> option to true to see the request object and
the response object from C<Amazon::API>.

=item 2. Excecute the AWS CLI with the --debug option and compare the
request and response with that of your calls.

=back

=item * Payloads

Pay attention to the payloads that are required by each service.  B<Do
not> assume that sending nothing when you have no parameters to pass
is correct. For example, the C<ListSecrets> API of SecretsManager
requires at least an empty JSON object.

 $api->invoke_api('ListSecrets', {});

Failure to send at least an empty JSON object will result in a 400
response. 

=back

=head1 VERSION

This documentation refers to version 2.1.4  of C<Amazon::API>.

=head1 DIAGNOSTICS

To enable diagnostic output, set C<debug> to a true value when calling
the constructor. You can also set the C<DEBUG> environment variable to a
true value to enable diagnostics.

=head2 Logging

By default L<Amazon::API> uses L<Log::Log4perl>'s stealth loggers to
log at the DEBUG and TRACE levels. Setting the environment variable
DEBUG to some value or passing a true value for C<debug> in the
constructor will trigger verbose logging.

If you pass a logger to the constructor, C<Amazon::API> will attempt
to use that if it has the appropriate logging level methods (error,
warn, info, debug, trace). If L<Log::Log4perl> is unavailable and you
do not pass a logger, logging is essentially disabled at any level.

If, for some reason you set the enviroment variable DEBUG to a true
value but do not want C<Amazon::API> to log messages you can turn off
logging as shown below:

 my $ec2 = Amazon::API::EC2->new();

 $ec2->set_log_level('fatal');

=head1 BUGS AND LIMITATIONS

This module has not been tested on Windows OS. Please report any
issues found by opening an issue here:

L<https://github.com/rlauer6/perl-Amazon-API/issues>

=head1 FAQs

=head2 Why should I use this module instead of Paws?

Maybe you shouldn't. Paws is a community supported project and may be
a better choice for most people. The programmers who created Paws are
luminaries in the pantheon of Perl programming (alliteration
intended). If you don't want to install of the AWS services but only
need to use a single service, L<Amazon::API> may be the right choice
for you.  Paws may also have some edge cases for some of the seldom
used services and you might find this module easier to use and debug.

=head2 Does it perform better than Paws?

Probably not. But individual API calls to Amazon services have their
own performance characteristics and idiosyncracies.  The overhead
introduced by this module and Paws may be insignificant compared to
the API performance itself, however Paws is implemented using Moose
and the startup time for a Moose script can longer than the startup
script when using this module. YMMV.

=head2 Does this work for all APIs?

I don't know. Probably not? Feedback is appreciated. L<Amazon::API>
has been developed based on my needs and used accordingly.  Although I
have tested it on many APIs, there may still be some cases that are
not handled properly and I am still deciphering the nuances of
flattening, boxing and serializing objects to send to Amazon APIs. The
newer versions of this module using Botocore metadata have become
increasingly reliable over time and I'm somewhat confident that my
interpretation of the Botocore data produces working classes.

However, keep in mind that Amazon APIs are not created equal,
homogenous or invoked in the the same way for all services. Some
accept parameters as a query strings, some parameters are embedded in
the URI, some are sent as JSON payloads and others as XML. Content
types for payloads are all over the map.  Likewise with return values.

Luckily, the Botocore metadata describes the protocols, parameters and
return values for all APIs. The Botocore metadata is quite amazing
actually. It is used to provide information to the Botocore library
for calling any of the AWS services and even for creating
documentation!

L<Amazon::API> can use that information for creating the Perl classes
that invoke each API but may not interpret the metadata correctly in
all circumstances, so it is likely bugs may still exist.

If you want to use this to invoke S3 APIs, don't. I haven't tried it
and I'm pretty sure it would not work anyway. There are modules
designed specifically for S3; L<Amazon::S3>, L<Net::Amazon::S3>. Use
them instead.

=head2 Do I have to create the shape classes when I generate stubs for a service?

Probably. If you create stubs manually, then you do not need the shape
classes. If you use the scripts provided to create the API stubs using
Botocore metadata, then yes, you must create the shapes so that the
Botocore API methods know how to serialize requests. Note that you can
create the shape stubs using the Botocore metadata while not creating
the API services. You might want to do that if you want a lean stub
but want the benefits of using the shape stubs for serialization of
the parameters (or you want the pod that comes with those classes).

If you produce your stubs manually and do not create the shape stubs,
then you must pass parameters to your API methods that are ready to be
serialized by L<Amazon::API>.  Creating data structures that will be
serialized correctly however is done for you I<if you use the shape
classes>.  For example, to create an SQS queue using the shape stubs,
you can call the C<CreateQueue> API method as describe in the Botocore
documentation.

  $sqs->CreateQueue(
    { QueueName => $queue_name,
      tags      => { Name => 'my-new-queue' },
      { Env => 'dev' },
      Attributes => { VisibilityTimeout => 40 },
      { DelaySeconds => 60 }
    }
  );

If you do not use the shape classes, then you must pass the arguments
in the form that will eventually be serialized in the correct manner
as a query string.

 $sqs->CreateQueue([
  'QueueName=foo',
  'Attributes.1.Value=100',
  'Attributes.1.Name=VisibilityTimeout',
  'Tag.1.Key=Name',
  'Tag.1.Value=foo',
  'Tag.2.Key=Env',
  'Tag.2.Value=dev'
 ]);

=head2 This code does not use "Modern Perl". Why?

This code has evolved over the years from being I<ONLY> a way to make
RESTful calls to a few Amazon APIs, to incorporating the use of the
Botocore metadata. It I<was> one person's effort to create a somewhat
lightweight interface to selected AWS APIs.

The code did not start out as well designed attempt to interpret the
Botocore data by creating a monolithic framework to call ANY AWS
API. Perhaps if it were designed today it might use more of Modern
Perl, like Moose as does Paws. The code does however embrace Perl Best
Practices.  Running C<perlcritic> with the Perl Best Practices theme
should show no or very few findings.

=head2 How do I pass AWS credentials to the API?

There is a bit of magic here as L<Amazon::API> will use
L<Amazon::Credentials> transparently if you do not explicitly pass the
credentials object. I've taken great pains to try to make the
aforementioned module somewhat useful and I<secure>.

See L<Amazon::Credentials>.

=head2 Can I use more than one set of credentials to invoke different APIs?

Yes. See L<Amazon::Credentials>.

=head2 How stable is the interface?

As of version 2.1.0 the interface is quite stable.  I'm not aware of
any current bugs and now consider this project "production ready".

=head2 Why are you using XML::Simple when it clearly says "DO NOT"?

It's simple. And it seems easier to build than other modules that
almost do the same thing.

=head2 I tried to use this with XYZ service and it didn't work. What should do I do?

There are several reasons why your call might not have worked. The
most likely place for API calls to fail is when serializing requests
or serializing results. Enable debugging and see how far the API gets.
Report whether the serialization on the request or response failed.
If the serialization of the results failed, you can set
C<decode_always> to false which will prevent serialization of the
result and return the raw content sent from the API.  Other reasons
your call may have failed include:

=over 5

=item * You passed bad data

Take a look at the data you passed, how was it serialized and
ultimately passed to the API?  Setting the C<debug> flag is usually
helpful in understanding how requests and responses are serialized.

=item * You didn't read the docs and passed bad data

 amazon-api -s sqs CreateQueue

=item * The serialization of Amazon::API::Botocore::Shape isn't working

Serialization output for every class for every API has not been fully
tested and my never be given the breadth of objects and services. You
may find that some API methods return C<Bad Request> or do not
serialize the results (or more likely requests) in the manner
expected. Requests are serialized based I<solely> on the metadata
found in the Botocore project. There lie the clues for each API
(protocol, end points, etc) and the models (shapes) for requests and
response elements.

Some requests require a query string, some an XML or JSON payload. The
Botocore based API classes use the metadata to determine how to send a
request and how to interpret the results. This module uses
L<XML::Simple> or L<JSON> to parse the results. It then uses the
L<Amazon::API::Botocore::Serializer> to turn the parsed results into a
Perl object that respresents the response shape.

It's likely that there are exceptions that are handled as special
cases in the Python or Java libraries that also use the Botocore
metadata. In that case use the C<aws> CLI command in C<--debug> mode
to examine the request and response.

You can find information about each API's request and response from
the documentation created for each service.

 perldoc Amazon::API::Botocore::Shape::EC2:DescribeInstancesRequest

or more succinctly:

 amazon-api -s ec2 help DescribeInstancesRequest

Make sure you understand what the API request should look
like. C<amazon_api> will help illuminate the structure of requests you
should be sending to APIs.

 amazon-api -s sqs help CreateQueue

You can also dump the Botocore metadata from the generated classes using
C<amazon-api>.

 amazon-api -s sqs describe

=over 10

=item Additional Details

Some APIs, most notably query protocol APIs like EC2 seem to require
special serializers. Looking at the Python implementation of the
Botocore library reveals a separate EC2 serializer.  This API has no
such "hook" for APIs that require a unique intepretation of the
Botocore metadata. 

You can however create the correct payloads expected by an API and
pass those when you make a request. For example, the EC2
DescribeSecurityGroups API accepts a Filter object to filter the
results. The Python Botocore signature looks like this:

  response = client.describe_security_groups(
      Filters=[
          {
              'Name': 'string',
              'Values': [
                  'string',
              ]
          },
      ],
      GroupIds=[
          'string',
      ],
      GroupNames=[
          'string',
      ],
      DryRun=True|False,
      NextToken='string',
      MaxResults=123
  )

That signature provides a convenient way to pass the required
parameters to the API. However, when actually passed to the API the
payload is serialized into a query string parameter that might look
something like:

 Filter.1.Name=group-name&Filter.1.Value.1=some-value&Action=DescribeSecurityGroups&Version=2016-11-15

The Filters object you passed gets serialized into I<param.n> notation
as described earlier in this documentation. Knowing that fact (by
looking at the AWS API for DescribeSecurityGroups) and experiencing a
failure when sending what should be the correct request to the API
using this class, you could send correctly formatted payloads to query
protocol APIs like this one.

 my @filter = param_n(
    { Filter => [
        { Name  => 'group-name',
          Value => ['tbc-ssh-only']
        }
      ]
    }
  );

 print Dumper([filter => \@filter]);

Would result in:

 $VAR1 = [
           'filter',
           [
             'Filter.1.Name=group-name',
             'Filter.1.Value.1=tbc-ssh-only'
           ]
         ];

Arrays passed to query protocol requests are assumed to be lists of
query variables and values and are added to the URL when the request
is made.

I<Hopefully, as more is learned about serializing those kinds of API
requests this class will be able to successfully make those API
calls.>

I<UPDATE: Try using the Botocore protocol for APIs by passing a hash
reference of expected variables first. Recent updates have been made
to create special serializers for these older query protocol APIs.>

=back

If you find this project's serializer deficient, please log an issue
and I will attempt to address it.

=back

=head1 LICENSE AND COPYRIGHT

This module is free software. It may be used, redistributed and/or
modified under the same terms as Perl itself.

=head1 TBD

Over the last few years as the classes in this project have evolved,
the number of dependencies has increased to the point where it is no
longer a "lightweight" distribution. In fact, the start up time for
C<Amazon::API> is to be honest, now a bit disappointing.  Accordingly,
the biggest "to do" on the list is to see if the load time can be
reduced. Having said that, the cost of invoking some Amazon APIs and
the fact that you may be using these classes in a manner where initial
load time is not important, may not make this a high priority for
some.

=over 5

=item * decrease load time of C<Amazon::API>

=item * reduce dependencies

=item * reduce generated class modules sizes by separating out pod

=item * investigate a different way to load Botocore metadata rather than embedding it in each module

=back

=head1 SEE OTHER

L<Amazon::Credentials>, L<Amazon::API::Error>, L<AWS::Signature4>, L<Amazon::API::Botocore>, L<Paws>

=head1 AUTHOR

Rob Lauer - <rlauer6@comcast.net>

=cut
