diff --git a/.gitignore b/.gitignore index 5cc2872..c224826 100755 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,18 @@ *.pyc *.pyo +# Distribution / packaging +bin/ +build/ +develop-eggs/ +dist/ +eggs/ +parts/ +sdist/ +*.egg-info/ +.installed.cfg +*.egg + # CMake CMakeFiles/ *.cmake @@ -39,4 +51,9 @@ CMakeCache.txt .cproject .project +# VScode +.vscode/ + venv/ +.venv/ + diff --git a/bin/aws-utils/README.md b/bin/aws-utils/README.md new file mode 100644 index 0000000..692563f --- /dev/null +++ b/bin/aws-utils/README.md @@ -0,0 +1,65 @@ +# EC2 Instance Manager + +# Table of contents + + +- [EC2 Instance Manager](#ec2-instance-manager) +- [Table of contents](#table-of-contents) + - [Python Version](#python-version) + - [Third Party Libraries and Dependencies](#third-party-libraries-and-dependencies) + - [Usage](#usage) + - [Examples](#examples) + + +## Python Version +Python 3.9+ are supported and tested + + +## Third Party Libraries and Dependencies + +The following libraries will be installed when you install the client library: +* [typer](https://github.com/tiangolo/typer) +* [boto3](https://github.com/boto/boto3) + +## Usage + +To start using the library you need to setup a list of ENV variables with your AWS credentials as follows: +```sh +export AWS_DEFAULT_REGION=<...> +export AWS_ACCESS_KEY_ID=<...> +export AWS_SECRET_ACCESS_KEY=<...> +``` +Then you can install a library with pip: + +```sh +pip install . +``` + +If you plan to do a local developent you may also want install it in [editable mode](https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#working-in-development-mode). +```sh +pip install --editable . +``` + +Afther the library is installed you can simply start it with --help option for retrieveing the further insructions +```sh +ecmgm --help +``` + +## Examples + +Here is the list of example commands: + +Create a VM, using existing AWS SSH key +```sh +ecmgm create --osnick --ssh-key-name +``` + +Retrieve VM description +```sh +ecmgm describe +``` + +Teardown VM +```sh +ecmgm teardown +``` \ No newline at end of file diff --git a/bin/aws-utils/ecmgm/__init__.py b/bin/aws-utils/ecmgm/__init__.py new file mode 100644 index 0000000..5b9a997 --- /dev/null +++ b/bin/aws-utils/ecmgm/__init__.py @@ -0,0 +1,2 @@ +__app_name__ = "ecmgm" +__version__ = "0.1.0" diff --git a/bin/aws-utils/ecmgm/__main__.py b/bin/aws-utils/ecmgm/__main__.py new file mode 100644 index 0000000..da656a6 --- /dev/null +++ b/bin/aws-utils/ecmgm/__main__.py @@ -0,0 +1,9 @@ +from ecmgm import cli, __app_name__ + + +def main(): + cli.app(prog_name=__app_name__) + + +if __name__ == "__main__": + main() diff --git a/bin/aws-utils/ecmgm/cli.py b/bin/aws-utils/ecmgm/cli.py new file mode 100644 index 0000000..bd72807 --- /dev/null +++ b/bin/aws-utils/ecmgm/cli.py @@ -0,0 +1,233 @@ +import getpass +from typing import Optional + +import boto3 +import typer +from rich.console import Console, Group +from rich.panel import Panel +from rich.prompt import Confirm +from rich.syntax import Syntax +from rich.table import Table + +from ecmgm import __app_name__, __version__, schemas, utils + +app = typer.Typer() +console = Console() + + +@app.command() +def listvm(): + ec2 = boto3.client("ec2") + response = ec2.describe_instances() + table = Table(title="Instance list") + + for column in [ + "Tags", + "ImageId", + "InstanceId", + "InstanceType", + "LaunchTime", + "Monitoring", + "PublicDnsName", + ]: + table.add_column(column, style="magenta") + + for r in response["Reservations"]: + for i in r["Instances"]: + table.add_row( + i["Tags"][0]["Value"], + i["ImageId"], + i["InstanceId"], + i["InstanceType"], + str(i["LaunchTime"]), + str(i["Monitoring"]), + i["PublicDnsName"], + ) + + console.print(table) + + +@app.command() +def describe(instance_id: str = typer.Argument(..., help="EC2 Instance Id")): + ec2 = boto3.client("ec2") + table = Table(title="Instance details") + table.add_column("Attribute", style="magenta") + table.add_column("Value", justify="left", style="cyan") + + response = ec2.describe_instances( + InstanceIds=[ + instance_id, + ], + ) + + for k, v in response["Reservations"][0]["Instances"][0].items(): + table.add_row(k, str(v)) + + console.print(table) + + +@app.command() +def teardown(instance_id: str = typer.Argument(..., help="EC2 Instance Id")): + instance = utils.terminate_instance(instance_id) + with console.status("[bold green]Waiting for EC2 Instance being terminated..."): + instance.wait_until_terminated() + + +@app.command() +def create( + name: str = typer.Argument(..., help="Virtual machine name"), + os_image: schemas.OsImage = typer.Argument(..., help="OS image"), + instance_type: schemas.InstanceTypes = typer.Option( + schemas.InstanceTypes.small.value, help="Instance types" + ), + instance_arch: schemas.InstanceArch = typer.Option( + schemas.InstanceArch.x86_64.value, help="Instance Arch" + ), + osnick: str = typer.Option("", help="Optionally filter images by specified osnick"), + ssh_key_name: str = typer.Option( + "", help="You can specify your SSH key name in case its already created in AWS" + ), + host_id: str = typer.Option( + "", + help="Dedicated host id (in case you are spinning macos instance) if not provided will be created by default", + ), +): + if ( + instance_type == schemas.InstanceTypes.macmetal.value + and instance_arch != schemas.InstanceArch.x86_64_mac.value + ): + raise ValueError( + f"Unsoported instance_arch, mac instance types supports only {schemas.InstanceArch.x86_64_mac.value}" + ) + if ( + instance_arch == schemas.InstanceArch.x86_64_mac.value + and instance_type != schemas.InstanceTypes.macmetal.value + ): + raise ValueError( + f"Unsoported instance type, mac instance arch supports only {schemas.InstanceTypes.macmetal.value}" + ) + ec2 = boto3.client("ec2") + image_filter_query = schemas.OS_SEARCH_MAPPING[os_image] + + if osnick: + image_filter_query += f"{osnick}*" + + with console.status( + f"[bold green]Searching for image. Search params: query: {image_filter_query} arch: {instance_arch} osnick: {osnick}..." + ): + images = ec2.describe_images( + Filters=[ + { + "Name": "architecture", + "Values": [ + instance_arch, + ], + }, + {"Name": "root-device-type", "Values": ["ebs"]}, + {"Name": "state", "Values": ["available"]}, + {"Name": "virtualization-type", "Values": ["hvm"]}, + {"Name": "hypervisor", "Values": ["xen"]}, + {"Name": "image-type", "Values": ["machine"]}, + { + "Name": "name", + "Values": [image_filter_query], + }, + ], + Owners=["amazon"], + ) + + sorted_amis = sorted( + images["Images"], key=lambda x: x["CreationDate"], reverse=True + ) + target_image = sorted_amis[0] + panel_group = Group( + Panel( + target_image["ImageId"], + title="ami-id", + ), + Panel(target_image["Description"], title="Description"), + Panel(target_image["ImageLocation"], title="Image"), + ) + console.print("Image found, details:") + console.print(Panel(panel_group)) + + is_continue = Confirm.ask("Do you want to continue?") + if not is_continue: + raise typer.Exit() + + if not ssh_key_name: + console.print(f"No SSH key was specified, creating new") + private_key_name = f"{getpass.getuser()}-ec2-{os_image.value}-key" + private_key_filename = f"{getpass.getuser()}-ec2-{os_image.value}-key-file.pem" + key_pair = utils.create_key_pair(private_key_name, private_key_filename) + console.print(f"Created a key pair [bold cyan]{key_pair.key_name}[/bold cyan]") + + if instance_type == schemas.InstanceTypes.macmetal.value and not host_id: + console.print( + f"Creating dedicated host for instance type: [bold cyan]{instance_type}[/bold cyan]" + ) + host_id = utils.allocate_hosts(instance_type) + console.print( + f"Dedicated host with id [bold cyan]{host_id}[/bold cyan] created" + ) + + instance = utils.create_instance( + target_image["ImageId"], + name, + instance_type, + ssh_key_name or key_pair.key_name, + ["redis-io-group"], # todo default security group is temporary hardcoded + host_id, + ) + + with console.status("[bold green]Waiting for EC2 Instance to start..."): + instance.wait_until_running() + # updating instance attributes to obtain public ip/dns immediately + instance.reload() + + private_key_filename = private_key_filename if not ssh_key_name else "your-key.pem" + panel_group = Group( + Panel("You can now connect to your ec2 machine :thumbs_up:\n"), + Panel( + Syntax( + f"ssh -i {private_key_filename} {schemas.OS_DEFAULT_USER_MAP[os_image]}@{instance.public_dns_name}", + "shell", + theme="monokai", + ), + title="Command", + ), + Panel( + Syntax( + f"# you also might need to make key to be only readable by you\nchmod 400 {private_key_filename}\n" + f"# You may use that instance id -> [{instance.id} <- in other CLI commands like\n" + "# Get VM details\n" + f"ecmgm describe {instance.id}\n" + "# Terminate VM\n" + f"ecmgm teardown {instance.id}", + "shell", + theme="monokai", + ), + title="Tip", + ), + ) + console.print(Panel(panel_group)) + + +def _version_callback(value: bool) -> None: + if value: + typer.echo(f"{__app_name__} v{__version__}") + raise typer.Exit() + + +@app.callback() +def main( + version: Optional[bool] = typer.Option( + None, + "--version", + "-v", + help="Show the application's version and exit.", + callback=_version_callback, + is_eager=True, + ) +) -> None: + return diff --git a/bin/aws-utils/ecmgm/schemas.py b/bin/aws-utils/ecmgm/schemas.py new file mode 100644 index 0000000..956b83c --- /dev/null +++ b/bin/aws-utils/ecmgm/schemas.py @@ -0,0 +1,61 @@ +from enum import Enum + + +class InstanceTypes(str, Enum): + """ + Instance vCPU* Mem (GiB) + t2.nano 1 0.5 + t2.micro 1 1 + t2.small 1 2 + t2.medium 2 4 + t2.large 2 8 + t2.xlarge 4 16 + t2.2xlarge 8 32 + mac1.metal 12 32 + mac2.metal 12 16 + """ + + nano = "t2.nano" + micro = "t2.micro" + small = "t2.small" + medium = "t2.medium" + large = "t2.large" + xlarge = "t2.xlarge" + doublelarge = "t2.2xlarge" + macmetal = "mac1.metal" + + +class InstanceArch(str, Enum): + i386 = "i386" + x86_64 = "x86_64" + x86_64_mac = "x86_64_mac" + arm64 = "arm64" + + +class OsImage(str, Enum): + windows = "windows" + ubuntu = "ubuntu" + debian = "debian" + suse = "suse" + amazon_linux = "amazon_linux" + redhat = "redhat" + macos = "macos" + + +OS_SEARCH_MAPPING = { + OsImage.windows: "*Windows*", + OsImage.ubuntu: "*ubuntu*/images/*", + OsImage.debian: "*debian", + OsImage.suse: "*suse*", + OsImage.amazon_linux: "amzn2-ami-hvm-*", + OsImage.redhat: "*RHEL*", + OsImage.macos: "*macos*", +} +OS_DEFAULT_USER_MAP = { + OsImage.ubuntu: "ubuntu", + OsImage.debian: "admin", + OsImage.redhat: "ec2-user", + OsImage.suse: "ec2-user", + OsImage.amazon_linux: "ec2-user", + OsImage.macos: "ec2-user", +} diff --git a/bin/aws-utils/ecmgm/utils.py b/bin/aws-utils/ecmgm/utils.py new file mode 100644 index 0000000..21002b7 --- /dev/null +++ b/bin/aws-utils/ecmgm/utils.py @@ -0,0 +1,248 @@ +import base64 +import os +import struct + +import boto3 +import paramiko +from botocore.exceptions import ClientError +from paramiko.util import deflate_long +from rich.console import Console + +console = Console() +ec2 = boto3.resource("ec2") +client = boto3.client("ec2") + + +def start_instance(instance_id: str) -> ec2.Instance: + try: + instance = ec2.Instance(instance_id).start() + console.print(f"Started instance {instance_id}") + except ClientError: + console.print(f"Couldn't start instance {instance_id}") + raise + else: + return instance + + +def stop_instance(instance_id: str) -> ec2.Instance: + try: + instance = ec2.Instance(instance_id).stop() + console.print(f"Stopped instance {instance_id}") + except ClientError: + console.print(f"Couldn't stop instance {instance_id}") + raise + else: + return instance + + +def get_console_output(instance_id: str): + try: + output = ec2.Instance(instance_id).console_output()["Output"] + console.print(f"Got console output for instance {instance_id}") + except ClientError: + console.print((f"Couldn't get console output for instance {instance_id}")) + raise + else: + return output + + +def import_key_pair(key_name: str, private_key_file_path: str) -> ec2.KeyPair: + """ + Import existing key pair in AWS to allow SSH access + """ + key = paramiko.RSAKey.from_private_key_file(private_key_file_path) + + output = b"" + parts = [ + b"ssh-rsa", + deflate_long(key.public_numbers.e), + deflate_long(key.public_numbers.n), + ] + + for part in parts: + output += struct.pack(">I", len(part)) + part + public_key = b"ssh-rsa " + base64.b64encode(output) + b"\n" + + key_pair = ec2.import_key_pair(KeyName=key_name, PublicKeyMaterial=public_key) + return key_pair + + +def create_key_pair(key_name: str, private_key_file_name: str = None) -> ec2.KeyPair: + """ + Create key pair in AWS to allow SSH access + """ + try: + key_pair = ec2.create_key_pair(KeyName=key_name) + console.print(f"Created key [bold cyan]{key_pair.name}[/bold cyan].") + if private_key_file_name is not None: + with open(private_key_file_name, "w") as pk_file: + pk_file.write(key_pair.key_material) + console.print( + f"Wrote private key to [bold cyan]{private_key_file_name}[/bold cyan]." + ) + except ClientError: + console.print(f"Couldn't create key {key_name}.") + raise + else: + return key_pair + + +def setup_security_group(group_name: str, group_description: str) -> ec2.SecurityGroup: + """ + Create security group + """ + try: + default_vpc = list( + ec2.vpcs.filter(Filters=[{"Name": "isDefault", "Values": ["true"]}]) + )[0] + console.print(f"Got default VPC {default_vpc.id}") + except ClientError: + console.print("Couldn't get VPCs.") + raise + except IndexError: + console.print("No default VPC in the list.") + raise + + try: + security_group = default_vpc.create_security_group( + GroupName=group_name, Description=group_description + ) + console.print(f"Created security group {group_name} in VPC {default_vpc.id}.") + except ClientError: + console.print(f"Couldn't create security group {group_name}.") + raise + + try: + ip_permissions = [ + { + # HTTP ingress open to anyone + "IpProtocol": "tcp", + "FromPort": 80, + "ToPort": 80, + "IpRanges": [{"CidrIp": "0.0.0.0/0"}], + }, + { + # HTTPS ingress open to anyone + "IpProtocol": "tcp", + "FromPort": 443, + "ToPort": 443, + "IpRanges": [{"CidrIp": "0.0.0.0/0"}], + }, + { + # SSH ingress open to anyone + "IpProtocol": "tcp", + "FromPort": 22, + "ToPort": 22, + "IpRanges": [{"CidrIp": "0.0.0.0/0"}], + }, + ] + + security_group.authorize_ingress(IpPermissions=ip_permissions) + + except ClientError: + console.print(f"Couldnt authorize inbound rules for {group_name}.") + raise + else: + return security_group + + +def create_instance( + image_id: str, + name: str, + instance_type: str, + key_name: str, + security_group_names: list = None, + host_id: str = None, +) -> ec2.Instance: + """ + Create instance + """ + try: + instance_params = { + "ImageId": image_id, + "InstanceType": instance_type, + "KeyName": key_name, + "TagSpecifications": [ + { + "ResourceType": "instance", + "Tags": [ + {"Key": "Name", "Value": name}, + ], + }, + ], + } + if host_id: + instance_params["Placement"] = { + "HostId": host_id, + # "Tenancy": "default" | "dedicated" | "host", + } + if security_group_names is not None: + instance_params["SecurityGroups"] = security_group_names + instance = ec2.create_instances(**instance_params, MinCount=1, MaxCount=1)[0] + console.print(f"Created instance [bold cyan]{instance.id}[/bold cyan].") + except ClientError: + console.print( + f"Couldn't create instance with image {image_id}, instance type {instance_type}, and key {key_name}." + ) + raise + else: + return instance + + +def delete_key_pair(key_name: str, key_file_name: str) -> None: + """ + Deletes a key pair and the specified private key file. + """ + try: + ec2.KeyPair(key_name).delete() + os.remove(key_file_name) + console.print(f"Deleted key {key_name} and private key file {key_file_name}.") + except ClientError: + console.print(f"Couldn't delete key {key_name}.") + raise + + +def delete_security_group(group_id: str) -> None: + """ + Deletes a security group. + """ + try: + ec2.SecurityGroup(group_id).delete() + console.print(f"Deleted security group {group_id}.") + except ClientError: + console.print( + f"Couldn't delete security group {group_id}.", + ) + raise + + +def terminate_instance(instance_id: str) -> ec2.Instance: + """ + Terminates an instance. The request returns immediately. + """ + try: + instance = ec2.Instance(instance_id) + instance.terminate() + console.print(f"Terminating instance [bold cyan]{instance_id}[/bold cyan].") + return instance + except ClientError: + console.print(f"Couldn't terminate instance {instance_id}.") + raise + + +def allocate_hosts(instance_type: str) -> list: + """ + Create dedicated host for aws default region + """ + try: + response = client.allocate_hosts( + AutoPlacement="on", + # available zones ussualy are aws region + [a,b,c] postfix + AvailabilityZone=os.environ.get("AWS_DEFAULT_REGION") + "a", + InstanceType=instance_type, + Quantity=1, + ) + except ClientError: + console.print(f"Couldn't allocate dedicated host.") + raise + return response["HostIds"][0] diff --git a/bin/aws-utils/pyproject.toml b/bin/aws-utils/pyproject.toml new file mode 100644 index 0000000..bf91296 --- /dev/null +++ b/bin/aws-utils/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +build-backend = "setuptools.build_meta" +requires = ["setuptools", "wheel"] diff --git a/bin/aws-utils/setup.cfg b/bin/aws-utils/setup.cfg new file mode 100644 index 0000000..81d6733 --- /dev/null +++ b/bin/aws-utils/setup.cfg @@ -0,0 +1,35 @@ +[metadata] +name = ecmgm +version = attr: ecmgm.__version__ +classifiers = + Programming Language :: Python :: 3 + +[options] +zip_safe = False +include_package_data = True +packages=find: +install_requires = + bcrypt ==4.0.0 + boto3 ==1.24.70 + botocore ==1.27.70 + cffi ==1.15.1 + click ==8.1.3 + colorama ==0.4.5 + commonmark ==0.9.1 + cryptography ==38.0.1 + jmespath ==1.0.1 + paramiko ==2.11.0 + pycparser ==2.21 + Pygments ==2.13.0 + PyNaCl ==1.5.0 + python-dateutil ==2.8.2 + rich ==12.5.1 + s3transfer ==0.6.0 + shellingham ==1.5.0 + six ==1.16.0 + typer ==0.6.1 + urllib3 ==1.26.12 + +[options.entry_points] +console_scripts = + ecmgm = ecmgm.__main__:main diff --git a/bin/aws-utils/setup.py b/bin/aws-utils/setup.py new file mode 100644 index 0000000..3762150 --- /dev/null +++ b/bin/aws-utils/setup.py @@ -0,0 +1,6 @@ +import setuptools + +if __name__ == '__main__': + setuptools.setup( + # see 'setup.cfg' + )