+++ /dev/null
-import logging
-import json
-import tempfile
-import random
-import threading
-import subprocess
-import socket
-import time
-import os
-from http import server as http_server
-from random import randint
-from .tests import get_realm, \
- ZonegroupConns, \
- zonegroup_meta_checkpoint, \
- zone_meta_checkpoint, \
- zone_bucket_checkpoint, \
- zone_data_checkpoint, \
- zonegroup_bucket_checkpoint, \
- check_bucket_eq, \
- gen_bucket_name, \
- get_user, \
- get_tenant
-from .zone_ps import PSTopic, \
- PSTopicS3, \
- PSNotification, \
- PSSubscription, \
- PSNotificationS3, \
- print_connection_info, \
- delete_all_s3_topics, \
- put_object_tagging, \
- get_object_tagging, \
- delete_all_objects
-from .multisite import User
-from nose import SkipTest
-from nose.tools import assert_not_equal, assert_equal
-import boto.s3.tagging
-
-# configure logging for the tests module
-log = logging.getLogger(__name__)
-
-skip_push_tests = True
-
-####################################
-# utility functions for pubsub tests
-####################################
-
-def set_contents_from_string(key, content):
- try:
- key.set_contents_from_string(content)
- except Exception as e:
- print('Error: ' + str(e))
-
-
-# HTTP endpoint functions
-# multithreaded streaming server, based on: https://stackoverflow.com/questions/46210672/
-
-class HTTPPostHandler(http_server.BaseHTTPRequestHandler):
- """HTTP POST hanler class storing the received events in its http server"""
- def do_POST(self):
- """implementation of POST handler"""
- try:
- content_length = int(self.headers['Content-Length'])
- body = self.rfile.read(content_length)
- log.info('HTTP Server (%d) received event: %s', self.server.worker_id, str(body))
- self.server.append(json.loads(body))
- except:
- log.error('HTTP Server received empty event')
- self.send_response(400)
- else:
- if self.headers.get('Expect') == '100-continue':
- self.send_response(100)
- else:
- self.send_response(200)
- finally:
- if self.server.delay > 0:
- time.sleep(self.server.delay)
- self.end_headers()
-
-
-class HTTPServerWithEvents(http_server.HTTPServer):
- """HTTP server used by the handler to store events"""
- def __init__(self, addr, handler, worker_id, delay=0):
- http_server.HTTPServer.__init__(self, addr, handler, False)
- self.worker_id = worker_id
- self.events = []
- self.delay = delay
-
- def append(self, event):
- self.events.append(event)
-
-
-class HTTPServerThread(threading.Thread):
- """thread for running the HTTP server. reusing the same socket for all threads"""
- def __init__(self, i, sock, addr, delay=0):
- threading.Thread.__init__(self)
- self.i = i
- self.daemon = True
- self.httpd = HTTPServerWithEvents(addr, HTTPPostHandler, i, delay)
- self.httpd.socket = sock
- # prevent the HTTP server from re-binding every handler
- self.httpd.server_bind = self.server_close = lambda self: None
- self.start()
-
- def run(self):
- try:
- log.info('HTTP Server (%d) started on: %s', self.i, self.httpd.server_address)
- self.httpd.serve_forever()
- log.info('HTTP Server (%d) ended', self.i)
- except Exception as error:
- # could happen if the server r/w to a closing socket during shutdown
- log.info('HTTP Server (%d) ended unexpectedly: %s', self.i, str(error))
-
- def close(self):
- self.httpd.shutdown()
-
- def get_events(self):
- return self.httpd.events
-
- def reset_events(self):
- self.httpd.events = []
-
-
-class StreamingHTTPServer:
- """multi-threaded http server class also holding list of events received into the handler
- each thread has its own server, and all servers share the same socket"""
- def __init__(self, host, port, num_workers=100, delay=0):
- addr = (host, port)
- self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- self.sock.bind(addr)
- self.sock.listen(num_workers)
- self.workers = [HTTPServerThread(i, self.sock, addr, delay) for i in range(num_workers)]
-
- def verify_s3_events(self, keys, exact_match=False, deletions=False, expected_sizes={}):
- """verify stored s3 records agains a list of keys"""
- events = []
- for worker in self.workers:
- events += worker.get_events()
- worker.reset_events()
- verify_s3_records_by_elements(events, keys, exact_match=exact_match, deletions=deletions, expected_sizes=expected_sizes)
-
- def verify_events(self, keys, exact_match=False, deletions=False):
- """verify stored events agains a list of keys"""
- events = []
- for worker in self.workers:
- events += worker.get_events()
- worker.reset_events()
- verify_events_by_elements(events, keys, exact_match=exact_match, deletions=deletions)
-
- def get_and_reset_events(self):
- events = []
- for worker in self.workers:
- events += worker.get_events()
- worker.reset_events()
- return events
-
- def close(self):
- """close all workers in the http server and wait for it to finish"""
- # make sure that the shared socket is closed
- # this is needed in case that one of the threads is blocked on the socket
- self.sock.shutdown(socket.SHUT_RDWR)
- self.sock.close()
- # wait for server threads to finish
- for worker in self.workers:
- worker.close()
- worker.join()
-
-
-# AMQP endpoint functions
-
-
-class AMQPReceiver(object):
- """class for receiving and storing messages on a topic from the AMQP broker"""
- def __init__(self, exchange, topic, external_endpoint_address=None, ca_location=None):
- import pika
- import ssl
-
- if ca_location:
- ssl_context = ssl.create_default_context()
- ssl_context.load_verify_locations(cafile=ca_location)
- ssl_options = pika.SSLOptions(ssl_context)
- rabbitmq_port = 5671
- else:
- rabbitmq_port = 5672
- ssl_options = None
-
- if external_endpoint_address:
- params = pika.URLParameters(external_endpoint_address, ssl_options=ssl_options)
- else:
- hostname = get_ip()
- params = pika.ConnectionParameters(host=hostname, port=rabbitmq_port, ssl_options=ssl_options)
- remaining_retries = 10
- while remaining_retries > 0:
- try:
- connection = pika.BlockingConnection(params)
- break
- except Exception as error:
- remaining_retries -= 1
- print('failed to connect to rabbitmq (remaining retries '
- + str(remaining_retries) + '): ' + str(error))
- time.sleep(1)
-
- if remaining_retries == 0:
- raise Exception('failed to connect to rabbitmq - no retries left')
-
- self.channel = connection.channel()
- self.channel.exchange_declare(exchange=exchange, exchange_type='topic', durable=True)
- result = self.channel.queue_declare('', exclusive=True)
- queue_name = result.method.queue
- self.channel.queue_bind(exchange=exchange, queue=queue_name, routing_key=topic)
- self.channel.basic_consume(queue=queue_name,
- on_message_callback=self.on_message,
- auto_ack=True)
- self.events = []
- self.topic = topic
-
- def on_message(self, ch, method, properties, body):
- """callback invoked when a new message arrive on the topic"""
- log.info('AMQP received event for topic %s:\n %s', self.topic, body)
- self.events.append(json.loads(body))
-
- # TODO create a base class for the AMQP and HTTP cases
- def verify_s3_events(self, keys, exact_match=False, deletions=False, expected_sizes={}):
- """verify stored s3 records agains a list of keys"""
- verify_s3_records_by_elements(self.events, keys, exact_match=exact_match, deletions=deletions, expected_sizes=expected_sizes)
- self.events = []
-
- def verify_events(self, keys, exact_match=False, deletions=False):
- """verify stored events agains a list of keys"""
- verify_events_by_elements(self.events, keys, exact_match=exact_match, deletions=deletions)
- self.events = []
-
- def get_and_reset_events(self):
- tmp = self.events
- self.events = []
- return tmp
-
-
-def amqp_receiver_thread_runner(receiver):
- """main thread function for the amqp receiver"""
- try:
- log.info('AMQP receiver started')
- receiver.channel.start_consuming()
- log.info('AMQP receiver ended')
- except Exception as error:
- log.info('AMQP receiver ended unexpectedly: %s', str(error))
-
-
-def create_amqp_receiver_thread(exchange, topic, external_endpoint_address=None, ca_location=None):
- """create amqp receiver and thread"""
- receiver = AMQPReceiver(exchange, topic, external_endpoint_address, ca_location)
- task = threading.Thread(target=amqp_receiver_thread_runner, args=(receiver,))
- task.daemon = True
- return task, receiver
-
-
-def stop_amqp_receiver(receiver, task):
- """stop the receiver thread and wait for it to finis"""
- try:
- receiver.channel.stop_consuming()
- log.info('stopping AMQP receiver')
- except Exception as error:
- log.info('failed to gracefuly stop AMQP receiver: %s', str(error))
- task.join(5)
-
-def check_ps_configured():
- """check if at least one pubsub zone exist"""
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- ps_zones = zonegroup.zones_by_type.get("pubsub")
- if not ps_zones:
- raise SkipTest("Requires at least one PS zone")
-
-
-def is_ps_zone(zone_conn):
- """check if a specific zone is pubsub zone"""
- if not zone_conn:
- return False
- return zone_conn.zone.tier_type() == "pubsub"
-
-
-def verify_events_by_elements(events, keys, exact_match=False, deletions=False):
- """ verify there is at least one event per element """
- err = ''
- for key in keys:
- key_found = False
- if type(events) is list:
- for event_list in events:
- if key_found:
- break
- for event in event_list['events']:
- if event['info']['bucket']['name'] == key.bucket.name and \
- event['info']['key']['name'] == key.name:
- if deletions and event['event'] == 'OBJECT_DELETE':
- key_found = True
- break
- elif not deletions and event['event'] == 'OBJECT_CREATE':
- key_found = True
- break
- else:
- for event in events['events']:
- if event['info']['bucket']['name'] == key.bucket.name and \
- event['info']['key']['name'] == key.name:
- if deletions and event['event'] == 'OBJECT_DELETE':
- key_found = True
- break
- elif not deletions and event['event'] == 'OBJECT_CREATE':
- key_found = True
- break
-
- if not key_found:
- err = 'no ' + ('deletion' if deletions else 'creation') + ' event found for key: ' + str(key)
- log.error(events)
- assert False, err
-
- if not len(events) == len(keys):
- err = 'superfluous events are found'
- log.debug(err)
- if exact_match:
- log.error(events)
- assert False, err
-
-
-def verify_s3_records_by_elements(records, keys, exact_match=False, deletions=False, expected_sizes={}):
- """ verify there is at least one record per element """
- err = ''
- for key in keys:
- key_found = False
- object_size = 0
- if type(records) is list:
- for record_list in records:
- if key_found:
- break
- for record in record_list['Records']:
- if record['s3']['bucket']['name'] == key.bucket.name and \
- record['s3']['object']['key'] == key.name:
- if deletions and record['eventName'].startswith('ObjectRemoved'):
- key_found = True
- object_size = record['s3']['object']['size']
- break
- elif not deletions and record['eventName'].startswith('ObjectCreated'):
- key_found = True
- object_size = record['s3']['object']['size']
- break
- else:
- for record in records['Records']:
- if record['s3']['bucket']['name'] == key.bucket.name and \
- record['s3']['object']['key'] == key.name:
- if deletions and record['eventName'].startswith('ObjectRemoved'):
- key_found = True
- object_size = record['s3']['object']['size']
- break
- elif not deletions and record['eventName'].startswith('ObjectCreated'):
- key_found = True
- object_size = record['s3']['object']['size']
- break
-
- if not key_found:
- err = 'no ' + ('deletion' if deletions else 'creation') + ' event found for key: ' + str(key)
- assert False, err
- elif expected_sizes:
- assert_equal(object_size, expected_sizes.get(key.name))
-
- if not len(records) == len(keys):
- err = 'superfluous records are found'
- log.warning(err)
- if exact_match:
- for record_list in records:
- for record in record_list['Records']:
- log.error(str(record['s3']['bucket']['name']) + ',' + str(record['s3']['object']['key']))
- assert False, err
-
-
-def init_rabbitmq():
- """ start a rabbitmq broker """
- hostname = get_ip()
- #port = str(random.randint(20000, 30000))
- #data_dir = './' + port + '_data'
- #log_dir = './' + port + '_log'
- #print('')
- #try:
- # os.mkdir(data_dir)
- # os.mkdir(log_dir)
- #except:
- # print('rabbitmq directories already exists')
- #env = {'RABBITMQ_NODE_PORT': port,
- # 'RABBITMQ_NODENAME': 'rabbit'+ port + '@' + hostname,
- # 'RABBITMQ_USE_LONGNAME': 'true',
- # 'RABBITMQ_MNESIA_BASE': data_dir,
- # 'RABBITMQ_LOG_BASE': log_dir}
- # TODO: support multiple brokers per host using env
- # make sure we don't collide with the default
- try:
- proc = subprocess.Popen(['sudo', '--preserve-env=RABBITMQ_CONFIG_FILE', 'rabbitmq-server'])
- except Exception as error:
- log.info('failed to execute rabbitmq-server: %s', str(error))
- print('failed to execute rabbitmq-server: %s' % str(error))
- return None
- # TODO add rabbitmq checkpoint instead of sleep
- time.sleep(5)
- return proc #, port, data_dir, log_dir
-
-
-def clean_rabbitmq(proc): #, data_dir, log_dir)
- """ stop the rabbitmq broker """
- try:
- subprocess.call(['sudo', 'rabbitmqctl', 'stop'])
- time.sleep(5)
- proc.terminate()
- except:
- log.info('rabbitmq server already terminated')
- # TODO: add directory cleanup once multiple brokers are supported
- #try:
- # os.rmdir(data_dir)
- # os.rmdir(log_dir)
- #except:
- # log.info('rabbitmq directories already removed')
-
-
-# Kafka endpoint functions
-
-kafka_server = 'localhost'
-
-class KafkaReceiver(object):
- """class for receiving and storing messages on a topic from the kafka broker"""
- def __init__(self, topic, security_type):
- from kafka import KafkaConsumer
- remaining_retries = 10
- port = 9092
- if security_type != 'PLAINTEXT':
- security_type = 'SSL'
- port = 9093
- while remaining_retries > 0:
- try:
- self.consumer = KafkaConsumer(topic, bootstrap_servers = kafka_server+':'+str(port), security_protocol=security_type)
- print('Kafka consumer created on topic: '+topic)
- break
- except Exception as error:
- remaining_retries -= 1
- print('failed to connect to kafka (remaining retries '
- + str(remaining_retries) + '): ' + str(error))
- time.sleep(1)
-
- if remaining_retries == 0:
- raise Exception('failed to connect to kafka - no retries left')
-
- self.events = []
- self.topic = topic
- self.stop = False
-
- def verify_s3_events(self, keys, exact_match=False, deletions=False):
- """verify stored s3 records agains a list of keys"""
- verify_s3_records_by_elements(self.events, keys, exact_match=exact_match, deletions=deletions)
- self.events = []
-
-
-def kafka_receiver_thread_runner(receiver):
- """main thread function for the kafka receiver"""
- try:
- log.info('Kafka receiver started')
- print('Kafka receiver started')
- while not receiver.stop:
- for msg in receiver.consumer:
- receiver.events.append(json.loads(msg.value))
- timer.sleep(0.1)
- log.info('Kafka receiver ended')
- print('Kafka receiver ended')
- except Exception as error:
- log.info('Kafka receiver ended unexpectedly: %s', str(error))
- print('Kafka receiver ended unexpectedly: ' + str(error))
-
-
-def create_kafka_receiver_thread(topic, security_type='PLAINTEXT'):
- """create kafka receiver and thread"""
- receiver = KafkaReceiver(topic, security_type)
- task = threading.Thread(target=kafka_receiver_thread_runner, args=(receiver,))
- task.daemon = True
- return task, receiver
-
-def stop_kafka_receiver(receiver, task):
- """stop the receiver thread and wait for it to finis"""
- receiver.stop = True
- task.join(1)
- try:
- receiver.consumer.close()
- except Exception as error:
- log.info('failed to gracefuly stop Kafka receiver: %s', str(error))
-
-
-# follow the instruction here to create and sign a broker certificate:
-# https://github.com/edenhill/librdkafka/wiki/Using-SSL-with-librdkafka
-
-# the generated broker certificate should be stored in the java keystore for the use of the server
-# assuming the jks files were copied to $KAFKA_DIR and broker name is "localhost"
-# following lines must be added to $KAFKA_DIR/config/server.properties
-# listeners=PLAINTEXT://localhost:9092,SSL://localhost:9093,SASL_SSL://localhost:9094
-# sasl.enabled.mechanisms=PLAIN
-# ssl.keystore.location = $KAFKA_DIR/server.keystore.jks
-# ssl.keystore.password = abcdefgh
-# ssl.key.password = abcdefgh
-# ssl.truststore.location = $KAFKA_DIR/server.truststore.jks
-# ssl.truststore.password = abcdefgh
-
-# notes:
-# (1) we dont test client authentication, hence, no need to generate client keys
-# (2) our client is not using the keystore, and the "rootCA.crt" file generated in the process above
-# should be copied to: $KAFKA_DIR
-
-def init_kafka():
- """ start kafka/zookeeper """
- try:
- KAFKA_DIR = os.environ['KAFKA_DIR']
- except:
- KAFKA_DIR = ''
-
- if KAFKA_DIR == '':
- log.info('KAFKA_DIR must be set to where kafka is installed')
- print('KAFKA_DIR must be set to where kafka is installed')
- return None, None, None
-
- DEVNULL = open(os.devnull, 'wb')
-
- print('\nStarting zookeeper...')
- try:
- zk_proc = subprocess.Popen([KAFKA_DIR+'bin/zookeeper-server-start.sh', KAFKA_DIR+'config/zookeeper.properties'], stdout=DEVNULL)
- except Exception as error:
- log.info('failed to execute zookeeper: %s', str(error))
- print('failed to execute zookeeper: %s' % str(error))
- return None, None, None
-
- time.sleep(5)
- if zk_proc.poll() is not None:
- print('zookeeper failed to start')
- return None, None, None
- print('Zookeeper started')
- print('Starting kafka...')
- kafka_log = open('./kafka.log', 'w')
- try:
- kafka_env = os.environ.copy()
- kafka_env['KAFKA_OPTS']='-Djava.security.auth.login.config='+KAFKA_DIR+'config/kafka_server_jaas.conf'
- kafka_proc = subprocess.Popen([
- KAFKA_DIR+'bin/kafka-server-start.sh',
- KAFKA_DIR+'config/server.properties'],
- stdout=kafka_log,
- env=kafka_env)
- except Exception as error:
- log.info('failed to execute kafka: %s', str(error))
- print('failed to execute kafka: %s' % str(error))
- zk_proc.terminate()
- kafka_log.close()
- return None, None, None
-
- # TODO add kafka checkpoint instead of sleep
- time.sleep(15)
- if kafka_proc.poll() is not None:
- zk_proc.terminate()
- print('kafka failed to start. details in: ./kafka.log')
- kafka_log.close()
- return None, None, None
-
- print('Kafka started')
- return kafka_proc, zk_proc, kafka_log
-
-
-def clean_kafka(kafka_proc, zk_proc, kafka_log):
- """ stop kafka/zookeeper """
- try:
- kafka_log.close()
- print('Shutdown Kafka...')
- kafka_proc.terminate()
- time.sleep(5)
- if kafka_proc.poll() is None:
- print('Failed to shutdown Kafka... killing')
- kafka_proc.kill()
- print('Shutdown zookeeper...')
- zk_proc.terminate()
- time.sleep(5)
- if zk_proc.poll() is None:
- print('Failed to shutdown zookeeper... killing')
- zk_proc.kill()
- except:
- log.info('kafka/zookeeper already terminated')
-
-
-def init_env(require_ps=True):
- """initialize the environment"""
- if require_ps:
- check_ps_configured()
-
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
- zonegroup_conns = ZonegroupConns(zonegroup)
-
- zonegroup_meta_checkpoint(zonegroup)
-
- ps_zone = None
- master_zone = None
- for conn in zonegroup_conns.zones:
- if conn.zone == zonegroup.master_zone:
- master_zone = conn
- if is_ps_zone(conn):
- zone_meta_checkpoint(conn.zone)
- ps_zone = conn
-
- assert_not_equal(master_zone, None)
- if require_ps:
- assert_not_equal(ps_zone, None)
- return master_zone, ps_zone
-
-
-def get_ip():
- """ This method returns the "primary" IP on the local box (the one with a default route)
- source: https://stackoverflow.com/a/28950776/711085
- this is needed because on the teuthology machines: socket.getfqdn()/socket.gethostname() return 127.0.0.1 """
- s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
- try:
- # address should not be reachable
- s.connect(('10.255.255.255', 1))
- ip = s.getsockname()[0]
- finally:
- s.close()
- return ip
-
-
-TOPIC_SUFFIX = "_topic"
-SUB_SUFFIX = "_sub"
-NOTIFICATION_SUFFIX = "_notif"
-
-##############
-# pubsub tests
-##############
-
-def test_ps_info():
- """ log information for manual testing """
- return SkipTest("only used in manual testing")
- master_zone, ps_zone = init_env()
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
- bucket_name = gen_bucket_name()
- # create bucket on the first of the rados zones
- bucket = master_zone.create_bucket(bucket_name)
- # create objects in the bucket
- number_of_objects = 10
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- key.set_contents_from_string('bar')
- print('Zonegroup: ' + zonegroup.name)
- print('user: ' + get_user())
- print('tenant: ' + get_tenant())
- print('Master Zone')
- print_connection_info(master_zone.conn)
- print('PubSub Zone')
- print_connection_info(ps_zone.conn)
- print('Bucket: ' + bucket_name)
-
-
-def test_ps_s3_notification_low_level():
- """ test low level implementation of s3 notifications """
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- # create bucket on the first of the rados zones
- master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- # create topic
- topic_name = bucket_name + TOPIC_SUFFIX
- topic_conf = PSTopic(ps_zone.conn, topic_name)
- result, status = topic_conf.set_config()
- assert_equal(status/100, 2)
- parsed_result = json.loads(result)
- topic_arn = parsed_result['arn']
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- generated_topic_name = notification_name+'_'+topic_name
- topic_conf_list = [{'Id': notification_name,
- 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:*']
- }]
- s3_notification_conf = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
- _, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
- zone_meta_checkpoint(ps_zone.zone)
- # get auto-generated topic
- generated_topic_conf = PSTopic(ps_zone.conn, generated_topic_name)
- result, status = generated_topic_conf.get_config()
- parsed_result = json.loads(result)
- assert_equal(status/100, 2)
- assert_equal(parsed_result['topic']['name'], generated_topic_name)
- # get auto-generated notification
- notification_conf = PSNotification(ps_zone.conn, bucket_name,
- generated_topic_name)
- result, status = notification_conf.get_config()
- parsed_result = json.loads(result)
- assert_equal(status/100, 2)
- assert_equal(len(parsed_result['topics']), 1)
- # get auto-generated subscription
- sub_conf = PSSubscription(ps_zone.conn, notification_name,
- generated_topic_name)
- result, status = sub_conf.get_config()
- parsed_result = json.loads(result)
- assert_equal(status/100, 2)
- assert_equal(parsed_result['topic'], generated_topic_name)
- # delete s3 notification
- _, status = s3_notification_conf.del_config(notification=notification_name)
- assert_equal(status/100, 2)
- # delete topic
- _, status = topic_conf.del_config()
- assert_equal(status/100, 2)
-
- # verify low-level cleanup
- _, status = generated_topic_conf.get_config()
- assert_equal(status, 404)
- result, status = notification_conf.get_config()
- parsed_result = json.loads(result)
- assert_equal(len(parsed_result['topics']), 0)
- # TODO should return 404
- # assert_equal(status, 404)
- result, status = sub_conf.get_config()
- parsed_result = json.loads(result)
- assert_equal(parsed_result['topic'], '')
- # TODO should return 404
- # assert_equal(status, 404)
-
- # cleanup
- topic_conf.del_config()
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
-
-
-def test_ps_s3_notification_records():
- """ test s3 records fetching """
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- # create bucket on the first of the rados zones
- bucket = master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- # create topic
- topic_name = bucket_name + TOPIC_SUFFIX
- topic_conf = PSTopic(ps_zone.conn, topic_name)
- result, status = topic_conf.set_config()
- assert_equal(status/100, 2)
- parsed_result = json.loads(result)
- topic_arn = parsed_result['arn']
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name,
- 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:*']
- }]
- s3_notification_conf = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
- _, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
- zone_meta_checkpoint(ps_zone.zone)
- # get auto-generated subscription
- sub_conf = PSSubscription(ps_zone.conn, notification_name,
- topic_name)
- _, status = sub_conf.get_config()
- assert_equal(status/100, 2)
- # create objects in the bucket
- number_of_objects = 10
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- key.set_contents_from_string('bar')
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
-
- # get the events from the subscription
- result, _ = sub_conf.get_events()
- records = json.loads(result)
- for record in records['Records']:
- log.debug(record)
- keys = list(bucket.list())
- # TODO: use exact match
- verify_s3_records_by_elements(records, keys, exact_match=False)
-
- # cleanup
- _, status = s3_notification_conf.del_config()
- topic_conf.del_config()
- # delete the keys
- for key in bucket.list():
- key.delete()
- master_zone.delete_bucket(bucket_name)
-
-
-def test_ps_s3_notification():
- """ test s3 notification set/get/delete """
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- # create bucket on the first of the rados zones
- master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- topic_name = bucket_name + TOPIC_SUFFIX
- # create topic
- topic_name = bucket_name + TOPIC_SUFFIX
- topic_conf = PSTopic(ps_zone.conn, topic_name)
- response, status = topic_conf.set_config()
- assert_equal(status/100, 2)
- parsed_result = json.loads(response)
- topic_arn = parsed_result['arn']
- # create one s3 notification
- notification_name1 = bucket_name + NOTIFICATION_SUFFIX + '_1'
- topic_conf_list = [{'Id': notification_name1,
- 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:*']
- }]
- s3_notification_conf1 = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf1.set_config()
- assert_equal(status/100, 2)
- # create another s3 notification with the same topic
- notification_name2 = bucket_name + NOTIFICATION_SUFFIX + '_2'
- topic_conf_list = [{'Id': notification_name2,
- 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:*', 's3:ObjectRemoved:*']
- }]
- s3_notification_conf2 = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf2.set_config()
- assert_equal(status/100, 2)
- zone_meta_checkpoint(ps_zone.zone)
-
- # get all notification on a bucket
- response, status = s3_notification_conf1.get_config()
- assert_equal(status/100, 2)
- assert_equal(len(response['TopicConfigurations']), 2)
- assert_equal(response['TopicConfigurations'][0]['TopicArn'], topic_arn)
- assert_equal(response['TopicConfigurations'][1]['TopicArn'], topic_arn)
-
- # get specific notification on a bucket
- response, status = s3_notification_conf1.get_config(notification=notification_name1)
- assert_equal(status/100, 2)
- assert_equal(response['NotificationConfiguration']['TopicConfiguration']['Topic'], topic_arn)
- assert_equal(response['NotificationConfiguration']['TopicConfiguration']['Id'], notification_name1)
- response, status = s3_notification_conf2.get_config(notification=notification_name2)
- assert_equal(status/100, 2)
- assert_equal(response['NotificationConfiguration']['TopicConfiguration']['Topic'], topic_arn)
- assert_equal(response['NotificationConfiguration']['TopicConfiguration']['Id'], notification_name2)
-
- # delete specific notifications
- _, status = s3_notification_conf1.del_config(notification=notification_name1)
- assert_equal(status/100, 2)
- _, status = s3_notification_conf2.del_config(notification=notification_name2)
- assert_equal(status/100, 2)
-
- # cleanup
- topic_conf.del_config()
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
-
-
-def test_ps_s3_topic_on_master():
- """ test s3 topics set/get/delete on master """
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name + TOPIC_SUFFIX
-
- # clean all topics
- delete_all_s3_topics(master_zone, zonegroup.name)
-
- # create s3 topics
- endpoint_address = 'amqp://127.0.0.1:7001/vhost_1'
- endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=amqp.direct&amqp-ack-level=none'
- topic_conf1 = PSTopicS3(master_zone.conn, topic_name+'_1', zonegroup.name, endpoint_args=endpoint_args)
- topic_arn = topic_conf1.set_config()
- assert_equal(topic_arn,
- 'arn:aws:sns:' + zonegroup.name + ':' + get_tenant() + ':' + topic_name + '_1')
-
- endpoint_address = 'http://127.0.0.1:9001'
- endpoint_args = 'push-endpoint='+endpoint_address
- topic_conf2 = PSTopicS3(master_zone.conn, topic_name+'_2', zonegroup.name, endpoint_args=endpoint_args)
- topic_arn = topic_conf2.set_config()
- assert_equal(topic_arn,
- 'arn:aws:sns:' + zonegroup.name + ':' + get_tenant() + ':' + topic_name + '_2')
- endpoint_address = 'http://127.0.0.1:9002'
- endpoint_args = 'push-endpoint='+endpoint_address
- topic_conf3 = PSTopicS3(master_zone.conn, topic_name+'_3', zonegroup.name, endpoint_args=endpoint_args)
- topic_arn = topic_conf3.set_config()
- assert_equal(topic_arn,
- 'arn:aws:sns:' + zonegroup.name + ':' + get_tenant() + ':' + topic_name + '_3')
-
- # get topic 3
- result, status = topic_conf3.get_config()
- assert_equal(status, 200)
- assert_equal(topic_arn, result['GetTopicResponse']['GetTopicResult']['Topic']['TopicArn'])
- assert_equal(endpoint_address, result['GetTopicResponse']['GetTopicResult']['Topic']['EndPoint']['EndpointAddress'])
- # Note that endpoint args may be ordered differently in the result
- result = topic_conf3.get_attributes()
- assert_equal(topic_arn, result['Attributes']['TopicArn'])
- json_endpoint = json.loads(result['Attributes']['EndPoint'])
- assert_equal(endpoint_address, json_endpoint['EndpointAddress'])
-
- # delete topic 1
- result = topic_conf1.del_config()
- assert_equal(status, 200)
-
- # try to get a deleted topic
- _, status = topic_conf1.get_config()
- assert_equal(status, 404)
- try:
- topic_conf1.get_attributes()
- except:
- print('topic already deleted - this is expected')
- else:
- assert False, 'topic 1 should be deleted at this point'
-
- # get the remaining 2 topics
- result, status = topic_conf1.get_list()
- assert_equal(status, 200)
- assert_equal(len(result['ListTopicsResponse']['ListTopicsResult']['Topics']['member']), 2)
-
- # delete topics
- result = topic_conf2.del_config()
- # TODO: should be 200OK
- # assert_equal(status, 200)
- result = topic_conf3.del_config()
- # TODO: should be 200OK
- # assert_equal(status, 200)
-
- # get topic list, make sure it is empty
- result, status = topic_conf1.get_list()
- assert_equal(result['ListTopicsResponse']['ListTopicsResult']['Topics'], None)
-
-
-def test_ps_s3_topic_with_secret_on_master():
- """ test s3 topics with secret set/get/delete on master """
- master_zone, _ = init_env(require_ps=False)
- if master_zone.secure_conn is None:
- return SkipTest('secure connection is needed to test topic with secrets')
-
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name + TOPIC_SUFFIX
-
- # clean all topics
- delete_all_s3_topics(master_zone, zonegroup.name)
-
- # create s3 topics
- endpoint_address = 'amqp://user:password@127.0.0.1:7001'
- endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=amqp.direct&amqp-ack-level=none'
- bad_topic_conf = PSTopicS3(master_zone.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
- try:
- result = bad_topic_conf.set_config()
- except Exception as err:
- print('Error is expected: ' + str(err))
- else:
- assert False, 'user password configuration set allowed only over HTTPS'
-
- topic_conf = PSTopicS3(master_zone.secure_conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
- topic_arn = topic_conf.set_config()
-
- assert_equal(topic_arn,
- 'arn:aws:sns:' + zonegroup.name + ':' + get_tenant() + ':' + topic_name)
-
- _, status = bad_topic_conf.get_config()
- assert_equal(status/100, 4)
-
- # get topic
- result, status = topic_conf.get_config()
- assert_equal(status, 200)
- assert_equal(topic_arn, result['GetTopicResponse']['GetTopicResult']['Topic']['TopicArn'])
- assert_equal(endpoint_address, result['GetTopicResponse']['GetTopicResult']['Topic']['EndPoint']['EndpointAddress'])
-
- _, status = bad_topic_conf.get_config()
- assert_equal(status/100, 4)
-
- _, status = topic_conf.get_list()
- assert_equal(status/100, 2)
-
- # delete topics
- result = topic_conf.del_config()
-
-
-def test_ps_s3_notification_on_master():
- """ test s3 notification set/get/delete on master """
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
- bucket_name = gen_bucket_name()
- # create bucket
- bucket = master_zone.create_bucket(bucket_name)
- topic_name = bucket_name + TOPIC_SUFFIX
- # create s3 topic
- endpoint_address = 'amqp://127.0.0.1:7001'
- endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=amqp.direct&amqp-ack-level=none'
- topic_conf1 = PSTopicS3(master_zone.conn, topic_name+'_1', zonegroup.name, endpoint_args=endpoint_args)
- topic_arn1 = topic_conf1.set_config()
- topic_conf2 = PSTopicS3(master_zone.conn, topic_name+'_2', zonegroup.name, endpoint_args=endpoint_args)
- topic_arn2 = topic_conf2.set_config()
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name+'_1',
- 'TopicArn': topic_arn1,
- 'Events': ['s3:ObjectCreated:*']
- },
- {'Id': notification_name+'_2',
- 'TopicArn': topic_arn1,
- 'Events': ['s3:ObjectRemoved:*']
- },
- {'Id': notification_name+'_3',
- 'TopicArn': topic_arn1,
- 'Events': []
- }]
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- _, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # get notifications on a bucket
- response, status = s3_notification_conf.get_config(notification=notification_name+'_1')
- assert_equal(status/100, 2)
- assert_equal(response['NotificationConfiguration']['TopicConfiguration']['Topic'], topic_arn1)
-
- # delete specific notifications
- _, status = s3_notification_conf.del_config(notification=notification_name+'_1')
- assert_equal(status/100, 2)
-
- # get the remaining 2 notifications on a bucket
- response, status = s3_notification_conf.get_config()
- assert_equal(status/100, 2)
- assert_equal(len(response['TopicConfigurations']), 2)
- assert_equal(response['TopicConfigurations'][0]['TopicArn'], topic_arn1)
- assert_equal(response['TopicConfigurations'][1]['TopicArn'], topic_arn1)
-
- # delete remaining notifications
- _, status = s3_notification_conf.del_config()
- assert_equal(status/100, 2)
-
- # make sure that the notifications are now deleted
- response, status = s3_notification_conf.get_config()
- try:
- dummy = response['TopicConfigurations']
- except:
- print('"TopicConfigurations" is not in response')
- else:
- assert False, '"TopicConfigurations" should not be in response'
-
- # create another s3 notification
- topic_conf_list = [{'Id': notification_name+'_1',
- 'TopicArn': topic_arn1,
- 'Events': ['s3:ObjectCreated:*']
- }]
- _, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # make sure the notification and auto-genrated topic are deleted
- response, status = topic_conf1.get_list()
- topics = response['ListTopicsResponse']['ListTopicsResult']['Topics']['member']
- before_delete = len(topics)
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
- response, status = topic_conf2.get_list()
- topics = response['ListTopicsResponse']['ListTopicsResult']['Topics']['member']
- after_delete = len(topics)
- assert_equal(before_delete - after_delete, 3)
-
- # cleanup
- topic_conf1.del_config()
- topic_conf2.del_config()
-
-
-def ps_s3_notification_filter(on_master):
- """ test s3 notification filter on master """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- hostname = get_ip()
- proc = init_rabbitmq()
- if proc is None:
- return SkipTest('end2end amqp tests require rabbitmq-server installed')
- if on_master:
- master_zone, _ = init_env(require_ps=False)
- ps_zone = master_zone
- else:
- master_zone, ps_zone = init_env(require_ps=True)
- ps_zone = ps_zone
-
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- topic_name = bucket_name + TOPIC_SUFFIX
-
- # start amqp receivers
- exchange = 'ex1'
- task, receiver = create_amqp_receiver_thread(exchange, topic_name)
- task.start()
-
- # create s3 topic
- endpoint_address = 'amqp://' + hostname
- endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=' + exchange +'&amqp-ack-level=broker'
- if on_master:
- topic_conf = PSTopicS3(ps_zone.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
- topic_arn = topic_conf.set_config()
- else:
- topic_conf = PSTopic(ps_zone.conn, topic_name, endpoint=endpoint_address, endpoint_args=endpoint_args)
- result, _ = topic_conf.set_config()
- parsed_result = json.loads(result)
- topic_arn = parsed_result['arn']
- zone_meta_checkpoint(ps_zone.zone)
-
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name+'_1',
- 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:*'],
- 'Filter': {
- 'Key': {
- 'FilterRules': [{'Name': 'prefix', 'Value': 'hello'}]
- }
- }
- },
- {'Id': notification_name+'_2',
- 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:*'],
- 'Filter': {
- 'Key': {
- 'FilterRules': [{'Name': 'prefix', 'Value': 'world'},
- {'Name': 'suffix', 'Value': 'log'}]
- }
- }
- },
- {'Id': notification_name+'_3',
- 'TopicArn': topic_arn,
- 'Events': [],
- 'Filter': {
- 'Key': {
- 'FilterRules': [{'Name': 'regex', 'Value': '([a-z]+)\\.txt'}]
- }
- }
- }]
-
- s3_notification_conf = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
- result, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- if on_master:
- topic_conf_list = [{'Id': notification_name+'_4',
- 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:*', 's3:ObjectRemoved:*'],
- 'Filter': {
- 'Metadata': {
- 'FilterRules': [{'Name': 'x-amz-meta-foo', 'Value': 'bar'},
- {'Name': 'x-amz-meta-hello', 'Value': 'world'}]
- },
- 'Key': {
- 'FilterRules': [{'Name': 'regex', 'Value': '([a-z]+)'}]
- }
- }
- }]
-
- try:
- s3_notification_conf4 = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
- _, status = s3_notification_conf4.set_config()
- assert_equal(status/100, 2)
- skip_notif4 = False
- except Exception as error:
- print('note: metadata filter is not supported by boto3 - skipping test')
- skip_notif4 = True
- else:
- print('filtering by attributes only supported on master zone')
- skip_notif4 = True
-
-
- # get all notifications
- result, status = s3_notification_conf.get_config()
- assert_equal(status/100, 2)
- for conf in result['TopicConfigurations']:
- filter_name = conf['Filter']['Key']['FilterRules'][0]['Name']
- assert filter_name == 'prefix' or filter_name == 'suffix' or filter_name == 'regex', filter_name
-
- if not skip_notif4:
- result, status = s3_notification_conf4.get_config(notification=notification_name+'_4')
- assert_equal(status/100, 2)
- filter_name = result['NotificationConfiguration']['TopicConfiguration']['Filter']['S3Metadata']['FilterRule'][0]['Name']
- assert filter_name == 'x-amz-meta-foo' or filter_name == 'x-amz-meta-hello'
-
- expected_in1 = ['hello.kaboom', 'hello.txt', 'hello123.txt', 'hello']
- expected_in2 = ['world1.log', 'world2log', 'world3.log']
- expected_in3 = ['hello.txt', 'hell.txt', 'worldlog.txt']
- expected_in4 = ['foo', 'bar', 'hello', 'world']
- filtered = ['hell.kaboom', 'world.og', 'world.logg', 'he123ll.txt', 'wo', 'log', 'h', 'txt', 'world.log.txt']
- filtered_with_attr = ['nofoo', 'nobar', 'nohello', 'noworld']
- # create objects in bucket
- for key_name in expected_in1:
- key = bucket.new_key(key_name)
- key.set_contents_from_string('bar')
- for key_name in expected_in2:
- key = bucket.new_key(key_name)
- key.set_contents_from_string('bar')
- for key_name in expected_in3:
- key = bucket.new_key(key_name)
- key.set_contents_from_string('bar')
- if not skip_notif4:
- for key_name in expected_in4:
- key = bucket.new_key(key_name)
- key.set_metadata('foo', 'bar')
- key.set_metadata('hello', 'world')
- key.set_metadata('goodbye', 'cruel world')
- key.set_contents_from_string('bar')
- for key_name in filtered:
- key = bucket.new_key(key_name)
- key.set_contents_from_string('bar')
- for key_name in filtered_with_attr:
- key.set_metadata('foo', 'nobar')
- key.set_metadata('hello', 'noworld')
- key.set_metadata('goodbye', 'cruel world')
- key = bucket.new_key(key_name)
- key.set_contents_from_string('bar')
-
- if on_master:
- print('wait for 5sec for the messages...')
- time.sleep(5)
- else:
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
-
- found_in1 = []
- found_in2 = []
- found_in3 = []
- found_in4 = []
-
- for event in receiver.get_and_reset_events():
- notif_id = event['Records'][0]['s3']['configurationId']
- key_name = event['Records'][0]['s3']['object']['key']
- if notif_id == notification_name+'_1':
- found_in1.append(key_name)
- elif notif_id == notification_name+'_2':
- found_in2.append(key_name)
- elif notif_id == notification_name+'_3':
- found_in3.append(key_name)
- elif not skip_notif4 and notif_id == notification_name+'_4':
- found_in4.append(key_name)
- else:
- assert False, 'invalid notification: ' + notif_id
-
- assert_equal(set(found_in1), set(expected_in1))
- assert_equal(set(found_in2), set(expected_in2))
- assert_equal(set(found_in3), set(expected_in3))
- if not skip_notif4:
- assert_equal(set(found_in4), set(expected_in4))
-
- # cleanup
- s3_notification_conf.del_config()
- if not skip_notif4:
- s3_notification_conf4.del_config()
- topic_conf.del_config()
- # delete the bucket
- for key in bucket.list():
- key.delete()
- master_zone.delete_bucket(bucket_name)
- stop_amqp_receiver(receiver, task)
- clean_rabbitmq(proc)
-
-
-def test_ps_s3_notification_filter_on_master():
- ps_s3_notification_filter(on_master=True)
-
-
-def test_ps_s3_notification_filter():
- ps_s3_notification_filter(on_master=False)
-
-
-def test_ps_s3_notification_errors_on_master():
- """ test s3 notification set/get/delete on master """
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
- bucket_name = gen_bucket_name()
- # create bucket
- bucket = master_zone.create_bucket(bucket_name)
- topic_name = bucket_name + TOPIC_SUFFIX
- # create s3 topic
- endpoint_address = 'amqp://127.0.0.1:7001'
- endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=amqp.direct&amqp-ack-level=none'
- topic_conf = PSTopicS3(master_zone.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
- topic_arn = topic_conf.set_config()
-
- # create s3 notification with invalid event name
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name,
- 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:Kaboom']
- }]
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- try:
- result, status = s3_notification_conf.set_config()
- except Exception as error:
- print(str(error) + ' - is expected')
- else:
- assert False, 'invalid event name is expected to fail'
-
- # create s3 notification with missing name
- topic_conf_list = [{'Id': '',
- 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:Put']
- }]
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- try:
- _, _ = s3_notification_conf.set_config()
- except Exception as error:
- print(str(error) + ' - is expected')
- else:
- assert False, 'missing notification name is expected to fail'
-
- # create s3 notification with invalid topic ARN
- invalid_topic_arn = 'kaboom'
- topic_conf_list = [{'Id': notification_name,
- 'TopicArn': invalid_topic_arn,
- 'Events': ['s3:ObjectCreated:Put']
- }]
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- try:
- _, _ = s3_notification_conf.set_config()
- except Exception as error:
- print(str(error) + ' - is expected')
- else:
- assert False, 'invalid ARN is expected to fail'
-
- # create s3 notification with unknown topic ARN
- invalid_topic_arn = 'arn:aws:sns:a::kaboom'
- topic_conf_list = [{'Id': notification_name,
- 'TopicArn': invalid_topic_arn ,
- 'Events': ['s3:ObjectCreated:Put']
- }]
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- try:
- _, _ = s3_notification_conf.set_config()
- except Exception as error:
- print(str(error) + ' - is expected')
- else:
- assert False, 'unknown topic is expected to fail'
-
- # create s3 notification with wrong bucket
- topic_conf_list = [{'Id': notification_name,
- 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:Put']
- }]
- s3_notification_conf = PSNotificationS3(master_zone.conn, 'kaboom', topic_conf_list)
- try:
- _, _ = s3_notification_conf.set_config()
- except Exception as error:
- print(str(error) + ' - is expected')
- else:
- assert False, 'unknown bucket is expected to fail'
-
- topic_conf.del_config()
-
- status = topic_conf.del_config()
- # deleting an unknown notification is not considered an error
- assert_equal(status, 200)
-
- _, status = topic_conf.get_config()
- assert_equal(status, 404)
-
- # cleanup
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
-
-
-def test_objcet_timing():
- return SkipTest("only used in manual testing")
- master_zone, _ = init_env(require_ps=False)
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- # create objects in the bucket (async)
- print('creating objects...')
- number_of_objects = 1000
- client_threads = []
- start_time = time.time()
- content = str(bytearray(os.urandom(1024*1024)))
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- time_diff = time.time() - start_time
- print('average time for object creation: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
-
- print('total number of objects: ' + str(len(list(bucket.list()))))
-
- print('deleting objects...')
- client_threads = []
- start_time = time.time()
- for key in bucket.list():
- thr = threading.Thread(target = key.delete, args=())
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- time_diff = time.time() - start_time
- print('average time for object deletion: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
-
- # cleanup
- master_zone.delete_bucket(bucket_name)
-
-
-def test_ps_s3_notification_push_amqp_on_master():
- """ test pushing amqp s3 notification on master """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- hostname = get_ip()
- proc = init_rabbitmq()
- if proc is None:
- return SkipTest('end2end amqp tests require rabbitmq-server installed')
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- topic_name1 = bucket_name + TOPIC_SUFFIX + '_1'
- topic_name2 = bucket_name + TOPIC_SUFFIX + '_2'
-
- # start amqp receivers
- exchange = 'ex1'
- task1, receiver1 = create_amqp_receiver_thread(exchange, topic_name1)
- task2, receiver2 = create_amqp_receiver_thread(exchange, topic_name2)
- task1.start()
- task2.start()
-
- # create two s3 topic
- endpoint_address = 'amqp://' + hostname
- # with acks from broker
- endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=' + exchange +'&amqp-ack-level=broker'
- topic_conf1 = PSTopicS3(master_zone.conn, topic_name1, zonegroup.name, endpoint_args=endpoint_args)
- topic_arn1 = topic_conf1.set_config()
- # without acks from broker
- endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=' + exchange +'&amqp-ack-level=routable'
- topic_conf2 = PSTopicS3(master_zone.conn, topic_name2, zonegroup.name, endpoint_args=endpoint_args)
- topic_arn2 = topic_conf2.set_config()
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name+'_1', 'TopicArn': topic_arn1,
- 'Events': []
- },
- {'Id': notification_name+'_2', 'TopicArn': topic_arn2,
- 'Events': ['s3:ObjectCreated:*']
- }]
-
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # create objects in the bucket (async)
- number_of_objects = 100
- client_threads = []
- start_time = time.time()
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- content = str(os.urandom(1024*1024))
- thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- time_diff = time.time() - start_time
- print('average time for creation + qmqp notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
-
- print('wait for 5sec for the messages...')
- time.sleep(5)
-
- # check amqp receiver
- keys = list(bucket.list())
- print('total number of objects: ' + str(len(keys)))
- receiver1.verify_s3_events(keys, exact_match=True)
- receiver2.verify_s3_events(keys, exact_match=True)
-
- # delete objects from the bucket
- client_threads = []
- start_time = time.time()
- for key in bucket.list():
- thr = threading.Thread(target = key.delete, args=())
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- time_diff = time.time() - start_time
- print('average time for deletion + amqp notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
-
- print('wait for 5sec for the messages...')
- time.sleep(5)
-
- # check amqp receiver 1 for deletions
- receiver1.verify_s3_events(keys, exact_match=True, deletions=True)
- # check amqp receiver 2 has no deletions
- try:
- receiver1.verify_s3_events(keys, exact_match=False, deletions=True)
- except:
- pass
- else:
- err = 'amqp receiver 2 should have no deletions'
- assert False, err
-
- # cleanup
- stop_amqp_receiver(receiver1, task1)
- stop_amqp_receiver(receiver2, task2)
- s3_notification_conf.del_config()
- topic_conf1.del_config()
- topic_conf2.del_config()
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
- clean_rabbitmq(proc)
-
-
-def test_ps_s3_persistent_cleanup():
- """ test reservation cleanup after gateway crash """
- return SkipTest("only used in manual testing")
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create random port for the http server
- host = get_ip()
- port = random.randint(10000, 20000)
- # start an http server in a separate thread
- number_of_objects = 200
- http_server = StreamingHTTPServer(host, port, num_workers=number_of_objects)
-
- gw = master_zone
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = gw.create_bucket(bucket_name)
- topic_name = bucket_name + TOPIC_SUFFIX
-
- # create s3 topic
- endpoint_address = 'http://'+host+':'+str(port)
- endpoint_args = 'push-endpoint='+endpoint_address+'&persistent=true'
- topic_conf = PSTopicS3(gw.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
- topic_arn = topic_conf.set_config()
-
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:Put']
- }]
- s3_notification_conf = PSNotificationS3(gw.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- client_threads = []
- start_time = time.time()
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- content = str(os.urandom(1024*1024))
- thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
- thr.start()
- client_threads.append(thr)
- # stop gateway while clients are sending
- os.system("killall -9 radosgw");
- zonegroup.master_zone.gateways[0].stop()
- print('wait for 10 sec for before restarting the gateway')
- time.sleep(10)
- zonegroup.master_zone.gateways[0].start()
- [thr.join() for thr in client_threads]
-
- keys = list(bucket.list())
-
- # delete objects from the bucket
- client_threads = []
- start_time = time.time()
- for key in bucket.list():
- thr = threading.Thread(target = key.delete, args=())
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- # check http receiver
- events = http_server.get_and_reset_events()
-
- print(str(len(events) ) + " events found out of " + str(number_of_objects))
-
- # make sure that things are working now
- client_threads = []
- start_time = time.time()
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- content = str(os.urandom(1024*1024))
- thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- keys = list(bucket.list())
-
- # delete objects from the bucket
- client_threads = []
- start_time = time.time()
- for key in bucket.list():
- thr = threading.Thread(target = key.delete, args=())
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- print('wait for 180 sec for reservations to be stale before queue deletion')
- time.sleep(180)
-
- # check http receiver
- events = http_server.get_and_reset_events()
-
- print(str(len(events)) + " events found out of " + str(number_of_objects))
-
- # cleanup
- s3_notification_conf.del_config()
- topic_conf.del_config()
- gw.delete_bucket(bucket_name)
- http_server.close()
-
-
-def test_ps_s3_persistent_gateways_recovery():
- """ test gateway recovery of persistent notifications """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
- if len(zonegroup.master_zone.gateways) < 2:
- return SkipTest("this test requires two gateways")
-
- # create random port for the http server
- host = get_ip()
- port = random.randint(10000, 20000)
- # start an http server in a separate thread
- number_of_objects = 10
- http_server = StreamingHTTPServer(host, port, num_workers=number_of_objects)
-
- gw1 = master_zone
- gw2 = zonegroup.master_zone.gateways[1]
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = gw1.create_bucket(bucket_name)
- topic_name = bucket_name + TOPIC_SUFFIX
-
- # create two s3 topics
- endpoint_address = 'http://'+host+':'+str(port)
- endpoint_args = 'push-endpoint='+endpoint_address+'&persistent=true'
- topic_conf1 = PSTopicS3(gw1.conn, topic_name+'_1', zonegroup.name, endpoint_args=endpoint_args+'&OpaqueData=fromgw1')
- topic_arn1 = topic_conf1.set_config()
- topic_conf2 = PSTopicS3(gw2.connection, topic_name+'_2', zonegroup.name, endpoint_args=endpoint_args+'&OpaqueData=fromgw2')
- topic_arn2 = topic_conf2.set_config()
-
- # create two s3 notifications
- notification_name = bucket_name + NOTIFICATION_SUFFIX+'_1'
- topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn1,
- 'Events': ['s3:ObjectCreated:Put']
- }]
- s3_notification_conf1 = PSNotificationS3(gw1.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf1.set_config()
- assert_equal(status/100, 2)
- notification_name = bucket_name + NOTIFICATION_SUFFIX+'_2'
- topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn2,
- 'Events': ['s3:ObjectRemoved:Delete']
- }]
- s3_notification_conf2 = PSNotificationS3(gw2.connection, bucket_name, topic_conf_list)
- response, status = s3_notification_conf2.set_config()
- assert_equal(status/100, 2)
-
- # stop gateway 2
- print('stopping gateway2...')
- gw2.stop()
-
- client_threads = []
- start_time = time.time()
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- content = str(os.urandom(1024*1024))
- thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- keys = list(bucket.list())
-
- # delete objects from the bucket
- client_threads = []
- start_time = time.time()
- for key in bucket.list():
- thr = threading.Thread(target = key.delete, args=())
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- print('wait for 60 sec for before restarting the gateway')
- time.sleep(60)
- gw2.start()
-
- # check http receiver
- events = http_server.get_and_reset_events()
- for key in keys:
- creations = 0
- deletions = 0
- for event in events:
- if event['Records'][0]['eventName'] == 's3:ObjectCreated:Put' and \
- key.name == event['Records'][0]['s3']['object']['key']:
- creations += 1
- elif event['Records'][0]['eventName'] == 's3:ObjectRemoved:Delete' and \
- key.name == event['Records'][0]['s3']['object']['key']:
- deletions += 1
- assert_equal(creations, 1)
- assert_equal(deletions, 1)
-
- # cleanup
- s3_notification_conf1.del_config()
- topic_conf1.del_config()
- gw1.delete_bucket(bucket_name)
- time.sleep(10)
- s3_notification_conf2.del_config()
- topic_conf2.del_config()
- http_server.close()
-
-
-def test_ps_s3_persistent_multiple_gateways():
- """ test pushing persistent notification via two gateways """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
- if len(zonegroup.master_zone.gateways) < 2:
- return SkipTest("this test requires two gateways")
-
- # create random port for the http server
- host = get_ip()
- port = random.randint(10000, 20000)
- # start an http server in a separate thread
- number_of_objects = 10
- http_server = StreamingHTTPServer(host, port, num_workers=number_of_objects)
-
- gw1 = master_zone
- gw2 = zonegroup.master_zone.gateways[1]
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket1 = gw1.create_bucket(bucket_name)
- bucket2 = gw2.connection.get_bucket(bucket_name)
- topic_name = bucket_name + TOPIC_SUFFIX
-
- # create two s3 topics
- endpoint_address = 'http://'+host+':'+str(port)
- endpoint_args = 'push-endpoint='+endpoint_address+'&persistent=true'
- topic1_opaque = 'fromgw1'
- topic_conf1 = PSTopicS3(gw1.conn, topic_name+'_1', zonegroup.name, endpoint_args=endpoint_args+'&OpaqueData='+topic1_opaque)
- topic_arn1 = topic_conf1.set_config()
- topic2_opaque = 'fromgw2'
- topic_conf2 = PSTopicS3(gw2.connection, topic_name+'_2', zonegroup.name, endpoint_args=endpoint_args+'&OpaqueData='+topic2_opaque)
- topic_arn2 = topic_conf2.set_config()
-
- # create two s3 notifications
- notification_name = bucket_name + NOTIFICATION_SUFFIX+'_1'
- topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn1,
- 'Events': []
- }]
- s3_notification_conf1 = PSNotificationS3(gw1.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf1.set_config()
- assert_equal(status/100, 2)
- notification_name = bucket_name + NOTIFICATION_SUFFIX+'_2'
- topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn2,
- 'Events': []
- }]
- s3_notification_conf2 = PSNotificationS3(gw2.connection, bucket_name, topic_conf_list)
- response, status = s3_notification_conf2.set_config()
- assert_equal(status/100, 2)
-
- client_threads = []
- start_time = time.time()
- for i in range(number_of_objects):
- key = bucket1.new_key('gw1_'+str(i))
- content = str(os.urandom(1024*1024))
- thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
- thr.start()
- client_threads.append(thr)
- key = bucket2.new_key('gw2_'+str(i))
- content = str(os.urandom(1024*1024))
- thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- keys = list(bucket1.list())
-
- delay = 30
- print('wait for '+str(delay)+'sec for the messages...')
- time.sleep(delay)
-
- events = http_server.get_and_reset_events()
- for key in keys:
- topic1_count = 0
- topic2_count = 0
- for event in events:
- if event['Records'][0]['eventName'] == 's3:ObjectCreated:Put' and \
- key.name == event['Records'][0]['s3']['object']['key'] and \
- topic1_opaque == event['Records'][0]['opaqueData']:
- topic1_count += 1
- elif event['Records'][0]['eventName'] == 's3:ObjectCreated:Put' and \
- key.name == event['Records'][0]['s3']['object']['key'] and \
- topic2_opaque == event['Records'][0]['opaqueData']:
- topic2_count += 1
- assert_equal(topic1_count, 1)
- assert_equal(topic2_count, 1)
-
- # delete objects from the bucket
- client_threads = []
- start_time = time.time()
- for key in bucket1.list():
- thr = threading.Thread(target = key.delete, args=())
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- print('wait for '+str(delay)+'sec for the messages...')
- time.sleep(delay)
-
- events = http_server.get_and_reset_events()
- for key in keys:
- topic1_count = 0
- topic2_count = 0
- for event in events:
- if event['Records'][0]['eventName'] == 's3:ObjectRemoved:Delete' and \
- key.name == event['Records'][0]['s3']['object']['key'] and \
- topic1_opaque == event['Records'][0]['opaqueData']:
- topic1_count += 1
- elif event['Records'][0]['eventName'] == 's3:ObjectRemoved:Delete' and \
- key.name == event['Records'][0]['s3']['object']['key'] and \
- topic2_opaque == event['Records'][0]['opaqueData']:
- topic2_count += 1
- assert_equal(topic1_count, 1)
- assert_equal(topic2_count, 1)
-
- # cleanup
- s3_notification_conf1.del_config()
- topic_conf1.del_config()
- s3_notification_conf2.del_config()
- topic_conf2.del_config()
- gw1.delete_bucket(bucket_name)
- http_server.close()
-
-
-def test_ps_s3_persistent_multiple_endpoints():
- """ test pushing persistent notification when one of the endpoints has error """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create random port for the http server
- host = get_ip()
- port = random.randint(10000, 20000)
- # start an http server in a separate thread
- number_of_objects = 10
- http_server = StreamingHTTPServer(host, port, num_workers=number_of_objects)
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- topic_name = bucket_name + TOPIC_SUFFIX
-
- # create two s3 topics
- endpoint_address = 'http://'+host+':'+str(port)
- endpoint_args = 'push-endpoint='+endpoint_address+'&persistent=true'
- topic_conf1 = PSTopicS3(master_zone.conn, topic_name+'_1', zonegroup.name, endpoint_args=endpoint_args)
- topic_arn1 = topic_conf1.set_config()
- endpoint_address = 'http://kaboom:9999'
- endpoint_args = 'push-endpoint='+endpoint_address+'&persistent=true'
- topic_conf2 = PSTopicS3(master_zone.conn, topic_name+'_2', zonegroup.name, endpoint_args=endpoint_args)
- topic_arn2 = topic_conf2.set_config()
-
- # create two s3 notifications
- notification_name = bucket_name + NOTIFICATION_SUFFIX+'_1'
- topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn1,
- 'Events': []
- }]
- s3_notification_conf1 = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf1.set_config()
- assert_equal(status/100, 2)
- notification_name = bucket_name + NOTIFICATION_SUFFIX+'_2'
- topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn2,
- 'Events': []
- }]
- s3_notification_conf2 = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf2.set_config()
- assert_equal(status/100, 2)
-
- client_threads = []
- start_time = time.time()
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- content = str(os.urandom(1024*1024))
- thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- keys = list(bucket.list())
-
- delay = 30
- print('wait for '+str(delay)+'sec for the messages...')
- time.sleep(delay)
-
- http_server.verify_s3_events(keys, exact_match=False, deletions=False)
-
- # delete objects from the bucket
- client_threads = []
- start_time = time.time()
- for key in bucket.list():
- thr = threading.Thread(target = key.delete, args=())
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- print('wait for '+str(delay)+'sec for the messages...')
- time.sleep(delay)
-
- http_server.verify_s3_events(keys, exact_match=False, deletions=True)
-
- # cleanup
- s3_notification_conf1.del_config()
- topic_conf1.del_config()
- s3_notification_conf2.del_config()
- topic_conf2.del_config()
- master_zone.delete_bucket(bucket_name)
- http_server.close()
-
-
-def persistent_notification(endpoint_type):
- """ test pushing persistent notification """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- topic_name = bucket_name + TOPIC_SUFFIX
-
- receiver = {}
- host = get_ip()
- if endpoint_type == 'http':
- # create random port for the http server
- port = random.randint(10000, 20000)
- # start an http server in a separate thread
- receiver = StreamingHTTPServer(host, port, num_workers=10)
- endpoint_address = 'http://'+host+':'+str(port)
- endpoint_args = 'push-endpoint='+endpoint_address+'&persistent=true'
- # the http server does not guarantee order, so duplicates are expected
- exact_match = False
- elif endpoint_type == 'amqp':
- proc = init_rabbitmq()
- if proc is None:
- return SkipTest('end2end amqp tests require rabbitmq-server installed')
- # start amqp receiver
- exchange = 'ex1'
- task, receiver = create_amqp_receiver_thread(exchange, topic_name)
- task.start()
- endpoint_address = 'amqp://' + host
- endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange='+exchange+'&amqp-ack-level=broker'+'&persistent=true'
- # amqp broker guarantee ordering
- exact_match = True
- else:
- return SkipTest('Unknown endpoint type: ' + endpoint_type)
-
-
- # create s3 topic
- topic_conf = PSTopicS3(master_zone.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
- topic_arn = topic_conf.set_config()
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn,
- 'Events': []
- }]
-
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # create objects in the bucket (async)
- number_of_objects = 100
- client_threads = []
- start_time = time.time()
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- content = str(os.urandom(1024*1024))
- thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- time_diff = time.time() - start_time
- print('average time for creation + async http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
-
- keys = list(bucket.list())
-
- delay = 40
- print('wait for '+str(delay)+'sec for the messages...')
- time.sleep(delay)
-
- receiver.verify_s3_events(keys, exact_match=exact_match, deletions=False)
-
- # delete objects from the bucket
- client_threads = []
- start_time = time.time()
- for key in bucket.list():
- thr = threading.Thread(target = key.delete, args=())
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- time_diff = time.time() - start_time
- print('average time for deletion + async http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
-
- print('wait for '+str(delay)+'sec for the messages...')
- time.sleep(delay)
-
- receiver.verify_s3_events(keys, exact_match=exact_match, deletions=True)
-
- # cleanup
- s3_notification_conf.del_config()
- topic_conf.del_config()
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
- if endpoint_type == 'http':
- receiver.close()
- else:
- stop_amqp_receiver(receiver, task)
- clean_rabbitmq(proc)
-
-
-def test_ps_s3_persistent_notification_http():
- """ test pushing persistent notification http """
- persistent_notification('http')
-
-
-def test_ps_s3_persistent_notification_amqp():
- """ test pushing persistent notification amqp """
- persistent_notification('amqp')
-
-
-def random_string(length):
- import string
- letters = string.ascii_letters
- return ''.join(random.choice(letters) for i in range(length))
-
-
-def test_ps_s3_persistent_notification_large():
- """ test pushing persistent notification of large notifications """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- topic_name = bucket_name + TOPIC_SUFFIX
-
- receiver = {}
- host = get_ip()
- proc = init_rabbitmq()
- if proc is None:
- return SkipTest('end2end amqp tests require rabbitmq-server installed')
- # start amqp receiver
- exchange = 'ex1'
- task, receiver = create_amqp_receiver_thread(exchange, topic_name)
- task.start()
- endpoint_address = 'amqp://' + host
- opaque_data = random_string(1024*2)
- endpoint_args = 'push-endpoint='+endpoint_address+'&OpaqueData='+opaque_data+'&amqp-exchange='+exchange+'&amqp-ack-level=broker'+'&persistent=true'
- # amqp broker guarantee ordering
- exact_match = True
-
- # create s3 topic
- topic_conf = PSTopicS3(master_zone.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
- topic_arn = topic_conf.set_config()
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn,
- 'Events': []
- }]
-
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # create objects in the bucket (async)
- number_of_objects = 100
- client_threads = []
- start_time = time.time()
- for i in range(number_of_objects):
- key_value = random_string(63)
- key = bucket.new_key(key_value)
- content = str(os.urandom(1024*1024))
- thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- time_diff = time.time() - start_time
- print('average time for creation + async http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
-
- keys = list(bucket.list())
-
- delay = 40
- print('wait for '+str(delay)+'sec for the messages...')
- time.sleep(delay)
-
- receiver.verify_s3_events(keys, exact_match=exact_match, deletions=False)
-
- # delete objects from the bucket
- client_threads = []
- start_time = time.time()
- for key in bucket.list():
- thr = threading.Thread(target = key.delete, args=())
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- time_diff = time.time() - start_time
- print('average time for deletion + async http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
-
- print('wait for '+str(delay)+'sec for the messages...')
- time.sleep(delay)
-
- receiver.verify_s3_events(keys, exact_match=exact_match, deletions=True)
-
- # cleanup
- s3_notification_conf.del_config()
- topic_conf.del_config()
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
- stop_amqp_receiver(receiver, task)
- clean_rabbitmq(proc)
-
-
-def test_ps_s3_persistent_notification_pushback():
- """ test pushing persistent notification pushback """
- return SkipTest("only used in manual testing")
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create random port for the http server
- host = get_ip()
- port = random.randint(10000, 20000)
- # start an http server in a separate thread
- http_server = StreamingHTTPServer(host, port, num_workers=10, delay=0.5)
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- topic_name = bucket_name + TOPIC_SUFFIX
-
- # create s3 topic
- endpoint_address = 'http://'+host+':'+str(port)
- endpoint_args = 'push-endpoint='+endpoint_address+'&persistent=true'
- topic_conf = PSTopicS3(master_zone.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
- topic_arn = topic_conf.set_config()
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn,
- 'Events': []
- }]
-
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # create objects in the bucket (async)
- for j in range(100):
- number_of_objects = randint(500, 1000)
- client_threads = []
- start_time = time.time()
- for i in range(number_of_objects):
- key = bucket.new_key(str(j)+'-'+str(i))
- content = str(os.urandom(1024*1024))
- thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
- time_diff = time.time() - start_time
- print('average time for creation + async http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
-
- keys = list(bucket.list())
-
- delay = 30
- print('wait for '+str(delay)+'sec for the messages...')
- time.sleep(delay)
-
- # delete objects from the bucket
- client_threads = []
- start_time = time.time()
- count = 0
- for key in bucket.list():
- count += 1
- thr = threading.Thread(target = key.delete, args=())
- thr.start()
- client_threads.append(thr)
- if count%100 == 0:
- [thr.join() for thr in client_threads]
- time_diff = time.time() - start_time
- print('average time for deletion + async http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
- client_threads = []
- start_time = time.time()
-
- print('wait for '+str(delay)+'sec for the messages...')
- time.sleep(delay)
-
- # cleanup
- s3_notification_conf.del_config()
- topic_conf.del_config()
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
- time.sleep(delay)
- http_server.close()
-
-
-def test_ps_s3_notification_push_kafka():
- """ test pushing kafka s3 notification on master """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- kafka_proc, zk_proc, kafka_log = init_kafka()
- if kafka_proc is None or zk_proc is None:
- return SkipTest('end2end kafka tests require kafka/zookeeper installed')
-
- master_zone, ps_zone = init_env()
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- # name is constant for manual testing
- topic_name = bucket_name+'_topic'
- # create consumer on the topic
- task, receiver = create_kafka_receiver_thread(topic_name)
- task.start()
-
- # create topic
- topic_conf = PSTopic(ps_zone.conn, topic_name,
- endpoint='kafka://' + kafka_server,
- endpoint_args='kafka-ack-level=broker')
- result, status = topic_conf.set_config()
- assert_equal(status/100, 2)
- parsed_result = json.loads(result)
- topic_arn = parsed_result['arn']
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn,
- 'Events': []
- }]
-
- s3_notification_conf = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # create objects in the bucket (async)
- number_of_objects = 10
- client_threads = []
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- content = str(os.urandom(1024*1024))
- thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
- keys = list(bucket.list())
- receiver.verify_s3_events(keys, exact_match=True)
-
- # delete objects from the bucket
- client_threads = []
- for key in bucket.list():
- thr = threading.Thread(target = key.delete, args=())
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
- receiver.verify_s3_events(keys, exact_match=True, deletions=True)
-
- # cleanup
- s3_notification_conf.del_config()
- topic_conf.del_config()
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
- stop_kafka_receiver(receiver, task)
- clean_kafka(kafka_proc, zk_proc, kafka_log)
-
-
-def test_ps_s3_notification_push_kafka_on_master():
- """ test pushing kafka s3 notification on master """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- kafka_proc, zk_proc, kafka_log = init_kafka()
- if kafka_proc is None or zk_proc is None:
- return SkipTest('end2end kafka tests require kafka/zookeeper installed')
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- # name is constant for manual testing
- topic_name = bucket_name+'_topic'
- # create consumer on the topic
- task, receiver = create_kafka_receiver_thread(topic_name+'_1')
- task.start()
-
- # create s3 topic
- endpoint_address = 'kafka://' + kafka_server
- # without acks from broker
- endpoint_args = 'push-endpoint='+endpoint_address+'&kafka-ack-level=broker'
- topic_conf1 = PSTopicS3(master_zone.conn, topic_name+'_1', zonegroup.name, endpoint_args=endpoint_args)
- topic_arn1 = topic_conf1.set_config()
- endpoint_args = 'push-endpoint='+endpoint_address+'&kafka-ack-level=none'
- topic_conf2 = PSTopicS3(master_zone.conn, topic_name+'_2', zonegroup.name, endpoint_args=endpoint_args)
- topic_arn2 = topic_conf2.set_config()
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name + '_1', 'TopicArn': topic_arn1,
- 'Events': []
- },
- {'Id': notification_name + '_2', 'TopicArn': topic_arn2,
- 'Events': []
- }]
-
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # create objects in the bucket (async)
- number_of_objects = 10
- client_threads = []
- start_time = time.time()
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- content = str(os.urandom(1024*1024))
- thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- time_diff = time.time() - start_time
- print('average time for creation + kafka notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
-
- print('wait for 5sec for the messages...')
- time.sleep(5)
- keys = list(bucket.list())
- receiver.verify_s3_events(keys, exact_match=True)
-
- # delete objects from the bucket
- client_threads = []
- start_time = time.time()
- for key in bucket.list():
- thr = threading.Thread(target = key.delete, args=())
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- time_diff = time.time() - start_time
- print('average time for deletion + kafka notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
-
- print('wait for 5sec for the messages...')
- time.sleep(5)
- receiver.verify_s3_events(keys, exact_match=True, deletions=True)
-
- # cleanup
- s3_notification_conf.del_config()
- topic_conf1.del_config()
- topic_conf2.del_config()
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
- stop_kafka_receiver(receiver, task)
- clean_kafka(kafka_proc, zk_proc, kafka_log)
-
-
-def kafka_security(security_type):
- """ test pushing kafka s3 notification on master """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- master_zone, _ = init_env(require_ps=False)
- if security_type == 'SSL_SASL' and master_zone.secure_conn is None:
- return SkipTest("secure connection is needed to test SASL_SSL security")
- kafka_proc, zk_proc, kafka_log = init_kafka()
- if kafka_proc is None or zk_proc is None:
- return SkipTest('end2end kafka tests require kafka/zookeeper installed')
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- # name is constant for manual testing
- topic_name = bucket_name+'_topic'
- # create consumer on the topic
- task, receiver = create_kafka_receiver_thread(topic_name)
- task.start()
-
- # create s3 topic
- if security_type == 'SSL_SASL':
- endpoint_address = 'kafka://alice:alice-secret@' + kafka_server + ':9094'
- else:
- # ssl only
- endpoint_address = 'kafka://' + kafka_server + ':9093'
-
- KAFKA_DIR = os.environ['KAFKA_DIR']
-
- # without acks from broker, with root CA
- endpoint_args = 'push-endpoint='+endpoint_address+'&kafka-ack-level=none&use-ssl=true&ca-location='+KAFKA_DIR+'rootCA.crt'
-
- if security_type == 'SSL_SASL':
- topic_conf = PSTopicS3(master_zone.secure_conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
- else:
- topic_conf = PSTopicS3(master_zone.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
-
- topic_arn = topic_conf.set_config()
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn,
- 'Events': []
- }]
-
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- s3_notification_conf.set_config()
-
- # create objects in the bucket (async)
- number_of_objects = 10
- client_threads = []
- start_time = time.time()
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- content = str(os.urandom(1024*1024))
- thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- time_diff = time.time() - start_time
- print('average time for creation + kafka notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
-
- try:
- print('wait for 5sec for the messages...')
- time.sleep(5)
- keys = list(bucket.list())
- receiver.verify_s3_events(keys, exact_match=True)
-
- # delete objects from the bucket
- client_threads = []
- start_time = time.time()
- for key in bucket.list():
- thr = threading.Thread(target = key.delete, args=())
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- time_diff = time.time() - start_time
- print('average time for deletion + kafka notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
-
- print('wait for 5sec for the messages...')
- time.sleep(5)
- receiver.verify_s3_events(keys, exact_match=True, deletions=True)
- except Exception as err:
- assert False, str(err)
- finally:
- # cleanup
- s3_notification_conf.del_config()
- topic_conf.del_config()
- # delete the bucket
- for key in bucket.list():
- key.delete()
- master_zone.delete_bucket(bucket_name)
- stop_kafka_receiver(receiver, task)
- clean_kafka(kafka_proc, zk_proc, kafka_log)
-
-
-def test_ps_s3_notification_push_kafka_security_ssl():
- kafka_security('SSL')
-
-def test_ps_s3_notification_push_kafka_security_ssl_sasl():
- kafka_security('SSL_SASL')
-
-
-def test_ps_s3_notification_multi_delete_on_master():
- """ test deletion of multiple keys on master """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- hostname = get_ip()
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create random port for the http server
- host = get_ip()
- port = random.randint(10000, 20000)
- # start an http server in a separate thread
- number_of_objects = 10
- http_server = StreamingHTTPServer(host, port, num_workers=number_of_objects)
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- topic_name = bucket_name + TOPIC_SUFFIX
-
- # create s3 topic
- endpoint_address = 'http://'+host+':'+str(port)
- endpoint_args = 'push-endpoint='+endpoint_address
- topic_conf = PSTopicS3(master_zone.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
- topic_arn = topic_conf.set_config()
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name,
- 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectRemoved:*']
- }]
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # create objects in the bucket
- client_threads = []
- objects_size = {}
- for i in range(number_of_objects):
- content = str(os.urandom(randint(1, 1024)))
- object_size = len(content)
- key = bucket.new_key(str(i))
- objects_size[key.name] = object_size
- thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- keys = list(bucket.list())
-
- start_time = time.time()
- delete_all_objects(master_zone.conn, bucket_name)
- time_diff = time.time() - start_time
- print('average time for deletion + http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
-
- print('wait for 5sec for the messages...')
- time.sleep(5)
-
- # check http receiver
- http_server.verify_s3_events(keys, exact_match=True, deletions=True, expected_sizes=objects_size)
-
- # cleanup
- topic_conf.del_config()
- s3_notification_conf.del_config(notification=notification_name)
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
- http_server.close()
-
-
-def test_ps_s3_notification_push_http_on_master():
- """ test pushing http s3 notification on master """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- hostname = get_ip()
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create random port for the http server
- host = get_ip()
- port = random.randint(10000, 20000)
- # start an http server in a separate thread
- number_of_objects = 10
- http_server = StreamingHTTPServer(host, port, num_workers=number_of_objects)
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- topic_name = bucket_name + TOPIC_SUFFIX
-
- # create s3 topic
- endpoint_address = 'http://'+host+':'+str(port)
- endpoint_args = 'push-endpoint='+endpoint_address
- topic_conf = PSTopicS3(master_zone.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
- topic_arn = topic_conf.set_config()
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name,
- 'TopicArn': topic_arn,
- 'Events': []
- }]
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # create objects in the bucket
- client_threads = []
- objects_size = {}
- start_time = time.time()
- for i in range(number_of_objects):
- content = str(os.urandom(randint(1, 1024)))
- object_size = len(content)
- key = bucket.new_key(str(i))
- objects_size[key.name] = object_size
- thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- time_diff = time.time() - start_time
- print('average time for creation + http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
-
- print('wait for 5sec for the messages...')
- time.sleep(5)
-
- # check http receiver
- keys = list(bucket.list())
- http_server.verify_s3_events(keys, exact_match=True, deletions=False, expected_sizes=objects_size)
-
- # delete objects from the bucket
- client_threads = []
- start_time = time.time()
- for key in bucket.list():
- thr = threading.Thread(target = key.delete, args=())
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- time_diff = time.time() - start_time
- print('average time for deletion + http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
-
- print('wait for 5sec for the messages...')
- time.sleep(5)
-
- # check http receiver
- http_server.verify_s3_events(keys, exact_match=True, deletions=True, expected_sizes=objects_size)
-
- # cleanup
- topic_conf.del_config()
- s3_notification_conf.del_config(notification=notification_name)
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
- http_server.close()
-
-
-def test_ps_s3_opaque_data():
- """ test that opaque id set in topic, is sent in notification """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- hostname = get_ip()
- master_zone, ps_zone = init_env()
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create random port for the http server
- host = get_ip()
- port = random.randint(10000, 20000)
- # start an http server in a separate thread
- number_of_objects = 10
- http_server = StreamingHTTPServer(host, port, num_workers=number_of_objects)
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- topic_name = bucket_name + TOPIC_SUFFIX
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
-
- # create s3 topic
- endpoint_address = 'http://'+host+':'+str(port)
- opaque_data = 'http://1.2.3.4:8888'
- endpoint_args = 'push-endpoint='+endpoint_address+'&OpaqueData='+opaque_data
- topic_conf = PSTopic(ps_zone.conn, topic_name, endpoint=endpoint_address, endpoint_args=endpoint_args)
- result, status = topic_conf.set_config()
- assert_equal(status/100, 2)
- parsed_result = json.loads(result)
- topic_arn = parsed_result['arn']
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name,
- 'TopicArn': topic_arn,
- 'Events': []
- }]
- s3_notification_conf = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # create objects in the bucket
- client_threads = []
- content = 'bar'
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
-
- # check http receiver
- keys = list(bucket.list())
- print('total number of objects: ' + str(len(keys)))
- events = http_server.get_and_reset_events()
- for event in events:
- assert_equal(event['Records'][0]['opaqueData'], opaque_data)
-
- # cleanup
- for key in keys:
- key.delete()
- [thr.join() for thr in client_threads]
- topic_conf.del_config()
- s3_notification_conf.del_config(notification=notification_name)
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
- http_server.close()
-
-
-def test_ps_s3_opaque_data_on_master():
- """ test that opaque id set in topic, is sent in notification on master """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- hostname = get_ip()
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create random port for the http server
- host = get_ip()
- port = random.randint(10000, 20000)
- # start an http server in a separate thread
- number_of_objects = 10
- http_server = StreamingHTTPServer(host, port, num_workers=number_of_objects)
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- topic_name = bucket_name + TOPIC_SUFFIX
-
- # create s3 topic
- endpoint_address = 'http://'+host+':'+str(port)
- endpoint_args = 'push-endpoint='+endpoint_address
- opaque_data = 'http://1.2.3.4:8888'
- topic_conf = PSTopicS3(master_zone.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args, opaque_data=opaque_data)
- topic_arn = topic_conf.set_config()
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name,
- 'TopicArn': topic_arn,
- 'Events': []
- }]
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # create objects in the bucket
- client_threads = []
- start_time = time.time()
- content = 'bar'
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- thr = threading.Thread(target = set_contents_from_string, args=(key, content,))
- thr.start()
- client_threads.append(thr)
- [thr.join() for thr in client_threads]
-
- time_diff = time.time() - start_time
- print('average time for creation + http notification is: ' + str(time_diff*1000/number_of_objects) + ' milliseconds')
-
- print('wait for 5sec for the messages...')
- time.sleep(5)
-
- # check http receiver
- keys = list(bucket.list())
- print('total number of objects: ' + str(len(keys)))
- events = http_server.get_and_reset_events()
- for event in events:
- assert_equal(event['Records'][0]['opaqueData'], opaque_data)
-
- # cleanup
- for key in keys:
- key.delete()
- [thr.join() for thr in client_threads]
- topic_conf.del_config()
- s3_notification_conf.del_config(notification=notification_name)
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
- http_server.close()
-
-def test_ps_topic():
- """ test set/get/delete of topic """
- _, ps_zone = init_env()
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name+TOPIC_SUFFIX
-
- # create topic
- topic_conf = PSTopic(ps_zone.conn, topic_name)
- _, status = topic_conf.set_config()
- assert_equal(status/100, 2)
- # get topic
- result, _ = topic_conf.get_config()
- # verify topic content
- parsed_result = json.loads(result)
- assert_equal(parsed_result['topic']['name'], topic_name)
- assert_equal(len(parsed_result['subs']), 0)
- assert_equal(parsed_result['topic']['arn'],
- 'arn:aws:sns:' + zonegroup.name + ':' + get_tenant() + ':' + topic_name)
- # delete topic
- _, status = topic_conf.del_config()
- assert_equal(status/100, 2)
- # verift topic is deleted
- result, status = topic_conf.get_config()
- assert_equal(status, 404)
- parsed_result = json.loads(result)
- assert_equal(parsed_result['Code'], 'NoSuchKey')
-
-
-def test_ps_topic_with_endpoint():
- """ test set topic with endpoint"""
- _, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name+TOPIC_SUFFIX
-
- # create topic
- dest_endpoint = 'amqp://localhost:7001'
- dest_args = 'amqp-exchange=amqp.direct&amqp-ack-level=none'
- topic_conf = PSTopic(ps_zone.conn, topic_name,
- endpoint=dest_endpoint,
- endpoint_args=dest_args)
- _, status = topic_conf.set_config()
- assert_equal(status/100, 2)
- # get topic
- result, _ = topic_conf.get_config()
- # verify topic content
- parsed_result = json.loads(result)
- assert_equal(parsed_result['topic']['name'], topic_name)
- assert_equal(parsed_result['topic']['dest']['push_endpoint'], dest_endpoint)
- # cleanup
- topic_conf.del_config()
-
-
-def test_ps_notification():
- """ test set/get/delete of notification """
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name+TOPIC_SUFFIX
-
- # create topic
- topic_conf = PSTopic(ps_zone.conn, topic_name)
- topic_conf.set_config()
- # create bucket on the first of the rados zones
- master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- # create notifications
- notification_conf = PSNotification(ps_zone.conn, bucket_name,
- topic_name)
- _, status = notification_conf.set_config()
- assert_equal(status/100, 2)
- # get notification
- result, _ = notification_conf.get_config()
- parsed_result = json.loads(result)
- assert_equal(len(parsed_result['topics']), 1)
- assert_equal(parsed_result['topics'][0]['topic']['name'],
- topic_name)
- # delete notification
- _, status = notification_conf.del_config()
- assert_equal(status/100, 2)
- result, status = notification_conf.get_config()
- parsed_result = json.loads(result)
- assert_equal(len(parsed_result['topics']), 0)
- # TODO should return 404
- # assert_equal(status, 404)
-
- # cleanup
- topic_conf.del_config()
- master_zone.delete_bucket(bucket_name)
-
-
-def test_ps_notification_events():
- """ test set/get/delete of notification on specific events"""
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name+TOPIC_SUFFIX
-
- # create topic
- topic_conf = PSTopic(ps_zone.conn, topic_name)
- topic_conf.set_config()
- # create bucket on the first of the rados zones
- master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- # create notifications
- events = "OBJECT_CREATE,OBJECT_DELETE"
- notification_conf = PSNotification(ps_zone.conn, bucket_name,
- topic_name,
- events)
- _, status = notification_conf.set_config()
- assert_equal(status/100, 2)
- # get notification
- result, _ = notification_conf.get_config()
- parsed_result = json.loads(result)
- assert_equal(len(parsed_result['topics']), 1)
- assert_equal(parsed_result['topics'][0]['topic']['name'],
- topic_name)
- assert_not_equal(len(parsed_result['topics'][0]['events']), 0)
- # TODO add test for invalid event name
-
- # cleanup
- notification_conf.del_config()
- topic_conf.del_config()
- master_zone.delete_bucket(bucket_name)
-
-
-def test_ps_subscription():
- """ test set/get/delete of subscription """
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name+TOPIC_SUFFIX
-
- # create topic
- topic_conf = PSTopic(ps_zone.conn, topic_name)
- topic_conf.set_config()
- # create bucket on the first of the rados zones
- bucket = master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- # create notifications
- notification_conf = PSNotification(ps_zone.conn, bucket_name,
- topic_name)
- _, status = notification_conf.set_config()
- assert_equal(status/100, 2)
- # create subscription
- sub_conf = PSSubscription(ps_zone.conn, bucket_name+SUB_SUFFIX,
- topic_name)
- _, status = sub_conf.set_config()
- assert_equal(status/100, 2)
- # get the subscription
- result, _ = sub_conf.get_config()
- parsed_result = json.loads(result)
- assert_equal(parsed_result['topic'], topic_name)
- # create objects in the bucket
- number_of_objects = 10
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- key.set_contents_from_string('bar')
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
-
- # get the create events from the subscription
- result, _ = sub_conf.get_events()
- events = json.loads(result)
- for event in events['events']:
- log.debug('Event: objname: "' + str(event['info']['key']['name']) + '" type: "' + str(event['event']) + '"')
- keys = list(bucket.list())
- # TODO: use exact match
- verify_events_by_elements(events, keys, exact_match=False)
- # delete objects from the bucket
- for key in bucket.list():
- key.delete()
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
-
- # get the delete events from the subscriptions
- #result, _ = sub_conf.get_events()
- #for event in events['events']:
- # log.debug('Event: objname: "' + str(event['info']['key']['name']) + '" type: "' + str(event['event']) + '"')
- # TODO: check deletions
- # TODO: use exact match
- # verify_events_by_elements(events, keys, exact_match=False, deletions=True)
- # we should see the creations as well as the deletions
- # delete subscription
- _, status = sub_conf.del_config()
- assert_equal(status/100, 2)
- result, status = sub_conf.get_config()
- parsed_result = json.loads(result)
- assert_equal(parsed_result['topic'], '')
- # TODO should return 404
- # assert_equal(status, 404)
-
- # cleanup
- notification_conf.del_config()
- topic_conf.del_config()
- master_zone.delete_bucket(bucket_name)
-
-
-def test_ps_admin():
- """ test radosgw-admin commands """
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name+TOPIC_SUFFIX
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create topic
- topic_conf = PSTopic(ps_zone.conn, topic_name)
- topic_conf.set_config()
- result, status = topic_conf.get_config()
- assert_equal(status, 200)
- parsed_result = json.loads(result)
- assert_equal(parsed_result['topic']['name'], topic_name)
- result, status = ps_zone.zone.cluster.admin(['topic', 'list', '--uid', get_user()] + ps_zone.zone.zone_arg())
- assert_equal(status, 0)
- parsed_result = json.loads(result)
- assert len(parsed_result['topics']) > 0
- result, status = ps_zone.zone.cluster.admin(['topic', 'get', '--uid', get_user(), '--topic', topic_name] + ps_zone.zone.zone_arg())
- assert_equal(status, 0)
- parsed_result = json.loads(result)
- assert_equal(parsed_result['topic']['name'], topic_name)
-
- # create s3 topics
- endpoint_address = 'amqp://127.0.0.1:7001/vhost_1'
- endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=amqp.direct&amqp-ack-level=none'
- topic_conf_s3 = PSTopicS3(master_zone.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
- topic_conf_s3.set_config()
- result, status = topic_conf_s3.get_config()
- assert_equal(status, 200)
- assert_equal(result['GetTopicResponse']['GetTopicResult']['Topic']['Name'], topic_name)
- result, status = master_zone.zone.cluster.admin(['topic', 'list', '--uid', get_user()] + master_zone.zone.zone_arg())
- assert_equal(status, 0)
- parsed_result = json.loads(result)
- assert len(parsed_result['topics']) > 0
- result, status = master_zone.zone.cluster.admin(['topic', 'get', '--uid', get_user(), '--topic', topic_name] + master_zone.zone.zone_arg())
- assert_equal(status, 0)
- parsed_result = json.loads(result)
- assert_equal(parsed_result['topic']['name'], topic_name)
-
- # create bucket on the first of the rados zones
- bucket = master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- # create notifications
- notification_conf = PSNotification(ps_zone.conn, bucket_name,
- topic_name)
- _, status = notification_conf.set_config()
- assert_equal(status/100, 2)
- # create subscription
- sub_conf = PSSubscription(ps_zone.conn, bucket_name+SUB_SUFFIX,
- topic_name)
- _, status = sub_conf.set_config()
- assert_equal(status/100, 2)
- result, status = ps_zone.zone.cluster.admin(['subscription', 'get', '--uid', get_user(), '--subscription', bucket_name+SUB_SUFFIX]
- + ps_zone.zone.zone_arg())
- assert_equal(status, 0)
- parsed_result = json.loads(result)
- assert_equal(parsed_result['name'], bucket_name+SUB_SUFFIX)
- # create objects in the bucket
- number_of_objects = 110
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- key.set_contents_from_string('bar')
- # wait for sync
- # get events from subscription
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
- result, status = ps_zone.zone.cluster.admin(['subscription', 'pull', '--uid', get_user(), '--subscription', bucket_name+SUB_SUFFIX]
- + ps_zone.zone.zone_arg())
- assert_equal(status, 0)
- parsed_result = json.loads(result)
- marker = parsed_result['next_marker']
- events1 = parsed_result['events']
- result, status = ps_zone.zone.cluster.admin(['subscription', 'pull', '--uid', get_user(), '--subscription', bucket_name+SUB_SUFFIX, '--marker', marker]
- + ps_zone.zone.zone_arg())
- assert_equal(status, 0)
- parsed_result = json.loads(result)
- events2 = parsed_result['events']
-
- keys = list(bucket.list())
- verify_events_by_elements({"events": events1+events2}, keys, exact_match=False)
-
- # ack an event in the subscription
- result, status = ps_zone.zone.cluster.admin(['subscription', 'ack', '--uid', get_user(), '--subscription', bucket_name+SUB_SUFFIX, '--event-id', events2[0]['id']]
- + ps_zone.zone.zone_arg())
- assert_equal(status, 0)
-
- # remove the subscription
- result, status = ps_zone.zone.cluster.admin(['subscription', 'rm', '--uid', get_user(), '--subscription', bucket_name+SUB_SUFFIX]
- + ps_zone.zone.zone_arg())
- assert_equal(status, 0)
-
- # remove the topics
- result, status = ps_zone.zone.cluster.admin(['topic', 'rm', '--uid', get_user(), '--topic', topic_name]
- + ps_zone.zone.zone_arg())
- assert_equal(status, 0)
- result, status = master_zone.zone.cluster.admin(['topic', 'rm', '--uid', get_user(), '--topic', topic_name]
- + master_zone.zone.zone_arg())
- assert_equal(status, 0)
-
- # cleanup
- for key in bucket.list():
- key.delete()
- notification_conf.del_config()
- master_zone.delete_bucket(bucket_name)
-
-
-def test_ps_incremental_sync():
- """ test that events are only sent on incremental sync """
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name+TOPIC_SUFFIX
-
- # create topic
- topic_conf = PSTopic(ps_zone.conn, topic_name)
- topic_conf.set_config()
- # create bucket on the first of the rados zones
- bucket = master_zone.create_bucket(bucket_name)
- # create objects in the bucket
- number_of_objects = 10
- for i in range(0, number_of_objects):
- key = bucket.new_key(str(i))
- key.set_contents_from_string('foo')
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
- # create notifications
- notification_conf = PSNotification(ps_zone.conn, bucket_name,
- topic_name)
- _, status = notification_conf.set_config()
- assert_equal(status/100, 2)
- # create subscription
- sub_conf = PSSubscription(ps_zone.conn, bucket_name+SUB_SUFFIX,
- topic_name)
- _, status = sub_conf.set_config()
- assert_equal(status/100, 2)
-
- # create more objects in the bucket
- for i in range(number_of_objects, 2*number_of_objects):
- key = bucket.new_key(str(i))
- key.set_contents_from_string('bar')
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
-
- # get the create events from the subscription
- result, _ = sub_conf.get_events()
- events = json.loads(result)
- count = 0
- for event in events['events']:
- log.debug('Event: objname: "' + str(event['info']['key']['name']) + '" type: "' + str(event['event']) + '"')
- count += 1
-
- # make sure we have 10 and not 20 events
- assert_equal(count, number_of_objects)
-
- # cleanup
- for key in bucket.list():
- key.delete()
- sub_conf.del_config()
- notification_conf.del_config()
- topic_conf.del_config()
- master_zone.delete_bucket(bucket_name)
-
-def test_ps_event_type_subscription():
- """ test subscriptions for different events """
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
-
- # create topic for objects creation
- topic_create_name = bucket_name+TOPIC_SUFFIX+'_create'
- topic_create_conf = PSTopic(ps_zone.conn, topic_create_name)
- topic_create_conf.set_config()
- # create topic for objects deletion
- topic_delete_name = bucket_name+TOPIC_SUFFIX+'_delete'
- topic_delete_conf = PSTopic(ps_zone.conn, topic_delete_name)
- topic_delete_conf.set_config()
- # create topic for all events
- topic_name = bucket_name+TOPIC_SUFFIX+'_all'
- topic_conf = PSTopic(ps_zone.conn, topic_name)
- topic_conf.set_config()
- # create bucket on the first of the rados zones
- bucket = master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
- # create notifications for objects creation
- notification_create_conf = PSNotification(ps_zone.conn, bucket_name,
- topic_create_name, "OBJECT_CREATE")
- _, status = notification_create_conf.set_config()
- assert_equal(status/100, 2)
- # create notifications for objects deletion
- notification_delete_conf = PSNotification(ps_zone.conn, bucket_name,
- topic_delete_name, "OBJECT_DELETE")
- _, status = notification_delete_conf.set_config()
- assert_equal(status/100, 2)
- # create notifications for all events
- notification_conf = PSNotification(ps_zone.conn, bucket_name,
- topic_name, "OBJECT_DELETE,OBJECT_CREATE")
- _, status = notification_conf.set_config()
- assert_equal(status/100, 2)
- # create subscription for objects creation
- sub_create_conf = PSSubscription(ps_zone.conn, bucket_name+SUB_SUFFIX+'_create',
- topic_create_name)
- _, status = sub_create_conf.set_config()
- assert_equal(status/100, 2)
- # create subscription for objects deletion
- sub_delete_conf = PSSubscription(ps_zone.conn, bucket_name+SUB_SUFFIX+'_delete',
- topic_delete_name)
- _, status = sub_delete_conf.set_config()
- assert_equal(status/100, 2)
- # create subscription for all events
- sub_conf = PSSubscription(ps_zone.conn, bucket_name+SUB_SUFFIX+'_all',
- topic_name)
- _, status = sub_conf.set_config()
- assert_equal(status/100, 2)
- # create objects in the bucket
- number_of_objects = 10
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- key.set_contents_from_string('bar')
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
-
- # get the events from the creation subscription
- result, _ = sub_create_conf.get_events()
- events = json.loads(result)
- for event in events['events']:
- log.debug('Event (OBJECT_CREATE): objname: "' + str(event['info']['key']['name']) +
- '" type: "' + str(event['event']) + '"')
- keys = list(bucket.list())
- # TODO: use exact match
- verify_events_by_elements(events, keys, exact_match=False)
- # get the events from the deletions subscription
- result, _ = sub_delete_conf.get_events()
- events = json.loads(result)
- for event in events['events']:
- log.debug('Event (OBJECT_DELETE): objname: "' + str(event['info']['key']['name']) +
- '" type: "' + str(event['event']) + '"')
- assert_equal(len(events['events']), 0)
- # get the events from the all events subscription
- result, _ = sub_conf.get_events()
- events = json.loads(result)
- for event in events['events']:
- log.debug('Event (OBJECT_CREATE,OBJECT_DELETE): objname: "' +
- str(event['info']['key']['name']) + '" type: "' + str(event['event']) + '"')
- # TODO: use exact match
- verify_events_by_elements(events, keys, exact_match=False)
- # delete objects from the bucket
- for key in bucket.list():
- key.delete()
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
- log.debug("Event (OBJECT_DELETE) synced")
-
- # get the events from the creations subscription
- result, _ = sub_create_conf.get_events()
- events = json.loads(result)
- for event in events['events']:
- log.debug('Event (OBJECT_CREATE): objname: "' + str(event['info']['key']['name']) +
- '" type: "' + str(event['event']) + '"')
- # deletions should not change the creation events
- # TODO: use exact match
- verify_events_by_elements(events, keys, exact_match=False)
- # get the events from the deletions subscription
- result, _ = sub_delete_conf.get_events()
- events = json.loads(result)
- for event in events['events']:
- log.debug('Event (OBJECT_DELETE): objname: "' + str(event['info']['key']['name']) +
- '" type: "' + str(event['event']) + '"')
- # only deletions should be listed here
- # TODO: use exact match
- verify_events_by_elements(events, keys, exact_match=False, deletions=True)
- # get the events from the all events subscription
- result, _ = sub_create_conf.get_events()
- events = json.loads(result)
- for event in events['events']:
- log.debug('Event (OBJECT_CREATE,OBJECT_DELETE): objname: "' + str(event['info']['key']['name']) +
- '" type: "' + str(event['event']) + '"')
- # both deletions and creations should be here
- # TODO: use exact match
- verify_events_by_elements(events, keys, exact_match=False, deletions=False)
- # verify_events_by_elements(events, keys, exact_match=False, deletions=True)
- # TODO: (1) test deletions (2) test overall number of events
-
- # test subscription deletion when topic is specified
- _, status = sub_create_conf.del_config(topic=True)
- assert_equal(status/100, 2)
- _, status = sub_delete_conf.del_config(topic=True)
- assert_equal(status/100, 2)
- _, status = sub_conf.del_config(topic=True)
- assert_equal(status/100, 2)
-
- # cleanup
- notification_create_conf.del_config()
- notification_delete_conf.del_config()
- notification_conf.del_config()
- topic_create_conf.del_config()
- topic_delete_conf.del_config()
- topic_conf.del_config()
- master_zone.delete_bucket(bucket_name)
-
-
-def test_ps_event_fetching():
- """ test incremental fetching of events from a subscription """
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name+TOPIC_SUFFIX
-
- # create topic
- topic_conf = PSTopic(ps_zone.conn, topic_name)
- topic_conf.set_config()
- # create bucket on the first of the rados zones
- bucket = master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- # create notifications
- notification_conf = PSNotification(ps_zone.conn, bucket_name,
- topic_name)
- _, status = notification_conf.set_config()
- assert_equal(status/100, 2)
- # create subscription
- sub_conf = PSSubscription(ps_zone.conn, bucket_name+SUB_SUFFIX,
- topic_name)
- _, status = sub_conf.set_config()
- assert_equal(status/100, 2)
- # create objects in the bucket
- number_of_objects = 100
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- key.set_contents_from_string('bar')
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
- max_events = 15
- total_events_count = 0
- next_marker = None
- all_events = []
- while True:
- # get the events from the subscription
- result, _ = sub_conf.get_events(max_events, next_marker)
- events = json.loads(result)
- total_events_count += len(events['events'])
- all_events.extend(events['events'])
- next_marker = events['next_marker']
- for event in events['events']:
- log.debug('Event: objname: "' + str(event['info']['key']['name']) + '" type: "' + str(event['event']) + '"')
- if next_marker == '':
- break
- keys = list(bucket.list())
- # TODO: use exact match
- verify_events_by_elements({'events': all_events}, keys, exact_match=False)
-
- # cleanup
- sub_conf.del_config()
- notification_conf.del_config()
- topic_conf.del_config()
- for key in bucket.list():
- key.delete()
- master_zone.delete_bucket(bucket_name)
-
-
-def test_ps_event_acking():
- """ test acking of some events in a subscription """
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name+TOPIC_SUFFIX
-
- # create topic
- topic_conf = PSTopic(ps_zone.conn, topic_name)
- topic_conf.set_config()
- # create bucket on the first of the rados zones
- bucket = master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- # create notifications
- notification_conf = PSNotification(ps_zone.conn, bucket_name,
- topic_name)
- _, status = notification_conf.set_config()
- assert_equal(status/100, 2)
- # create subscription
- sub_conf = PSSubscription(ps_zone.conn, bucket_name+SUB_SUFFIX,
- topic_name)
- _, status = sub_conf.set_config()
- assert_equal(status/100, 2)
- # create objects in the bucket
- number_of_objects = 10
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- key.set_contents_from_string('bar')
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
-
- # get the create events from the subscription
- result, _ = sub_conf.get_events()
- events = json.loads(result)
- original_number_of_events = len(events)
- for event in events['events']:
- log.debug('Event (before ack) id: "' + str(event['id']) + '"')
- keys = list(bucket.list())
- # TODO: use exact match
- verify_events_by_elements(events, keys, exact_match=False)
- # ack half of the events
- events_to_ack = number_of_objects/2
- for event in events['events']:
- if events_to_ack == 0:
- break
- _, status = sub_conf.ack_events(event['id'])
- assert_equal(status/100, 2)
- events_to_ack -= 1
-
- # verify that acked events are gone
- result, _ = sub_conf.get_events()
- events = json.loads(result)
- for event in events['events']:
- log.debug('Event (after ack) id: "' + str(event['id']) + '"')
- assert len(events) >= (original_number_of_events - number_of_objects/2)
-
- # cleanup
- sub_conf.del_config()
- notification_conf.del_config()
- topic_conf.del_config()
- for key in bucket.list():
- key.delete()
- master_zone.delete_bucket(bucket_name)
-
-
-def test_ps_creation_triggers():
- """ test object creation notifications in using put/copy/post """
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name+TOPIC_SUFFIX
-
- # create topic
- topic_conf = PSTopic(ps_zone.conn, topic_name)
- topic_conf.set_config()
- # create bucket on the first of the rados zones
- bucket = master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- # create notifications
- notification_conf = PSNotification(ps_zone.conn, bucket_name,
- topic_name)
- _, status = notification_conf.set_config()
- assert_equal(status/100, 2)
- # create subscription
- sub_conf = PSSubscription(ps_zone.conn, bucket_name+SUB_SUFFIX,
- topic_name)
- _, status = sub_conf.set_config()
- assert_equal(status/100, 2)
- # create objects in the bucket using PUT
- key = bucket.new_key('put')
- key.set_contents_from_string('bar')
- # create objects in the bucket using COPY
- bucket.copy_key('copy', bucket.name, key.name)
-
- # create objects in the bucket using multi-part upload
- fp = tempfile.NamedTemporaryFile(mode='w+b')
- object_size = 1024
- content = bytearray(os.urandom(object_size))
- fp.write(content)
- fp.flush()
- fp.seek(0)
- uploader = bucket.initiate_multipart_upload('multipart')
- uploader.upload_part_from_file(fp, 1)
- uploader.complete_upload()
- fp.close()
-
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
-
- # get the create events from the subscription
- result, _ = sub_conf.get_events()
- events = json.loads(result)
- for event in events['events']:
- log.debug('Event key: "' + str(event['info']['key']['name']) + '" type: "' + str(event['event']) + '"')
-
- # TODO: verify the specific 3 keys: 'put', 'copy' and 'multipart'
- assert len(events['events']) >= 3
- # cleanup
- sub_conf.del_config()
- notification_conf.del_config()
- topic_conf.del_config()
- for key in bucket.list():
- key.delete()
- master_zone.delete_bucket(bucket_name)
-
-def ps_s3_creation_triggers_on_master(external_endpoint_address=None, ca_location=None, verify_ssl='true'):
- """ test object creation s3 notifications in using put/copy/post on master"""
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- if not external_endpoint_address:
- hostname = get_ip()
- proc = init_rabbitmq()
- if proc is None:
- return SkipTest('end2end amqp tests require rabbitmq-server installed')
- else:
- proc = None
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- topic_name = bucket_name + TOPIC_SUFFIX
-
- # start amqp receiver
- exchange = 'ex1'
- task, receiver = create_amqp_receiver_thread(exchange, topic_name, external_endpoint_address, ca_location)
- task.start()
-
- # create s3 topic
- if external_endpoint_address:
- endpoint_address = external_endpoint_address
- elif ca_location:
- endpoint_address = 'amqps://' + hostname
- else:
- endpoint_address = 'amqp://' + hostname
- endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=' + exchange +'&amqp-ack-level=broker&verify-ssl='+verify_ssl
- if ca_location:
- endpoint_args += '&ca-location={}'.format(ca_location)
- if external_endpoint_address:
- topic_conf = PSTopicS3(master_zone.secure_conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
- else:
- topic_conf = PSTopicS3(master_zone.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
- topic_arn = topic_conf.set_config()
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name,'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:Put', 's3:ObjectCreated:Copy']
- }]
-
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- objects_size = {}
- # create objects in the bucket using PUT
- content = str(os.urandom(randint(1, 1024)))
- key_name = 'put'
- key = bucket.new_key(key_name)
- objects_size[key_name] = len(content)
- key.set_contents_from_string(content)
- # create objects in the bucket using COPY
- key_name = 'copy'
- bucket.copy_key(key_name, bucket.name, key.name)
- objects_size[key_name] = len(content)
-
- # create objects in the bucket using multi-part upload
- fp = tempfile.NamedTemporaryFile(mode='w+b')
- content = bytearray(os.urandom(10*1024*1024))
- key_name = 'multipart'
- objects_size[key_name] = len(content)
- fp.write(content)
- fp.flush()
- fp.seek(0)
- uploader = bucket.initiate_multipart_upload(key_name)
- uploader.upload_part_from_file(fp, 1)
- uploader.complete_upload()
- fp.close()
-
- print('wait for 5sec for the messages...')
- time.sleep(5)
-
- # check amqp receiver
- keys = list(bucket.list())
- receiver.verify_s3_events(keys, exact_match=True, expected_sizes=objects_size)
-
- # cleanup
- stop_amqp_receiver(receiver, task)
- s3_notification_conf.del_config()
- topic_conf.del_config()
- for key in bucket.list():
- key.delete()
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
- if proc:
- clean_rabbitmq(proc)
-
-
-def test_ps_s3_creation_triggers_on_master():
- ps_s3_creation_triggers_on_master()
-
-
-def test_ps_s3_creation_triggers_on_master_external():
- from distutils.util import strtobool
-
- if 'AMQP_EXTERNAL_ENDPOINT' in os.environ:
- try:
- if strtobool(os.environ['AMQP_VERIFY_SSL']):
- verify_ssl = 'true'
- else:
- verify_ssl = 'false'
- except Exception as e:
- verify_ssl = 'true'
-
- ps_s3_creation_triggers_on_master(
- external_endpoint_address=os.environ['AMQP_EXTERNAL_ENDPOINT'],
- verify_ssl=verify_ssl)
- else:
- return SkipTest("Set AMQP_EXTERNAL_ENDPOINT to a valid external AMQP endpoint url for this test to run")
-
-def test_ps_s3_creation_triggers_on_master_ssl():
- import datetime
- import textwrap
- import stat
- from cryptography import x509
- from cryptography.x509.oid import NameOID
- from cryptography.hazmat.primitives import hashes
- from cryptography.hazmat.backends import default_backend
- from cryptography.hazmat.primitives import serialization
- from cryptography.hazmat.primitives.asymmetric import rsa
- from tempfile import TemporaryDirectory
-
- with TemporaryDirectory() as tempdir:
- # modify permissions to ensure that the rabbitmq user can access them
- os.chmod(tempdir, mode=stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
- CACERTFILE = os.path.join(tempdir, 'ca_certificate.pem')
- CERTFILE = os.path.join(tempdir, 'server_certificate.pem')
- KEYFILE = os.path.join(tempdir, 'server_key.pem')
- RABBITMQ_CONF_FILE = os.path.join(tempdir, 'rabbitmq.config')
-
- root_key = rsa.generate_private_key(
- public_exponent=65537,
- key_size=2048,
- backend=default_backend()
- )
- subject = issuer = x509.Name([
- x509.NameAttribute(NameOID.COUNTRY_NAME, u"UK"),
- x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, u"Oxfordshire"),
- x509.NameAttribute(NameOID.LOCALITY_NAME, u"Harwell"),
- x509.NameAttribute(NameOID.ORGANIZATION_NAME, u"Rosalind Franklin Institute"),
- x509.NameAttribute(NameOID.COMMON_NAME, u"RFI CA"),
- ])
- root_cert = x509.CertificateBuilder().subject_name(
- subject
- ).issuer_name(
- issuer
- ).public_key(
- root_key.public_key()
- ).serial_number(
- x509.random_serial_number()
- ).not_valid_before(
- datetime.datetime.utcnow()
- ).not_valid_after(
- datetime.datetime.utcnow() + datetime.timedelta(days=3650)
- ).add_extension(
- x509.BasicConstraints(ca=True, path_length=None), critical=True
- ).sign(root_key, hashes.SHA256(), default_backend())
- with open(CACERTFILE, "wb") as f:
- f.write(root_cert.public_bytes(serialization.Encoding.PEM))
-
- # Now we want to generate a cert from that root
- cert_key = rsa.generate_private_key(
- public_exponent=65537,
- key_size=2048,
- backend=default_backend()
- )
- with open(KEYFILE, "wb") as f:
- f.write(cert_key.private_bytes(
- encoding=serialization.Encoding.PEM,
- format=serialization.PrivateFormat.TraditionalOpenSSL,
- encryption_algorithm=serialization.NoEncryption(),
- ))
- new_subject = x509.Name([
- x509.NameAttribute(NameOID.COUNTRY_NAME, u"UK"),
- x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, u"Oxfordshire"),
- x509.NameAttribute(NameOID.LOCALITY_NAME, u"Harwell"),
- x509.NameAttribute(NameOID.ORGANIZATION_NAME, u"Rosalind Franklin Institute"),
- ])
- cert = x509.CertificateBuilder().subject_name(
- new_subject
- ).issuer_name(
- root_cert.issuer
- ).public_key(
- cert_key.public_key()
- ).serial_number(
- x509.random_serial_number()
- ).not_valid_before(
- datetime.datetime.utcnow()
- ).not_valid_after(
- datetime.datetime.utcnow() + datetime.timedelta(days=30)
- ).add_extension(
- x509.SubjectAlternativeName([x509.DNSName(u"localhost")]),
- critical=False,
- ).sign(root_key, hashes.SHA256(), default_backend())
- # Write our certificate out to disk.
- with open(CERTFILE, "wb") as f:
- f.write(cert.public_bytes(serialization.Encoding.PEM))
-
- with open(RABBITMQ_CONF_FILE, "w") as f:
- # use the old style config format to ensure it also runs on older RabbitMQ versions.
- f.write(textwrap.dedent(f'''
- [
- {{rabbit, [
- {{ssl_listeners, [5671]}},
- {{ssl_options, [{{cacertfile, "{CACERTFILE}"}},
- {{certfile, "{CERTFILE}"}},
- {{keyfile, "{KEYFILE}"}},
- {{verify, verify_peer}},
- {{fail_if_no_peer_cert, false}}]}}]}}
- ].
- '''))
- os.environ['RABBITMQ_CONFIG_FILE'] = os.path.splitext(RABBITMQ_CONF_FILE)[0]
-
- ps_s3_creation_triggers_on_master(ca_location=CACERTFILE)
-
- del os.environ['RABBITMQ_CONFIG_FILE']
-
-
-def test_ps_s3_multipart_on_master():
- """ test multipart object upload on master"""
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- hostname = get_ip()
- proc = init_rabbitmq()
- if proc is None:
- return SkipTest('end2end amqp tests require rabbitmq-server installed')
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- topic_name = bucket_name + TOPIC_SUFFIX
-
- # start amqp receivers
- exchange = 'ex1'
- task1, receiver1 = create_amqp_receiver_thread(exchange, topic_name+'_1')
- task1.start()
- task2, receiver2 = create_amqp_receiver_thread(exchange, topic_name+'_2')
- task2.start()
- task3, receiver3 = create_amqp_receiver_thread(exchange, topic_name+'_3')
- task3.start()
-
- # create s3 topics
- endpoint_address = 'amqp://' + hostname
- endpoint_args = 'push-endpoint=' + endpoint_address + '&amqp-exchange=' + exchange + '&amqp-ack-level=broker'
- topic_conf1 = PSTopicS3(master_zone.conn, topic_name+'_1', zonegroup.name, endpoint_args=endpoint_args)
- topic_arn1 = topic_conf1.set_config()
- topic_conf2 = PSTopicS3(master_zone.conn, topic_name+'_2', zonegroup.name, endpoint_args=endpoint_args)
- topic_arn2 = topic_conf2.set_config()
- topic_conf3 = PSTopicS3(master_zone.conn, topic_name+'_3', zonegroup.name, endpoint_args=endpoint_args)
- topic_arn3 = topic_conf3.set_config()
-
- # create s3 notifications
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name+'_1', 'TopicArn': topic_arn1,
- 'Events': ['s3:ObjectCreated:*']
- },
- {'Id': notification_name+'_2', 'TopicArn': topic_arn2,
- 'Events': ['s3:ObjectCreated:Post']
- },
- {'Id': notification_name+'_3', 'TopicArn': topic_arn3,
- 'Events': ['s3:ObjectCreated:CompleteMultipartUpload']
- }]
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # create objects in the bucket using multi-part upload
- fp = tempfile.NamedTemporaryFile(mode='w+b')
- object_size = 1024
- content = bytearray(os.urandom(object_size))
- fp.write(content)
- fp.flush()
- fp.seek(0)
- uploader = bucket.initiate_multipart_upload('multipart')
- uploader.upload_part_from_file(fp, 1)
- uploader.complete_upload()
- fp.close()
-
- print('wait for 5sec for the messages...')
- time.sleep(5)
-
- # check amqp receiver
- events = receiver1.get_and_reset_events()
- assert_equal(len(events), 3)
-
- events = receiver2.get_and_reset_events()
- assert_equal(len(events), 1)
- assert_equal(events[0]['Records'][0]['eventName'], 's3:ObjectCreated:Post')
- assert_equal(events[0]['Records'][0]['s3']['configurationId'], notification_name+'_2')
-
- events = receiver3.get_and_reset_events()
- assert_equal(len(events), 1)
- assert_equal(events[0]['Records'][0]['eventName'], 's3:ObjectCreated:CompleteMultipartUpload')
- assert_equal(events[0]['Records'][0]['s3']['configurationId'], notification_name+'_3')
- print(events[0]['Records'][0]['s3']['object']['size'])
-
- # cleanup
- stop_amqp_receiver(receiver1, task1)
- stop_amqp_receiver(receiver2, task2)
- stop_amqp_receiver(receiver3, task3)
- s3_notification_conf.del_config()
- topic_conf1.del_config()
- topic_conf2.del_config()
- topic_conf3.del_config()
- for key in bucket.list():
- key.delete()
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
- clean_rabbitmq(proc)
-
-
-def test_ps_versioned_deletion():
- """ test notification of deletion markers """
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name+TOPIC_SUFFIX
-
- # create topics
- topic_conf1 = PSTopic(ps_zone.conn, topic_name+'_1')
- _, status = topic_conf1.set_config()
- assert_equal(status/100, 2)
- topic_conf2 = PSTopic(ps_zone.conn, topic_name+'_2')
- _, status = topic_conf2.set_config()
- assert_equal(status/100, 2)
-
- # create bucket on the first of the rados zones
- bucket = master_zone.create_bucket(bucket_name)
- bucket.configure_versioning(True)
-
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
-
- # create notifications
- event_type1 = 'OBJECT_DELETE'
- notification_conf1 = PSNotification(ps_zone.conn, bucket_name,
- topic_name+'_1',
- event_type1)
- _, status = notification_conf1.set_config()
- assert_equal(status/100, 2)
- event_type2 = 'DELETE_MARKER_CREATE'
- notification_conf2 = PSNotification(ps_zone.conn, bucket_name,
- topic_name+'_2',
- event_type2)
- _, status = notification_conf2.set_config()
- assert_equal(status/100, 2)
-
- # create subscriptions
- sub_conf1 = PSSubscription(ps_zone.conn, bucket_name+SUB_SUFFIX+'_1',
- topic_name+'_1')
- _, status = sub_conf1.set_config()
- assert_equal(status/100, 2)
- sub_conf2 = PSSubscription(ps_zone.conn, bucket_name+SUB_SUFFIX+'_2',
- topic_name+'_2')
- _, status = sub_conf2.set_config()
- assert_equal(status/100, 2)
-
- # create objects in the bucket
- key = bucket.new_key('foo')
- key.set_contents_from_string('bar')
- v1 = key.version_id
- key.set_contents_from_string('kaboom')
- v2 = key.version_id
- # create deletion marker
- delete_marker_key = bucket.delete_key(key.name)
-
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
-
- # delete the deletion marker
- delete_marker_key.delete()
- # delete versions
- bucket.delete_key(key.name, version_id=v2)
- bucket.delete_key(key.name, version_id=v1)
-
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
-
- # get the delete events from the subscription
- result, _ = sub_conf1.get_events()
- events = json.loads(result)
- for event in events['events']:
- log.debug('Event key: "' + str(event['info']['key']['name']) + '" type: "' + str(event['event']) + '"')
- assert_equal(str(event['event']), event_type1)
-
- result, _ = sub_conf2.get_events()
- events = json.loads(result)
- for event in events['events']:
- log.debug('Event key: "' + str(event['info']['key']['name']) + '" type: "' + str(event['event']) + '"')
- assert_equal(str(event['event']), event_type2)
-
- # cleanup
- # follwing is needed for the cleanup in the case of 3-zones
- # see: http://tracker.ceph.com/issues/39142
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
- zonegroup_conns = ZonegroupConns(zonegroup)
- try:
- zonegroup_bucket_checkpoint(zonegroup_conns, bucket_name)
- master_zone.delete_bucket(bucket_name)
- except:
- log.debug('zonegroup_bucket_checkpoint failed, cannot delete bucket')
- sub_conf1.del_config()
- sub_conf2.del_config()
- notification_conf1.del_config()
- notification_conf2.del_config()
- topic_conf1.del_config()
- topic_conf2.del_config()
-
-
-def test_ps_s3_metadata_on_master():
- """ test s3 notification of metadata on master """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- hostname = get_ip()
- proc = init_rabbitmq()
- if proc is None:
- return SkipTest('end2end amqp tests require rabbitmq-server installed')
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- topic_name = bucket_name + TOPIC_SUFFIX
-
- # start amqp receiver
- exchange = 'ex1'
- task, receiver = create_amqp_receiver_thread(exchange, topic_name)
- task.start()
-
- # create s3 topic
- endpoint_address = 'amqp://' + hostname
- endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=' + exchange +'&amqp-ack-level=routable'
- topic_conf = PSTopicS3(master_zone.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
- topic_arn = topic_conf.set_config()
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- meta_key = 'meta1'
- meta_value = 'This is my metadata value'
- meta_prefix = 'x-amz-meta-'
- topic_conf_list = [{'Id': notification_name,'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:*', 's3:ObjectRemoved:*'],
- 'Filter': {
- 'Metadata': {
- 'FilterRules': [{'Name': meta_prefix+meta_key, 'Value': meta_value}]
- }
- }
- }]
-
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- expected_keys = []
- # create objects in the bucket
- key_name = 'foo'
- key = bucket.new_key(key_name)
- key.set_metadata(meta_key, meta_value)
- key.set_contents_from_string('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa')
- expected_keys.append(key_name)
-
- # create objects in the bucket using COPY
- key_name = 'copy_of_foo'
- bucket.copy_key(key_name, bucket.name, key.name)
- expected_keys.append(key_name)
-
- # create another objects in the bucket using COPY
- # but override the metadata value
- key_name = 'another_copy_of_foo'
- bucket.copy_key(key_name, bucket.name, key.name, metadata={meta_key: 'kaboom'})
-
- # create objects in the bucket using multi-part upload
- fp = tempfile.NamedTemporaryFile(mode='w+b')
- chunk_size = 1024*1024*5 # 5MB
- object_size = 10*chunk_size
- content = bytearray(os.urandom(object_size))
- fp.write(content)
- fp.flush()
- fp.seek(0)
- key_name = 'multipart_foo'
- uploader = bucket.initiate_multipart_upload(key_name,
- metadata={meta_key: meta_value})
- for i in range(1,5):
- uploader.upload_part_from_file(fp, i, size=chunk_size)
- fp.seek(i*chunk_size)
- uploader.complete_upload()
- fp.close()
- expected_keys.append(key_name)
-
- print('wait for 5sec for the messages...')
- time.sleep(5)
- # check amqp receiver
- events = receiver.get_and_reset_events()
- assert_equal(len(events), 4) # PUT, COPY, Multipart start, Multipart End
- for event in events:
- assert(event['Records'][0]['s3']['object']['key'] in expected_keys)
-
- # delete objects
- for key in bucket.list():
- key.delete()
- print('wait for 5sec for the messages...')
- time.sleep(5)
- # check amqp receiver
- #assert_equal(len(receiver.get_and_reset_events()), len(expected_keys))
-
- # all 3 object has metadata when deleted
- assert_equal(event_count, 3)
-
- # cleanup
- stop_amqp_receiver(receiver, task)
- s3_notification_conf.del_config()
- topic_conf.del_config()
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
- clean_rabbitmq(proc)
-
-
-def test_ps_s3_tags_on_master():
- """ test s3 notification of tags on master """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- hostname = get_ip()
- proc = init_rabbitmq()
- if proc is None:
- return SkipTest('end2end amqp tests require rabbitmq-server installed')
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- topic_name = bucket_name + TOPIC_SUFFIX
-
- # start amqp receiver
- exchange = 'ex1'
- task, receiver = create_amqp_receiver_thread(exchange, topic_name)
- task.start()
-
- # create s3 topic
- endpoint_address = 'amqp://' + hostname
- endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=' + exchange +'&amqp-ack-level=routable'
- topic_conf = PSTopicS3(master_zone.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
- topic_arn = topic_conf.set_config()
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name,'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:*', 's3:ObjectRemoved:*'],
- 'Filter': {
- 'Tags': {
- 'FilterRules': [{'Name': 'hello', 'Value': 'world'}]
- }
- }
- }]
-
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # create objects in the bucket with tags
- tags = 'hello=world&ka=boom'
- key_name1 = 'key1'
- put_object_tagging(master_zone.conn, bucket_name, key_name1, tags)
- tags = 'foo=bar&ka=boom'
- key_name2 = 'key2'
- put_object_tagging(master_zone.conn, bucket_name, key_name2, tags)
- key_name3 = 'key3'
- key = bucket.new_key(key_name3)
- key.set_contents_from_string('bar')
- # create objects in the bucket using COPY
- bucket.copy_key('copy_of_'+key_name1, bucket.name, key_name1)
- print('wait for 5sec for the messages...')
- time.sleep(5)
- expected_tags = [{'val': 'world', 'key': 'hello'}, {'val': 'boom', 'key': 'ka'}]
- # check amqp receiver
- filtered_count = 0
- for event in receiver.get_and_reset_events():
- obj_tags = event['Records'][0]['s3']['object']['tags']
- assert_equal(obj_tags[0], expected_tags[0])
- filtered_count += 1
- assert_equal(filtered_count, 2)
-
- # delete the objects
- for key in bucket.list():
- key.delete()
- print('wait for 5sec for the messages...')
- time.sleep(5)
- # check amqp receiver
- filtered_count = 0
- for event in receiver.get_and_reset_events():
- obj_tags = event['Records'][0]['s3']['object']['tags']
- assert_equal(obj_tags[0], expected_tags[0])
- filtered_count += 1
- assert_equal(filtered_count, 2)
-
- # cleanup
- stop_amqp_receiver(receiver, task)
- s3_notification_conf.del_config()
- topic_conf.del_config()
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
- clean_rabbitmq(proc)
-
-
-def test_ps_s3_versioning_on_master():
- """ test s3 notification of object versions """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- hostname = get_ip()
- proc = init_rabbitmq()
- if proc is None:
- return SkipTest('end2end amqp tests require rabbitmq-server installed')
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- bucket.configure_versioning(True)
- topic_name = bucket_name + TOPIC_SUFFIX
-
- # start amqp receiver
- exchange = 'ex1'
- task, receiver = create_amqp_receiver_thread(exchange, topic_name)
- task.start()
-
- # create s3 topic
- endpoint_address = 'amqp://' + hostname
- endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=' + exchange +'&amqp-ack-level=broker'
- topic_conf = PSTopicS3(master_zone.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
- topic_arn = topic_conf.set_config()
- # create notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name, 'TopicArn': topic_arn,
- 'Events': []
- }]
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- _, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # create objects in the bucket
- key_value = 'foo'
- key = bucket.new_key(key_value)
- key.set_contents_from_string('hello')
- ver1 = key.version_id
- key.set_contents_from_string('world')
- ver2 = key.version_id
-
- print('wait for 5sec for the messages...')
- time.sleep(5)
-
- # check amqp receiver
- events = receiver.get_and_reset_events()
- num_of_versions = 0
- for event_list in events:
- for event in event_list['Records']:
- assert_equal(event['s3']['object']['key'], key_value)
- version = event['s3']['object']['versionId']
- num_of_versions += 1
- if version not in (ver1, ver2):
- print('version mismatch: '+version+' not in: ('+ver1+', '+ver2+')')
- assert_equal(1, 0)
- else:
- print('version ok: '+version+' in: ('+ver1+', '+ver2+')')
-
- assert_equal(num_of_versions, 2)
-
- # cleanup
- stop_amqp_receiver(receiver, task)
- s3_notification_conf.del_config()
- topic_conf.del_config()
- # delete the bucket
- bucket.delete_key(key.name, version_id=ver2)
- bucket.delete_key(key.name, version_id=ver1)
- master_zone.delete_bucket(bucket_name)
- clean_rabbitmq(proc)
-
-
-def test_ps_s3_versioned_deletion_on_master():
- """ test s3 notification of deletion markers on master """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- hostname = get_ip()
- proc = init_rabbitmq()
- if proc is None:
- return SkipTest('end2end amqp tests require rabbitmq-server installed')
- master_zone, _ = init_env(require_ps=False)
- realm = get_realm()
- zonegroup = realm.master_zonegroup()
-
- # create bucket
- bucket_name = gen_bucket_name()
- bucket = master_zone.create_bucket(bucket_name)
- bucket.configure_versioning(True)
- topic_name = bucket_name + TOPIC_SUFFIX
-
- # start amqp receiver
- exchange = 'ex1'
- task, receiver = create_amqp_receiver_thread(exchange, topic_name)
- task.start()
-
- # create s3 topic
- endpoint_address = 'amqp://' + hostname
- endpoint_args = 'push-endpoint='+endpoint_address+'&amqp-exchange=' + exchange +'&amqp-ack-level=broker'
- topic_conf = PSTopicS3(master_zone.conn, topic_name, zonegroup.name, endpoint_args=endpoint_args)
- topic_arn = topic_conf.set_config()
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name+'_1', 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectRemoved:*']
- },
- {'Id': notification_name+'_2', 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectRemoved:DeleteMarkerCreated']
- },
- {'Id': notification_name+'_3', 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectRemoved:Delete']
- }]
- s3_notification_conf = PSNotificationS3(master_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # create objects in the bucket
- key = bucket.new_key('foo')
- content = str(os.urandom(512))
- size1 = len(content)
- key.set_contents_from_string(content)
- ver1 = key.version_id
- content = str(os.urandom(511))
- size2 = len(content)
- key.set_contents_from_string(content)
- ver2 = key.version_id
- # create delete marker (non versioned deletion)
- delete_marker_key = bucket.delete_key(key.name)
-
- time.sleep(1)
-
- # versioned deletion
- bucket.delete_key(key.name, version_id=ver2)
- bucket.delete_key(key.name, version_id=ver1)
-
- print('wait for 5sec for the messages...')
- time.sleep(5)
-
- # check amqp receiver
- events = receiver.get_and_reset_events()
- delete_events = 0
- delete_marker_create_events = 0
- for event_list in events:
- for event in event_list['Records']:
- size = event['s3']['object']['size']
- if event['eventName'] == 's3:ObjectRemoved:Delete':
- delete_events += 1
- assert size in [size1, size2]
- assert event['s3']['configurationId'] in [notification_name+'_1', notification_name+'_3']
- if event['eventName'] == 's3:ObjectRemoved:DeleteMarkerCreated':
- delete_marker_create_events += 1
- assert size == size2
- assert event['s3']['configurationId'] in [notification_name+'_1', notification_name+'_2']
-
- # 2 key versions were deleted
- # notified over the same topic via 2 notifications (1,3)
- assert_equal(delete_events, 2*2)
- # 1 deletion marker was created
- # notified over the same topic over 2 notifications (1,2)
- assert_equal(delete_marker_create_events, 1*2)
-
- # cleanup
- delete_marker_key.delete()
- stop_amqp_receiver(receiver, task)
- s3_notification_conf.del_config()
- topic_conf.del_config()
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
- clean_rabbitmq(proc)
-
-
-def test_ps_push_http():
- """ test pushing to http endpoint """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name+TOPIC_SUFFIX
-
- # create random port for the http server
- host = get_ip()
- port = random.randint(10000, 20000)
- # start an http server in a separate thread
- http_server = StreamingHTTPServer(host, port)
-
- # create topic
- topic_conf = PSTopic(ps_zone.conn, topic_name)
- _, status = topic_conf.set_config()
- assert_equal(status/100, 2)
- # create bucket on the first of the rados zones
- bucket = master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- # create notifications
- notification_conf = PSNotification(ps_zone.conn, bucket_name,
- topic_name)
- _, status = notification_conf.set_config()
- assert_equal(status/100, 2)
- # create subscription
- sub_conf = PSSubscription(ps_zone.conn, bucket_name+SUB_SUFFIX,
- topic_name, endpoint='http://'+host+':'+str(port))
- _, status = sub_conf.set_config()
- assert_equal(status/100, 2)
- # create objects in the bucket
- number_of_objects = 10
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- key.set_contents_from_string('bar')
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
- # check http server
- keys = list(bucket.list())
- # TODO: use exact match
- http_server.verify_events(keys, exact_match=False)
-
- # delete objects from the bucket
- for key in bucket.list():
- key.delete()
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
- # check http server
- # TODO: use exact match
- http_server.verify_events(keys, deletions=True, exact_match=False)
-
- # cleanup
- sub_conf.del_config()
- notification_conf.del_config()
- topic_conf.del_config()
- master_zone.delete_bucket(bucket_name)
- http_server.close()
-
-
-def test_ps_s3_push_http():
- """ test pushing to http endpoint s3 record format"""
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name+TOPIC_SUFFIX
-
- # create random port for the http server
- host = get_ip()
- port = random.randint(10000, 20000)
- # start an http server in a separate thread
- http_server = StreamingHTTPServer(host, port)
-
- # create topic
- topic_conf = PSTopic(ps_zone.conn, topic_name,
- endpoint='http://'+host+':'+str(port))
- result, status = topic_conf.set_config()
- assert_equal(status/100, 2)
- parsed_result = json.loads(result)
- topic_arn = parsed_result['arn']
- # create bucket on the first of the rados zones
- bucket = master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name,
- 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:*', 's3:ObjectRemoved:*']
- }]
- s3_notification_conf = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
- _, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
- # create objects in the bucket
- number_of_objects = 10
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- key.set_contents_from_string('bar')
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
- # check http server
- keys = list(bucket.list())
- # TODO: use exact match
- http_server.verify_s3_events(keys, exact_match=False)
-
- # delete objects from the bucket
- for key in bucket.list():
- key.delete()
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
- # check http server
- # TODO: use exact match
- http_server.verify_s3_events(keys, deletions=True, exact_match=False)
-
- # cleanup
- s3_notification_conf.del_config()
- topic_conf.del_config()
- master_zone.delete_bucket(bucket_name)
- http_server.close()
-
-
-def test_ps_push_amqp():
- """ test pushing to amqp endpoint """
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- hostname = get_ip()
- proc = init_rabbitmq()
- if proc is None:
- return SkipTest('end2end amqp tests require rabbitmq-server installed')
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name+TOPIC_SUFFIX
-
- # create topic
- exchange = 'ex1'
- task, receiver = create_amqp_receiver_thread(exchange, topic_name)
- task.start()
- topic_conf = PSTopic(ps_zone.conn, topic_name)
- _, status = topic_conf.set_config()
- assert_equal(status/100, 2)
- # create bucket on the first of the rados zones
- bucket = master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- # create notifications
- notification_conf = PSNotification(ps_zone.conn, bucket_name,
- topic_name)
- _, status = notification_conf.set_config()
- assert_equal(status/100, 2)
- # create subscription
- sub_conf = PSSubscription(ps_zone.conn, bucket_name+SUB_SUFFIX,
- topic_name, endpoint='amqp://'+hostname,
- endpoint_args='amqp-exchange='+exchange+'&amqp-ack-level=broker')
- _, status = sub_conf.set_config()
- assert_equal(status/100, 2)
- # create objects in the bucket
- number_of_objects = 10
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- key.set_contents_from_string('bar')
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
- # check amqp receiver
- keys = list(bucket.list())
- # TODO: use exact match
- receiver.verify_events(keys, exact_match=False)
-
- # delete objects from the bucket
- for key in bucket.list():
- key.delete()
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
- # check amqp receiver
- # TODO: use exact match
- receiver.verify_events(keys, deletions=True, exact_match=False)
-
- # cleanup
- stop_amqp_receiver(receiver, task)
- sub_conf.del_config()
- notification_conf.del_config()
- topic_conf.del_config()
- master_zone.delete_bucket(bucket_name)
- clean_rabbitmq(proc)
-
-
-def test_ps_s3_push_amqp():
- """ test pushing to amqp endpoint s3 record format"""
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- hostname = get_ip()
- proc = init_rabbitmq()
- if proc is None:
- return SkipTest('end2end amqp tests require rabbitmq-server installed')
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name+TOPIC_SUFFIX
-
- # create topic
- exchange = 'ex1'
- task, receiver = create_amqp_receiver_thread(exchange, topic_name)
- task.start()
- topic_conf = PSTopic(ps_zone.conn, topic_name,
- endpoint='amqp://' + hostname,
- endpoint_args='amqp-exchange=' + exchange + '&amqp-ack-level=none')
- result, status = topic_conf.set_config()
- assert_equal(status/100, 2)
- parsed_result = json.loads(result)
- topic_arn = parsed_result['arn']
- # create bucket on the first of the rados zones
- bucket = master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name,
- 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:*', 's3:ObjectRemoved:*']
- }]
- s3_notification_conf = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
- _, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
- # create objects in the bucket
- number_of_objects = 10
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- key.set_contents_from_string('bar')
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
- # check amqp receiver
- keys = list(bucket.list())
- # TODO: use exact match
- receiver.verify_s3_events(keys, exact_match=False)
-
- # delete objects from the bucket
- for key in bucket.list():
- key.delete()
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
- # check amqp receiver
- # TODO: use exact match
- receiver.verify_s3_events(keys, deletions=True, exact_match=False)
-
- # cleanup
- stop_amqp_receiver(receiver, task)
- s3_notification_conf.del_config()
- topic_conf.del_config()
- master_zone.delete_bucket(bucket_name)
- clean_rabbitmq(proc)
-
-
-def test_ps_delete_bucket():
- """ test notification status upon bucket deletion """
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- # create bucket on the first of the rados zones
- bucket = master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- topic_name = bucket_name + TOPIC_SUFFIX
- # create topic
- topic_name = bucket_name + TOPIC_SUFFIX
- topic_conf = PSTopic(ps_zone.conn, topic_name)
- response, status = topic_conf.set_config()
- assert_equal(status/100, 2)
- parsed_result = json.loads(response)
- topic_arn = parsed_result['arn']
- # create one s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name,
- 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:*']
- }]
- s3_notification_conf = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
- response, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # create non-s3 notification
- notification_conf = PSNotification(ps_zone.conn, bucket_name,
- topic_name)
- _, status = notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # create objects in the bucket
- number_of_objects = 10
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- key.set_contents_from_string('bar')
- # wait for bucket sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
- keys = list(bucket.list())
- # delete objects from the bucket
- for key in bucket.list():
- key.delete()
- # wait for bucket sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
- # delete the bucket
- master_zone.delete_bucket(bucket_name)
- # wait for meta sync
- zone_meta_checkpoint(ps_zone.zone)
-
- # get the events from the auto-generated subscription
- sub_conf = PSSubscription(ps_zone.conn, notification_name,
- topic_name)
- result, _ = sub_conf.get_events()
- records = json.loads(result)
- # TODO: use exact match
- verify_s3_records_by_elements(records, keys, exact_match=False)
-
- # s3 notification is deleted with bucket
- _, status = s3_notification_conf.get_config(notification=notification_name)
- assert_equal(status, 404)
- # non-s3 notification is deleted with bucket
- _, status = notification_conf.get_config()
- assert_equal(status, 404)
- # cleanup
- sub_conf.del_config()
- topic_conf.del_config()
-
-
-def test_ps_missing_topic():
- """ test creating a subscription when no topic info exists"""
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name+TOPIC_SUFFIX
-
- # create bucket on the first of the rados zones
- master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_arn = 'arn:aws:sns:::' + topic_name
- topic_conf_list = [{'Id': notification_name,
- 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:*']
- }]
- s3_notification_conf = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
- try:
- s3_notification_conf.set_config()
- except:
- log.info('missing topic is expected')
- else:
- assert 'missing topic is expected'
-
- # cleanup
- master_zone.delete_bucket(bucket_name)
-
-
-def test_ps_s3_topic_update():
- """ test updating topic associated with a notification"""
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- rabbit_proc = init_rabbitmq()
- if rabbit_proc is None:
- return SkipTest('end2end amqp tests require rabbitmq-server installed')
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- topic_name = bucket_name+TOPIC_SUFFIX
-
- # create amqp topic
- hostname = get_ip()
- exchange = 'ex1'
- amqp_task, receiver = create_amqp_receiver_thread(exchange, topic_name)
- amqp_task.start()
- topic_conf = PSTopic(ps_zone.conn, topic_name,
- endpoint='amqp://' + hostname,
- endpoint_args='amqp-exchange=' + exchange + '&amqp-ack-level=none')
- result, status = topic_conf.set_config()
- assert_equal(status/100, 2)
- parsed_result = json.loads(result)
- topic_arn = parsed_result['arn']
- # get topic
- result, _ = topic_conf.get_config()
- # verify topic content
- parsed_result = json.loads(result)
- assert_equal(parsed_result['topic']['name'], topic_name)
- assert_equal(parsed_result['topic']['dest']['push_endpoint'], topic_conf.parameters['push-endpoint'])
-
- # create http server
- port = random.randint(10000, 20000)
- # start an http server in a separate thread
- http_server = StreamingHTTPServer(hostname, port)
-
- # create bucket on the first of the rados zones
- bucket = master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- # create s3 notification
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name,
- 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:*']
- }]
- s3_notification_conf = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
- _, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
- # create objects in the bucket
- number_of_objects = 10
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- key.set_contents_from_string('bar')
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
-
- keys = list(bucket.list())
- # TODO: use exact match
- receiver.verify_s3_events(keys, exact_match=False)
-
- # update the same topic with new endpoint
- topic_conf = PSTopic(ps_zone.conn, topic_name,
- endpoint='http://'+ hostname + ':' + str(port))
- _, status = topic_conf.set_config()
- assert_equal(status/100, 2)
- # get topic
- result, _ = topic_conf.get_config()
- # verify topic content
- parsed_result = json.loads(result)
- assert_equal(parsed_result['topic']['name'], topic_name)
- assert_equal(parsed_result['topic']['dest']['push_endpoint'], topic_conf.parameters['push-endpoint'])
-
- # delete current objects and create new objects in the bucket
- for key in bucket.list():
- key.delete()
- for i in range(number_of_objects):
- key = bucket.new_key(str(i+100))
- key.set_contents_from_string('bar')
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
-
- keys = list(bucket.list())
- # verify that notifications are still sent to amqp
- # TODO: use exact match
- receiver.verify_s3_events(keys, exact_match=False)
-
- # update notification to update the endpoint from the topic
- topic_conf_list = [{'Id': notification_name,
- 'TopicArn': topic_arn,
- 'Events': ['s3:ObjectCreated:*']
- }]
- s3_notification_conf = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
- _, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # delete current objects and create new objects in the bucket
- for key in bucket.list():
- key.delete()
- for i in range(number_of_objects):
- key = bucket.new_key(str(i+200))
- key.set_contents_from_string('bar')
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
-
- keys = list(bucket.list())
- # check that updates switched to http
- # TODO: use exact match
- http_server.verify_s3_events(keys, exact_match=False)
-
- # cleanup
- # delete objects from the bucket
- stop_amqp_receiver(receiver, amqp_task)
- for key in bucket.list():
- key.delete()
- s3_notification_conf.del_config()
- topic_conf.del_config()
- master_zone.delete_bucket(bucket_name)
- http_server.close()
- clean_rabbitmq(rabbit_proc)
-
-
-def test_ps_s3_notification_update():
- """ test updating the topic of a notification"""
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- hostname = get_ip()
- rabbit_proc = init_rabbitmq()
- if rabbit_proc is None:
- return SkipTest('end2end amqp tests require rabbitmq-server installed')
-
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- topic_name1 = bucket_name+'amqp'+TOPIC_SUFFIX
- topic_name2 = bucket_name+'http'+TOPIC_SUFFIX
-
- # create topics
- # start amqp receiver in a separate thread
- exchange = 'ex1'
- amqp_task, receiver = create_amqp_receiver_thread(exchange, topic_name1)
- amqp_task.start()
- # create random port for the http server
- http_port = random.randint(10000, 20000)
- # start an http server in a separate thread
- http_server = StreamingHTTPServer(hostname, http_port)
-
- topic_conf1 = PSTopic(ps_zone.conn, topic_name1,
- endpoint='amqp://' + hostname,
- endpoint_args='amqp-exchange=' + exchange + '&amqp-ack-level=none')
- result, status = topic_conf1.set_config()
- parsed_result = json.loads(result)
- topic_arn1 = parsed_result['arn']
- assert_equal(status/100, 2)
- topic_conf2 = PSTopic(ps_zone.conn, topic_name2,
- endpoint='http://'+hostname+':'+str(http_port))
- result, status = topic_conf2.set_config()
- parsed_result = json.loads(result)
- topic_arn2 = parsed_result['arn']
- assert_equal(status/100, 2)
-
- # create bucket on the first of the rados zones
- bucket = master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- # create s3 notification with topic1
- notification_name = bucket_name + NOTIFICATION_SUFFIX
- topic_conf_list = [{'Id': notification_name,
- 'TopicArn': topic_arn1,
- 'Events': ['s3:ObjectCreated:*']
- }]
- s3_notification_conf = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
- _, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
- # create objects in the bucket
- number_of_objects = 10
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- key.set_contents_from_string('bar')
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
-
- keys = list(bucket.list())
- # TODO: use exact match
- receiver.verify_s3_events(keys, exact_match=False);
-
- # update notification to use topic2
- topic_conf_list = [{'Id': notification_name,
- 'TopicArn': topic_arn2,
- 'Events': ['s3:ObjectCreated:*']
- }]
- s3_notification_conf = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
- _, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
-
- # delete current objects and create new objects in the bucket
- for key in bucket.list():
- key.delete()
- for i in range(number_of_objects):
- key = bucket.new_key(str(i+100))
- key.set_contents_from_string('bar')
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
-
- keys = list(bucket.list())
- # check that updates switched to http
- # TODO: use exact match
- http_server.verify_s3_events(keys, exact_match=False)
-
- # cleanup
- # delete objects from the bucket
- stop_amqp_receiver(receiver, amqp_task)
- for key in bucket.list():
- key.delete()
- s3_notification_conf.del_config()
- topic_conf1.del_config()
- topic_conf2.del_config()
- master_zone.delete_bucket(bucket_name)
- http_server.close()
- clean_rabbitmq(rabbit_proc)
-
-
-def test_ps_s3_multiple_topics_notification():
- """ test notification creation with multiple topics"""
- if skip_push_tests:
- return SkipTest("PubSub push tests don't run in teuthology")
- hostname = get_ip()
- rabbit_proc = init_rabbitmq()
- if rabbit_proc is None:
- return SkipTest('end2end amqp tests require rabbitmq-server installed')
-
- master_zone, ps_zone = init_env()
- bucket_name = gen_bucket_name()
- topic_name1 = bucket_name+'amqp'+TOPIC_SUFFIX
- topic_name2 = bucket_name+'http'+TOPIC_SUFFIX
-
- # create topics
- # start amqp receiver in a separate thread
- exchange = 'ex1'
- amqp_task, receiver = create_amqp_receiver_thread(exchange, topic_name1)
- amqp_task.start()
- # create random port for the http server
- http_port = random.randint(10000, 20000)
- # start an http server in a separate thread
- http_server = StreamingHTTPServer(hostname, http_port)
-
- topic_conf1 = PSTopic(ps_zone.conn, topic_name1,
- endpoint='amqp://' + hostname,
- endpoint_args='amqp-exchange=' + exchange + '&amqp-ack-level=none')
- result, status = topic_conf1.set_config()
- parsed_result = json.loads(result)
- topic_arn1 = parsed_result['arn']
- assert_equal(status/100, 2)
- topic_conf2 = PSTopic(ps_zone.conn, topic_name2,
- endpoint='http://'+hostname+':'+str(http_port))
- result, status = topic_conf2.set_config()
- parsed_result = json.loads(result)
- topic_arn2 = parsed_result['arn']
- assert_equal(status/100, 2)
-
- # create bucket on the first of the rados zones
- bucket = master_zone.create_bucket(bucket_name)
- # wait for sync
- zone_meta_checkpoint(ps_zone.zone)
- # create s3 notification
- notification_name1 = bucket_name + NOTIFICATION_SUFFIX + '_1'
- notification_name2 = bucket_name + NOTIFICATION_SUFFIX + '_2'
- topic_conf_list = [
- {
- 'Id': notification_name1,
- 'TopicArn': topic_arn1,
- 'Events': ['s3:ObjectCreated:*']
- },
- {
- 'Id': notification_name2,
- 'TopicArn': topic_arn2,
- 'Events': ['s3:ObjectCreated:*']
- }]
- s3_notification_conf = PSNotificationS3(ps_zone.conn, bucket_name, topic_conf_list)
- _, status = s3_notification_conf.set_config()
- assert_equal(status/100, 2)
- result, _ = s3_notification_conf.get_config()
- assert_equal(len(result['TopicConfigurations']), 2)
- assert_equal(result['TopicConfigurations'][0]['Id'], notification_name1)
- assert_equal(result['TopicConfigurations'][1]['Id'], notification_name2)
-
- # get auto-generated subscriptions
- sub_conf1 = PSSubscription(ps_zone.conn, notification_name1,
- topic_name1)
- _, status = sub_conf1.get_config()
- assert_equal(status/100, 2)
- sub_conf2 = PSSubscription(ps_zone.conn, notification_name2,
- topic_name2)
- _, status = sub_conf2.get_config()
- assert_equal(status/100, 2)
-
- # create objects in the bucket
- number_of_objects = 10
- for i in range(number_of_objects):
- key = bucket.new_key(str(i))
- key.set_contents_from_string('bar')
- # wait for sync
- zone_bucket_checkpoint(ps_zone.zone, master_zone.zone, bucket_name)
-
- # get the events from both of the subscription
- result, _ = sub_conf1.get_events()
- records = json.loads(result)
- for record in records['Records']:
- log.debug(record)
- keys = list(bucket.list())
- # TODO: use exact match
- verify_s3_records_by_elements(records, keys, exact_match=False)
- receiver.verify_s3_events(keys, exact_match=False)
-
- result, _ = sub_conf2.get_events()
- parsed_result = json.loads(result)
- for record in parsed_result['Records']:
- log.debug(record)
- keys = list(bucket.list())
- # TODO: use exact match
- verify_s3_records_by_elements(records, keys, exact_match=False)
- http_server.verify_s3_events(keys, exact_match=False)
-
- # cleanup
- stop_amqp_receiver(receiver, amqp_task)
- s3_notification_conf.del_config()
- topic_conf1.del_config()
- topic_conf2.del_config()
- # delete objects from the bucket
- for key in bucket.list():
- key.delete()
- master_zone.delete_bucket(bucket_name)
- http_server.close()
- clean_rabbitmq(rabbit_proc)
projects / ceph.git / commitdiff