From e7f77f283ffdccef5c201884b084fca246a6adee Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Tue, 9 Apr 2019 19:35:05 +0200 Subject: [PATCH] Some python yaml fixes libyaml now needs to Loader keyword to not throw a warning, use safe_load instead everywhere. sort_keys isn't supported for reasons surpassing understanding, so switch to ruamel.yaml for the places where we write contents back out. Our ruamel.yaml wrapper strips extra leading space, which is needed when the root object is a list. When the root object is NOT a list, it's not needed. Add a flag to the constructor to toggle the behavior. For gerrit/projects.yaml this leads to a different wrapping of description fields, which is included here. Let ruamel.yaml handle comments Since we've switched to ruamel.yaml, we don't need to strip comments. Stop stripping them. Intra-list comments aren't supported, so just remove them. Please don't ask me why we can't have a space before openstack-merlin. Change-Id: I72776774eabd4e397ee5e8bdd58b3632be8fd3a1 --- gerrit/projects.yaml | 151 +++++++++++++------------ gerritbot/channels.yaml | 3 - tools/check-channels-yaml.sh | 4 +- tools/check_gerrit_projects_changed.py | 4 +- tools/check_irc_access.py | 2 +- tools/fix-neutron-stadium.py | 2 +- tools/irc_tests.py | 4 +- tools/normalize_channels_yaml.py | 23 +--- tools/normalize_projects_yaml.py | 13 +-- tools/projectconfig_ruamellib.py | 13 ++- tools/projectconfig_yamllib.py | 80 ------------- tox.ini | 12 +- 12 files changed, 114 insertions(+), 197 deletions(-) delete mode 100755 tools/projectconfig_yamllib.py diff --git a/gerrit/projects.yaml b/gerrit/projects.yaml index bfffbcc7f2..671b0e4421 100755 --- a/gerrit/projects.yaml +++ b/gerrit/projects.yaml @@ -924,9 +924,9 @@ - zuul description: Large media files for the Zuul website - project: openstack/adjutant - description: A simple workflow framework to help automate admin and user tasks in - and around OpenStack via a pluggable API exposing tasks made up of easily chainable - actions. + description: A simple workflow framework to help automate admin and user tasks + in and around OpenStack via a pluggable API exposing tasks made up of easily + chainable actions. use-storyboard: true groups: - adjutant @@ -1080,8 +1080,8 @@ - project: openstack/almanach description: Record the utilization of OpenStack resources for each tenant - project: openstack/anchor - description: RETIRED - An Ephemeral PKI system that can act as a trust anchor for - OpenStack PKI operations + description: RETIRED - An Ephemeral PKI system that can act as a trust anchor + for OpenStack PKI operations acl-config: /home/gerrit2/acls/openstack/retired.config - project: openstack/ansible-config_template description: Ansible action plugin to template arbitrary json ini yaml data @@ -1100,7 +1100,8 @@ use-storyboard: true description: Ansible role to install and set up ARA - project: openstack/ansible-role-atos-hsm - description: Ansible role to manage ATOS Hardware Security Module (HSM) client software + description: Ansible role to manage ATOS Hardware Security Module (HSM) client + software - project: openstack/ansible-role-bindep description: Ansible role to manage bindep - project: openstack/ansible-role-borgbackup @@ -1458,7 +1459,8 @@ use-storyboard: true groups: - ara - description: Common integration and infrastructure code for the different ARA projects + description: Common integration and infrastructure code for the different ARA + projects acl-config: /home/gerrit2/acls/openstack/ara.config - project: openstack/ara-plugins description: RETIRED, this project was merged into a single ara repository @@ -1502,7 +1504,8 @@ description: Friendly state machines for python. - project: openstack/bandit acl-config: /home/gerrit2/acls/openstack/retired.config - description: RETIRED, Python AST-based static analyzer from OpenStack Security Group + description: RETIRED, Python AST-based static analyzer from OpenStack Security + Group - project: openstack/bansho description: Web interface for Surveil acl-config: /home/gerrit2/acls/openstack/surveil.config @@ -1531,8 +1534,8 @@ description: Dynamic allocation system for Bareon. acl-config: /home/gerrit2/acls/openstack/bareon.config - project: openstack/bareon-api - description: REST API interface for Bareon, flexible and easy to use operating system - installation tool. + description: REST API interface for Bareon, flexible and easy to use operating + system installation tool. acl-config: /home/gerrit2/acls/openstack/bareon.config - project: openstack/bareon-image-elements description: Disk image elements for Bareon @@ -2137,8 +2140,8 @@ - project: openstack/cognitive description: Machine Learning as a Service - project: openstack/collectd-openstack-plugins - description: A collection of collectd plugins to publish metrics and events to OpenStack - services. + description: A collection of collectd plugins to publish metrics and events to + OpenStack services. - project: openstack/compass-adapters groups: - compass @@ -3428,8 +3431,8 @@ - project: openstack/devstack-plugin-libvirt-qemu description: Libvirt QEMU plugin - project: openstack/devstack-plugin-mariadb - description: A DevStack plugin that allows ubuntu users to use MariaDB flavor as - the database backend + description: A DevStack plugin that allows ubuntu users to use MariaDB flavor + as the database backend - project: openstack/devstack-plugin-nfs description: DevStack plugin to configure NFS backend. - project: openstack/devstack-plugin-pika @@ -3535,8 +3538,8 @@ groups: - fenix - project: openstack/fixtures-git - description: A git fixture using the fixtures API for writing tests for tools that - use git. + description: A git fixture using the fixtures API for writing tests for tools + that use git. acl-config: /home/gerrit2/acls/openstack/git-upstream.config - project: openstack/flame description: Automatic Heat template generation @@ -3760,9 +3763,9 @@ - project: openstack/fuel-main groups: - fuel - description: Fuel is the control plane for installing and managing OpenStack. It - is capable of doing bare metal provisioning of physical nodes, installing OpenStack - on them, and much more. + description: Fuel is the control plane for installing and managing OpenStack. + It is capable of doing bare metal provisioning of physical nodes, installing + OpenStack on them, and much more. docimpact-group: fuel options: - delay-release @@ -3890,8 +3893,8 @@ - project: openstack/fuel-plugin-ceph-multibackend groups: - fuel-plugins - description: Fuel plugin to deploy configure separate Ceph pools for different storage - backends + description: Fuel plugin to deploy configure separate Ceph pools for different + storage backends docimpact-group: fuel-plugins options: - delay-release @@ -4053,8 +4056,8 @@ - project: openstack/fuel-plugin-external-lb groups: - fuel-plugins - description: Fuel plugin to enable use of external LB instead of default VIPs and - HAproxy + description: Fuel plugin to enable use of external LB instead of default VIPs + and HAproxy docimpact-group: fuel-plugins - project: openstack/fuel-plugin-external-zabbix groups: @@ -4134,8 +4137,8 @@ options: - delay-release - project: openstack/fuel-plugin-ldap-slaves - description: Fuel plugin that provides the functionality to use external LDAP directory - as source for authentication on OpenStack nodes + description: Fuel plugin that provides the functionality to use external LDAP + directory as source for authentication on OpenStack nodes docimpact-group: fuel-plugins options: - delay-release @@ -4530,7 +4533,8 @@ - project: openstack/futurist description: A collection of async functionality and additions from the future. - project: openstack/fuxi - description: RETIRED, Enable Docker container to use Cinder volume and Manila share + description: RETIRED, Enable Docker container to use Cinder volume and Manila + share acl-config: /home/gerrit2/acls/openstack/retired.config - project: openstack/fuxi-golang description: RETIRED, Implement fuxi in go language @@ -4718,8 +4722,8 @@ - project: openstack/inception description: Inception Cloud Project - project: openstack/instack - description: RETIRED. A utility to run diskimage-builder elements in place on a - running host + description: RETIRED. A utility to run diskimage-builder elements in place on + a running host acl-config: /home/gerrit2/acls/openstack/retired.config - project: openstack/instack-undercloud description: A utility to run diskimage-builder undercloud elements on a running @@ -4891,8 +4895,8 @@ keys. acl-config: /home/gerrit2/acls/openstack/retired.config - project: openstack/kloudbuster - description: A tool to load OpenStack clouds end to end in both control plane and - data plane. + description: A tool to load OpenStack clouds end to end in both control plane + and data plane. - project: openstack/kolla description: Kolla provides production-ready containers and deployment tools for operating OpenStack clouds @@ -4916,8 +4920,8 @@ - project: openstack/ku.stella description: Guarantee of performance and security SLA for OpenStack clouds - project: openstack/kuryr - description: Bridge between container framework networking and storage models to - OpenStack networking and storage abstractions. + description: Bridge between container framework networking and storage models + to OpenStack networking and storage abstractions. - project: openstack/kuryr-kubernetes description: Kubernetes integration with OpenStack networking - project: openstack/kuryr-libnetwork @@ -4929,7 +4933,8 @@ - project: openstack/ldappool description: A simple connector pool for python-ldap and related LDAP libraries. - project: openstack/liberasurecode - description: Erasure Code API library written in C with pluggable Erasure Code backends. + description: Erasure Code API library written in C with pluggable Erasure Code + backends. - project: openstack/limestone-ci-cloud description: CI/CD Deployment of Limestone Networks nodepool cloud - project: openstack/loci @@ -5177,7 +5182,8 @@ use-storyboard: true acl-config: /home/gerrit2/acls/openstack/monasca.config - project: openstack/monasca-statsd - description: Python Statsd library for sending statsd messages via the Monasca Agent + description: Python Statsd library for sending statsd messages via the Monasca + Agent groups: - monasca use-storyboard: true @@ -5392,7 +5398,8 @@ - project: openstack/networking-powervm description: Neutron drivers for PowerVM network management. - project: openstack/networking-sfc - description: API's and implementations to support Service Function Chaining in Neutron. + description: API's and implementations to support Service Function Chaining in + Neutron. - project: openstack/networking-spp description: Neutron drivers for Soft Patch Panel. - project: openstack/networking-vpp @@ -5957,8 +5964,8 @@ - translate description: OpenStack Manuals - project: openstack/openstack-map - description: Base data to generate the OpenStack map and the Software section of - the www.openstack.org website. + description: Base data to generate the OpenStack map and the Software section + of the www.openstack.org website. - project: openstack/openstack-mentoring-scripts description: Various scripts to make matching and following up with mentees easier - project: openstack/openstack-planet @@ -5980,8 +5987,8 @@ OpenStack resources in the Kubernetes Service Catalog. use-storyboard: true - project: openstack/openstack-spaceport - description: Environment, dependency and bootstrapping utilities for OpenStack and - other services + description: Environment, dependency and bootstrapping utilities for OpenStack + and other services - project: openstack/openstack-specs description: OpenStack wide specs, ran by the Cross-Project Spec Liaisons - project: openstack/openstack-ux @@ -6082,7 +6089,8 @@ groups: - placement - project: openstack/os-vif - description: Integration library between network (Neutron) and compute (Nova) providers + description: Integration library between network (Neutron) and compute (Nova) + providers - project: openstack/os-win description: Windows / Hyper-V library for OpenStack projects. - project: openstack/os-xenapi @@ -6157,7 +6165,8 @@ options: - translate - project: openstack/oslo.reports - description: OpenStack library for creating Guru Meditation Reports and other reports + description: OpenStack library for creating Guru Meditation Reports and other + reports options: - translate - project: openstack/oslo.rootwrap @@ -6189,8 +6198,8 @@ - project: openstack/oslotest description: OpenStack test classes - project: openstack/osops-coda - description: Coda is a Horizon dashboard and panel (both share the name) that facilitates - resource clean up of a project once that project is no longer needed + description: Coda is a Horizon dashboard and panel (both share the name) that + facilitates resource clean up of a project once that project is no longer needed acl-config: /home/gerrit2/acls/openstack/osops.config groups: - osops @@ -6259,8 +6268,8 @@ - project: openstack/performa description: Distributed scenario runner, results processor and report generator - project: openstack/performance-docs - description: OpenStack performance testing plans, results, investigations and accompanying - pieces of testing scripts and Rally scenarios. + description: OpenStack performance testing plans, results, investigations and + accompanying pieces of testing scripts and Rally scenarios. - project: openstack/picasso description: Functions as a Service - project: openstack/placement @@ -6283,8 +6292,8 @@ - project: openstack/project-team-guide description: OpenStack Project Team Guide - project: openstack/proliantutils - description: Utility library for managing HP Proliant servers (used in Ironic iLO - drivers). + description: Utility library for managing HP Proliant servers (used in Ironic + iLO drivers). - project: openstack/promstat description: Report stats to both statsd and prometheus use-storyboard: true @@ -6707,8 +6716,8 @@ groups: - namos - project: openstack/python-neutron-pd-driver - description: RETIRED, A prefix delegation driver written using pure python for use - with OpenStack Neutron. + description: RETIRED, A prefix delegation driver written using pure python for + use with OpenStack Neutron. acl-config: /home/gerrit2/acls/openstack/retired.config - project: openstack/python-neutronclient description: OpenStack Networking (Neutron) Client @@ -7455,7 +7464,8 @@ description: Blueprints for Surveil acl-config: /home/gerrit2/acls/openstack/surveil.config - project: openstack/sushy - description: Sushy is a small Python library to communicate with Redfish based systems + description: Sushy is a small Python library to communicate with Redfish based + systems use-storyboard: true groups: - ironic @@ -7494,8 +7504,9 @@ acl-config: /home/gerrit2/acls/openstack/synergy.config - project: openstack/synergy-service description: Synergy is an extensible general purpose management service for OpenStack. - Its capabilities are implemented by a collection of managers. It can be used to - implement advanced scheduling mechanisms to maximize the overall resource usage. + Its capabilities are implemented by a collection of managers. It can be used + to implement advanced scheduling mechanisms to maximize the overall resource + usage. acl-config: /home/gerrit2/acls/openstack/synergy.config - project: openstack/syntribos description: Python API security testing tool from OpenStack Security Group @@ -7579,8 +7590,8 @@ - project: openstack/terracotta description: Dynamic resource scheduling service for OpenStack cloud. - project: openstack/third-party-ci-tools - description: Repository for third party continuous integration teams to share tools, - configurations, plug-ins and documents. + description: Repository for third party continuous integration teams to share + tools, configurations, plug-ins and documents. - project: openstack/timmy description: 'OpenStack Ansible-like tool for parallel node operations: two-way data transfer, log collection, remote command execution' @@ -7691,8 +7702,8 @@ groups: - tripleo - project: openstack/tripleo-validations - description: A collection of Ansible playbooks to detect and report potential issues - during TripleO deployments + description: A collection of Ansible playbooks to detect and report potential + issues during TripleO deployments acl-config: /home/gerrit2/acls/openstack/tripleo.config use-storyboard: true groups: @@ -7729,8 +7740,8 @@ groups: - fuel description: A service to process unstructured serialized configuration data and - facilitate exchange of this data between deployment services such as Fuel or Puppet - Master and deployed OpenStack components. + facilitate exchange of this data between deployment services such as Fuel or + Puppet Master and deployed OpenStack components. docimpact-group: fuel options: - delay-release @@ -8055,10 +8066,10 @@ description: RETIRED, HAOS is a suite of HA/destructive tests for OpenStack clouds acl-config: /home/gerrit2/acls/stackforge/retired.config - project: stackforge/healthnmon - description: RETIRED, Healthnmon aims to deliver 'Cloud Resource Monitor', an extensible - service to OpenStack Cloud Operating system by providing monitoring service for - Cloud Resources and Infrastructure with a pluggable framework for 'Inventory Management', - 'Alerts and notifications' and 'Utilization Data.' + description: RETIRED, Healthnmon aims to deliver 'Cloud Resource Monitor', an + extensible service to OpenStack Cloud Operating system by providing monitoring + service for Cloud Resources and Infrastructure with a pluggable framework for + 'Inventory Management', 'Alerts and notifications' and 'Utilization Data.' acl-config: /home/gerrit2/acls/stackforge/retired.config - project: stackforge/heat-coe-templates description: RETIRED, A collection of Heat templates for deploying a variety of @@ -8138,8 +8149,8 @@ description: RETIRED, Vagrant environment providing a Packstack installation. acl-config: /home/gerrit2/acls/stackforge/retired.config - project: stackforge/pecan - description: RETIRED, A WSGI object-dispatching web framework, designed to be lean - and fast with few dependencies. + description: RETIRED, A WSGI object-dispatching web framework, designed to be + lean and fast with few dependencies. acl-config: /home/gerrit2/acls/stackforge/retired.config - project: stackforge/prep-source-repos description: RETIRED, Check out repos hosted on gerrit and apply a list of outstanding @@ -8152,8 +8163,8 @@ description: RETIRED, OpenStack Examples Puppet Module acl-config: /home/gerrit2/acls/stackforge/retired.config - project: stackforge/puppet-openstack-cloud - description: RETIRED, Flexible Puppet implementation capable of configuring a scalable - OpenStack Cloud + description: RETIRED, Flexible Puppet implementation capable of configuring a + scalable OpenStack Cloud acl-config: /home/gerrit2/acls/stackforge/retired.config - project: stackforge/puppet-openstack_dev_env description: RETIRED, OpenStack Puppet Dev Environment @@ -8197,8 +8208,8 @@ description: RETIRED, An OpenSource merge, build, and packaging tool for deployers. acl-config: /home/gerrit2/acls/stackforge/retired.config - project: stackforge/swiftpolicy - description: RETIRED, SwiftPolicy Middleware for OpenStack Swift allows to use a - JSON policy file to handle swift authorizations. + description: RETIRED, SwiftPolicy Middleware for OpenStack Swift allows to use + a JSON policy file to handle swift authorizations. acl-config: /home/gerrit2/acls/stackforge/retired.config - project: stackforge/swiftsync description: RETIRED, Swift mass syncronizer @@ -8211,8 +8222,8 @@ description: RETIRED, Update a TripleO installation using Ansible. acl-config: /home/gerrit2/acls/stackforge/retired.config - project: stackforge/warm - description: RETIRED, Provides the ability to deploy OpenStack resources from Yaml - templates. + description: RETIRED, Provides the ability to deploy OpenStack resources from + Yaml templates. acl-config: /home/gerrit2/acls/stackforge/retired.config - project: stackforge/windows-diskimage-builder description: RETIRED, Disk image builder for windows systems. diff --git a/gerritbot/channels.yaml b/gerritbot/channels.yaml index c61d5e72ce..7b1ba070c7 100755 --- a/gerritbot/channels.yaml +++ b/gerritbot/channels.yaml @@ -336,7 +336,6 @@ openstack-ansible: projects: # All openstack-ansible projects - ^openstack/openstack-ansible.* - # Then alphabetical order - openstack/ansible-config_template - openstack/ansible-hardening - openstack/ansible-role-atos-hsm @@ -450,7 +449,6 @@ openstack-chef: projects: # all openstack-chef projects - ^openstack/cookbook.* - # then alphabetical order - openstack/openstack-chef - openstack/openstack-chef-repo - openstack/openstack-chef-specs @@ -1427,7 +1425,6 @@ openstack-merges: - openstack/trove-dashboard - openstack/volume-api - openstack/zaqar - openstack-merlin: events: - patchset-created diff --git a/tools/check-channels-yaml.sh b/tools/check-channels-yaml.sh index 1f3dbaf1a6..641a88beff 100755 --- a/tools/check-channels-yaml.sh +++ b/tools/check-channels-yaml.sh @@ -5,9 +5,7 @@ trap "rm -rf $T" EXIT channels_file=${1:-gerritbot/channels.yaml} -# strip comments so that output can be compared meaningfully -(printf "# This file is sorted alphabetically by channel name.\n"; sed '/^[[:space:]]*#.*$/d;s/[[:space:]]*#.*$//' gerritbot/channels.yaml) > $T/comments-removed ./tools/normalize_channels_yaml.py >$T/regenned echo "Checking whether entries are sorted alphabetically" -diff -u $T/comments-removed $T/regenned +diff -u $channels_file $T/regenned diff --git a/tools/check_gerrit_projects_changed.py b/tools/check_gerrit_projects_changed.py index c18ceed143..3352732601 100644 --- a/tools/check_gerrit_projects_changed.py +++ b/tools/check_gerrit_projects_changed.py @@ -64,8 +64,8 @@ def main(): ) args = parser.parse_args() - projects_old = yaml.load(open(args.oldfile, 'r')) - projects_new = yaml.load(open(args.newfile, 'r')) + projects_old = yaml.safe_load(open(args.oldfile, 'r')) + projects_new = yaml.safe_load(open(args.newfile, 'r')) ps_old = {} for p in projects_old: diff --git a/tools/check_irc_access.py b/tools/check_irc_access.py index 41dcb2dbee..ff6a72081f 100755 --- a/tools/check_irc_access.py +++ b/tools/check_irc_access.py @@ -142,7 +142,7 @@ def main(): help='the nick for which access should be validated') args = parser.parse_args() - config = yaml.load(open(args.config)) + config = yaml.safe_load(open(args.config)) channels = [] for channel in config['channels']: channels.append('#' + channel['name']) diff --git a/tools/fix-neutron-stadium.py b/tools/fix-neutron-stadium.py index b264938b47..204f548db1 100644 --- a/tools/fix-neutron-stadium.py +++ b/tools/fix-neutron-stadium.py @@ -108,7 +108,7 @@ def ordered_load(stream, *args, **kwargs): yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_yaml_map) - return yaml.load(stream=stream, *args, **kwargs) + return yaml.safe_load(stream=stream, *args, **kwargs) def ordered_dump(data, stream=None, *args, **kwargs): diff --git a/tools/irc_tests.py b/tools/irc_tests.py index 7253651b12..b5194b2012 100755 --- a/tools/irc_tests.py +++ b/tools/irc_tests.py @@ -23,7 +23,7 @@ def access_gerrit_check(): errors = False - access_config = yaml.load(open('accessbot/channels.yaml', 'r')) + access_config = yaml.safe_load(open('accessbot/channels.yaml', 'r')) access_channels = [] for channel in access_config['channels']: @@ -31,7 +31,7 @@ def access_gerrit_check(): access_channel_set = set(access_channels) - gerrit_config = yaml.load(open('gerritbot/channels.yaml')) + gerrit_config = yaml.safe_load(open('gerritbot/channels.yaml')) print("Basic check of gerritbot/channels.yaml") REQUIRED_ENTRIES = ("branches", "events", "projects") diff --git a/tools/normalize_channels_yaml.py b/tools/normalize_channels_yaml.py index 7d317907d9..714b07794d 100755 --- a/tools/normalize_channels_yaml.py +++ b/tools/normalize_channels_yaml.py @@ -17,34 +17,19 @@ import locale import sys -import yaml -from collections import OrderedDict -import projectconfig_yamllib as pcy +import projectconfig_ruamellib def main(): + yaml = projectconfig_ruamellib.YAML(strip=False) locale.setlocale(locale.LC_COLLATE, 'C') - yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - pcy.construct_yaml_map) - - yaml.add_representer(OrderedDict, pcy.project_representer, - Dumper=pcy.IndentedDumper) - chandata = yaml.load(open('gerritbot/channels.yaml')) + for k, v in chandata.items(): v['projects'] = sorted(v['projects']) - sys.stdout.write('# This file is sorted alphabetically by channel name.\n') - first = True - for k in sorted(chandata.keys()): - if not first: - sys.stdout.write('\n') - first = False - sys.stdout.write(yaml.dump({k: chandata[k]}, default_flow_style=False, - Dumper=pcy.IndentedDumper, width=80, - sort_keys=False, - indent=2)) + yaml.dump(chandata, stream=sys.stdout) if __name__ == '__main__': main() diff --git a/tools/normalize_projects_yaml.py b/tools/normalize_projects_yaml.py index 10dcd47af2..f33f3affd0 100755 --- a/tools/normalize_projects_yaml.py +++ b/tools/normalize_projects_yaml.py @@ -15,18 +15,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import yaml -from collections import OrderedDict -import projectconfig_yamllib as pcy +import projectconfig_ruamellib def main(): - yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - pcy.construct_yaml_map) - - yaml.add_representer(OrderedDict, pcy.project_representer, - Dumper=pcy.IndentedDumper) + yaml = projectconfig_ruamellib.YAML() data = yaml.load(open('gerrit/projects.yaml')) for project in data: @@ -35,8 +29,7 @@ def main(): del project['upstream'] with open('gerrit/projects.yaml', 'w') as out: - out.write(yaml.dump(data, default_flow_style=False, - Dumper=pcy.IndentedDumper, width=80)) + yaml.dump(data, stream=out) if __name__ == '__main__': main() diff --git a/tools/projectconfig_ruamellib.py b/tools/projectconfig_ruamellib.py index 8ed6e7e5c1..c246fabc14 100755 --- a/tools/projectconfig_ruamellib.py +++ b/tools/projectconfig_ruamellib.py @@ -21,11 +21,18 @@ def none_representer(dumper, data): class YAML(object): - def __init__(self): + def __init__(self, strip=True): + """Wrap construction of ruamel yaml object. + + :param bool strip: + Whether or not to strip additional leading spaces at the beginning + of the line. This is only needed when the root object is a list. + """ self.yaml = ruamel.yaml.YAML() self.yaml.allow_duplicate_keys = True self.yaml.representer.add_representer(type(None), none_representer) self.yaml.indent(mapping=2, sequence=4, offset=2) + self.strip = strip def load(self, stream): return self.yaml.load(stream) @@ -41,4 +48,6 @@ class YAML(object): return '\n'.join(newlines) def dump(self, data, *args, **kwargs): - self.yaml.dump(data, *args, transform=self.tr, **kwargs) + if self.strip: + kwargs['transform'] = self.tr + self.yaml.dump(data, *args, **kwargs) diff --git a/tools/projectconfig_yamllib.py b/tools/projectconfig_yamllib.py deleted file mode 100755 index 2927cd4cd1..0000000000 --- a/tools/projectconfig_yamllib.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright (c) 2015 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import yaml -from collections import OrderedDict - - -def project_representer(dumper, data): - return dumper.represent_mapping('tag:yaml.org,2002:map', - data.items()) - - -def construct_yaml_map(self, node): - data = OrderedDict() - yield data - value = self.construct_mapping(node) - - if isinstance(node, yaml.MappingNode): - self.flatten_mapping(node) - else: - raise yaml.constructor.ConstructorError( - None, None, - 'expected a mapping node, but found %s' % node.id, - node.start_mark) - - mapping = OrderedDict() - for key_node, value_node in node.value: - key = self.construct_object(key_node, deep=False) - try: - hash(key) - except TypeError as exc: - raise yaml.constructor.ConstructorError( - 'while constructing a mapping', node.start_mark, - 'found unacceptable key (%s)' % exc, key_node.start_mark) - value = self.construct_object(value_node, deep=False) - mapping[key] = value - data.update(mapping) - - -class IndentedEmitter(yaml.emitter.Emitter): - def expect_block_sequence(self): - self.increase_indent(flow=False, indentless=False) - self.state = self.expect_first_block_sequence_item - - -class IndentedDumper(IndentedEmitter, yaml.serializer.Serializer, - yaml.representer.Representer, yaml.resolver.Resolver): - def __init__(self, stream, - default_style=None, default_flow_style=None, - canonical=None, indent=None, width=None, - allow_unicode=None, line_break=None, - encoding=None, explicit_start=None, explicit_end=None, - version=None, tags=None, sort_keys=True): - IndentedEmitter.__init__( - self, stream, canonical=canonical, - indent=indent, width=width, - allow_unicode=allow_unicode, - line_break=line_break) - yaml.serializer.Serializer.__init__( - self, encoding=encoding, - explicit_start=explicit_start, - explicit_end=explicit_end, - version=version, tags=tags) - yaml.representer.Representer.__init__( - self, default_style=default_style, - default_flow_style=default_flow_style, - sort_keys=sort_keys) - yaml.resolver.Resolver.__init__(self) diff --git a/tox.ini b/tox.ini index 0edd5fdf67..4e5cfb213d 100644 --- a/tox.ini +++ b/tox.ini @@ -55,8 +55,10 @@ commands = python -c 'import yaml ; yaml.load(open("zuul/site-variables.yaml", "r"))' [testenv:gerrit] -deps = PyYAML - GitPython +deps = + PyYAML + ruamel.yaml + GitPython commands = {toxinidir}/tools/check_valid_gerrit_projects.py gerrit/projects.yaml gerrit/acls {toxinidir}/tools/check_projects_yaml_alphabetized.sh gerrit/projects.yaml @@ -70,8 +72,10 @@ commands = {posargs} commands = python setup.py build_sphinx [testenv:irc] -deps = PyYAML - irc +deps = + PyYAML + ruamel.yaml + irc commands = {toxinidir}/tools/check_irc_access.py -l accessbot/channels.yaml openstackinfra {toxinidir}/tools/irc_tests.py