diff options
32 files changed, 535 insertions, 55 deletions
@@ -184,9 +184,10 @@ - designate-devstack-jobs - openstack-cover-jobs - openstack-lower-constraints-jobs - - openstack-python36-jobs - openstack-python-jobs - openstack-python35-jobs + - openstack-python36-jobs + - openstack-python37-jobs - publish-openstack-docs-pti - periodic-stable-jobs - check-requirements @@ -92,6 +92,7 @@ Execute a single test using py27 * Release notes: https://docs.openstack.org/releasenotes/designate/ * Source: https://git.openstack.org/cgit/openstack/designate * Bugs: https://bugs.launchpad.net/designate +* Blueprints: https://blueprints.launchpad.net/designate .. _OpenStack Gerrit Workflow: https://docs.openstack.org/infra/manual/developers.html#development-workflow diff --git a/contrib/archive/backends/impl_ipa/__init__.py b/contrib/archive/backends/impl_ipa/__init__.py index 2e5bc5dd..4576e76d 100644 --- a/contrib/archive/backends/impl_ipa/__init__.py +++ b/contrib/archive/backends/impl_ipa/__init__.py @@ -62,7 +62,11 @@ rectype2iparectype = {'A': ('arecord', '%(data)s'), 'NS': ('nsrecord', '%(data)s'), 'PTR': ('ptrrecord', '%(data)s'), 'SPF': ('spfrecord', '%(data)s'), - 'SSHFP': ('sshfprecord', '%(data)s')} + 'SSHFP': ('sshfprecord', '%(data)s'), + 'NAPTR': ('naptrrecord', '%(data)s'), + 'CAA': ('caarecord', '%(data)s'), + } + IPA_INVALID_DATA = 3009 IPA_NOT_FOUND = 4001 diff --git a/designate/__init__.py b/designate/__init__.py index 91afebd7..4dd3ceb4 100644 --- a/designate/__init__.py +++ b/designate/__init__.py @@ -69,7 +69,7 @@ designate_opts = [ # Supported record types cfg.ListOpt('supported-record-type', help='Supported record types', default=['A', 'AAAA', 'CNAME', 'MX', 'SRV', 'TXT', 'SPF', 'NS', - 'PTR', 'SSHFP', 'SOA']), + 'PTR', 'SSHFP', 'SOA', 'NAPTR', 'CAA']), ] # Set some Oslo Log defaults diff --git a/designate/dnsutils.py b/designate/dnsutils.py index 0fc9164a..b0342a6b 100644 --- a/designate/dnsutils.py +++ b/designate/dnsutils.py @@ -15,7 +15,6 @@ # under the License. import random import socket -import base64 import time from threading import Lock @@ -25,6 +24,7 @@ import dns.exception import dns.zone import eventlet from dns import rdatatype +from oslo_serialization import base64 from oslo_log import log as logging from oslo_config import cfg @@ -192,7 +192,7 @@ class TsigKeyring(object): tsigkey = self.storage.find_tsigkey( context.get_current(), criterion) - return base64.decodestring(tsigkey.secret) + return base64.decode_as_bytes(tsigkey.secret) except exceptions.TsigKeyNotFound: return default diff --git a/designate/objects/__init__.py b/designate/objects/__init__.py index c1acb072..451231a6 100644 --- a/designate/objects/__init__.py +++ b/designate/objects/__init__.py @@ -49,8 +49,10 @@ from designate.objects.zone_export import ZoneExport, ZoneExportList # noqa from designate.objects.rrdata_a import A, AList # noqa from designate.objects.rrdata_aaaa import AAAA, AAAAList # noqa +from designate.objects.rrdata_caa import CAA, CAAList # noqa from designate.objects.rrdata_cname import CNAME, CNAMEList # noqa from designate.objects.rrdata_mx import MX, MXList # noqa +from designate.objects.rrdata_naptr import NAPTR, NAPTRList # noqa from designate.objects.rrdata_ns import NS, NSList # noqa from designate.objects.rrdata_ptr import PTR, PTRList # noqa from designate.objects.rrdata_soa import SOA, SOAList # noqa diff --git a/designate/objects/fields.py b/designate/objects/fields.py index 33d5cd38..6fa9a691 100644 --- a/designate/objects/fields.py +++ b/designate/objects/fields.py @@ -98,6 +98,12 @@ class StringFields(ovoo_fields.StringField): RE_SSHFP_FINGERPRINT = r'^([0-9A-Fa-f]{10,40}|[0-9A-Fa-f]{64})\Z' RE_TLDNAME = r'^(?!.{255,})(?:(?!\-)[A-Za-z0-9_\-]{1,63}(?<!\-))' \ r'(?:\.(?:(?!\-)[A-Za-z0-9_\-]{1,63}(?<!\-)))*\Z' + RE_NAPTR_FLAGS = r'^(?!.*(.).*\1)[APSU]+$' + RE_NAPTR_SERVICE = r'^([A-Za-z]([A-Za-z0-9]*)(\+[A-Za-z]([A-Za-z0-9]{0,31}))*)?' # noqa + RE_NAPTR_REGEXP = r'^([^0-9i\\])(.*)\1((.+)|(\\[1-9]))\1(i?)' + RE_KVP = r'^\s[A-Za-z0-9]+=[A-Za-z0-9]+' + RE_URL_MAIL = r'^mailto:[A-Za-z0-9_\-]+@.*' + RE_URL_HTTP = r'^http(s)?://.*/' def __init__(self, nullable=False, read_only=False, default=ovoo_fields.UnspecifiedDefault, description='', @@ -291,6 +297,100 @@ class TldField(StringFields): return value +class NaptrFlagsField(StringFields): + def __init__(self, **kwargs): + super(NaptrFlagsField, self).__init__(**kwargs) + + def coerce(self, obj, attr, value): + value = super(NaptrFlagsField, self).coerce(obj, attr, value) + if (len(value) > 255): + raise ValueError("NAPTR record flags field cannot be longer than" + " 255 characters" % value) + if not re.match(self.RE_NAPTR_FLAGS, "%s" % value): + raise ValueError("NAPTR record flags can be S, A, U and P" % value) + return value + + +class NaptrServiceField(StringFields): + def __init__(self, **kwargs): + super(NaptrServiceField, self).__init__(**kwargs) + + def coerce(self, obj, attr, value): + value = super(NaptrServiceField, self).coerce(obj, attr, value) + if (len(value) > 255): + raise ValueError("NAPTR record service field cannot be longer than" + " 255 characters" % value) + if not re.match(self.RE_NAPTR_SERVICE, "%s" % value): + raise ValueError("%s NAPTR record service does not match" % value) + return value + + +class NaptrRegexpField(StringFields): + def __init__(self, **kwargs): + super(NaptrRegexpField, self).__init__(**kwargs) + + def coerce(self, obj, attr, value): + value = super(NaptrRegexpField, self).coerce(obj, attr, value) + if (len(value) > 255): + raise ValueError("NAPTR record regexp field cannot be longer than" + " 255 characters" % value) + if value: + if not re.match(self.RE_NAPTR_REGEXP, "%s" % value): + raise ValueError("%s is not a NAPTR record regexp" % value) + return value + + +class CaaPropertyField(StringFields): + def __init__(self, **kwargs): + super(CaaPropertyField, self).__init__(**kwargs) + + def coerce(self, obj, attr, value): + value = super(CaaPropertyField, self).coerce(obj, attr, value) + prpt = value.split(' ', 1) + tag = prpt[0] + val = prpt[1] + if (tag == 'issue' or tag == 'issuewild'): + entries = val.split(';') + idn = entries.pop(0) + domain = idn.split('.') + for host in domain: + if len(host) > 63: + raise ValueError("Host %s is too long" % host) + idn_with_dot = idn + '.' + if not re.match(self.RE_ZONENAME, idn_with_dot): + raise ValueError("Domain %s does not match" % idn) + for entry in entries: + if not re.match(self.RE_KVP, entry): + raise ValueError("%s is not valid key-value pair" % entry) + elif tag == 'iodef': + if re.match(self.RE_URL_MAIL, val): + parts = val.split('@') + idn = parts[1] + domain = idn.split('.') + for host in domain: + if len(host) > 63: + raise ValueError("Host %s is too long" % host) + idn_with_dot = idn + '.' + if not re.match(self.RE_ZONENAME, idn_with_dot): + raise ValueError("Domain %s does not match" % idn) + elif re.match(self.RE_URL_HTTP, val): + parts = val.split('/') + idn = parts[2] + domain = idn.split('.') + for host in domain: + if len(host) > 63: + raise ValueError("Host %s is too long" % host) + idn_with_dot = idn + '.' + if not re.match(self.RE_ZONENAME, idn_with_dot): + raise ValueError("Domain %s does not match" % idn) + else: + raise ValueError("%s is not valid URL" % val) + else: + raise ValueError("Property tag %s must be 'issue', 'issuewild'" + " or 'iodef'" % value) + return value + + class Any(ovoo_fields.FieldType): @staticmethod def coerce(obj, attr, value): diff --git a/designate/objects/rrdata_caa.py b/designate/objects/rrdata_caa.py new file mode 100644 index 00000000..fe5c6d92 --- /dev/null +++ b/designate/objects/rrdata_caa.py @@ -0,0 +1,54 @@ +# Copyright 2018 Canonical Ltd. +# +# Author: Tytus Kurek <tytus.kurek@canonical.com> +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from designate.objects.record import Record +from designate.objects.record import RecordList +from designate.objects import base +from designate.objects import fields + + +@base.DesignateRegistry.register +class CAA(Record): + """ + CAA Resource Record Type + Defined in: RFC6844 + """ + fields = { + 'flags': fields.IntegerFields(minimum=0, maximum=1), + 'prpt': fields.CaaPropertyField() + } + + def _to_string(self): + return ("%(flag)s %(prpt)s" % self) + + def _from_string(self, v): + flags, prpt = v.split(' ', 1) + self.flags = int(flags) + self.prpt = prpt + + # The record type is defined in the RFC. This will be used when the record + # is sent by mini-dns. + RECORD_TYPE = 257 + + +@base.DesignateRegistry.register +class CAAList(RecordList): + + LIST_ITEM_TYPE = CAA + + fields = { + 'objects': fields.ListOfObjectsField('CAA'), + } diff --git a/designate/objects/rrdata_naptr.py b/designate/objects/rrdata_naptr.py new file mode 100644 index 00000000..a126f971 --- /dev/null +++ b/designate/objects/rrdata_naptr.py @@ -0,0 +1,63 @@ +# Copyright 2018 Canonical Ltd. +# +# Author: Tytus Kurek <tytus.kurek@canonical.com> +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from designate.objects.record import Record +from designate.objects.record import RecordList +from designate.objects import base +from designate.objects import fields + + +@base.DesignateRegistry.register +class NAPTR(Record): + """ + NAPTR Resource Record Type + Defined in: RFC2915 + """ + fields = { + 'order': fields.IntegerFields(minimum=0, maximum=65535), + 'preference': fields.IntegerFields(minimum=0, maximum=65535), + 'flags': fields.NaptrFlagsField(), + 'service': fields.NaptrServiceField(), + 'regexp': fields.NaptrRegexpField(), + 'replacement': fields.DomainField(maxLength=255) + } + + def _to_string(self): + return ("%(order)s %(preference)s %(flags)s %(service)s %(regexp)s " + "%(replacement)s" % self) + + def _from_string(self, v): + order, preference, flags, service, regexp, replacement = v.split(' ') + self.order = int(order) + self.preference = int(preference) + self.flags = flags + self.service = service + self.regexp = regexp + self.replacement = replacement + + # The record type is defined in the RFC. This will be used when the record + # is sent by mini-dns. + RECORD_TYPE = 35 + + +@base.DesignateRegistry.register +class NAPTRList(RecordList): + + LIST_ITEM_TYPE = NAPTR + + fields = { + 'objects': fields.ListOfObjectsField('NAPTR'), + } diff --git a/designate/storage/impl_sqlalchemy/migrate_repo/versions/101_support_naptr_records.py b/designate/storage/impl_sqlalchemy/migrate_repo/versions/101_support_naptr_records.py new file mode 100644 index 00000000..4c03e6e8 --- /dev/null +++ b/designate/storage/impl_sqlalchemy/migrate_repo/versions/101_support_naptr_records.py @@ -0,0 +1,44 @@ +# Copyright 2018 Canonical Ltd. +# +# Author: Tytus Kurek <tytus.kurek@canonical.com> +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import MetaData, Table, Enum + +meta = MetaData() + + +def upgrade(migrate_engine): + meta.bind = migrate_engine + + RECORD_TYPES = ['A', 'AAAA', 'CNAME', 'MX', 'SRV', 'TXT', 'SPF', 'NS', + 'PTR', 'SSHFP', 'SOA', 'NAPTR'] + + records_table = Table('recordsets', meta, autoload=True) + records_table.columns.type.alter(name='type', type=Enum(*RECORD_TYPES)) + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + + RECORD_TYPES = ['A', 'AAAA', 'CNAME', 'MX', 'SRV', 'TXT', 'SPF', 'NS', + 'PTR', 'SSHFP', 'SOA'] + + records_table = Table('recordsets', meta, autoload=True) + + # Delete all NAPTR records + records_table.filter_by(name='type', type='NAPTR').delete() + + # Remove CAA from the ENUM + records_table.columns.type.alter(type=Enum(*RECORD_TYPES)) diff --git a/designate/storage/impl_sqlalchemy/migrate_repo/versions/102_support_caa_records.py b/designate/storage/impl_sqlalchemy/migrate_repo/versions/102_support_caa_records.py new file mode 100644 index 00000000..1bf61572 --- /dev/null +++ b/designate/storage/impl_sqlalchemy/migrate_repo/versions/102_support_caa_records.py @@ -0,0 +1,44 @@ +# Copyright 2018 Canonical Ltd. +# +# Author: Tytus Kurek <tytus.kurek@canonical.com> +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import MetaData, Table, Enum + +meta = MetaData() + + +def upgrade(migrate_engine): + meta.bind = migrate_engine + + RECORD_TYPES = ['A', 'AAAA', 'CNAME', 'MX', 'SRV', 'TXT', 'SPF', 'NS', + 'PTR', 'SSHFP', 'SOA', 'NAPTR', 'CAA'] + + records_table = Table('recordsets', meta, autoload=True) + records_table.columns.type.alter(name='type', type=Enum(*RECORD_TYPES)) + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + + RECORD_TYPES = ['A', 'AAAA', 'CNAME', 'MX', 'SRV', 'TXT', 'SPF', 'NS', + 'PTR', 'SSHFP', 'SOA', 'NAPTR'] + + records_table = Table('recordsets', meta, autoload=True) + + # Delete all CAA records + records_table.filter_by(name='type', type='CAA').delete() + + # Remove CAA from the ENUM + records_table.columns.type.alter(type=Enum(*RECORD_TYPES)) diff --git a/designate/storage/impl_sqlalchemy/tables.py b/designate/storage/impl_sqlalchemy/tables.py index feca30f3..8549cfc5 100644 --- a/designate/storage/impl_sqlalchemy/tables.py +++ b/designate/storage/impl_sqlalchemy/tables.py @@ -29,7 +29,8 @@ CONF = cfg.CONF RESOURCE_STATUSES = ['ACTIVE', 'PENDING', 'DELETED', 'ERROR'] RECORD_TYPES = ['A', 'AAAA', 'CNAME', 'MX', 'SRV', 'TXT', 'SPF', 'NS', 'PTR', - 'SSHFP', 'SOA'] + 'SSHFP', 'SOA', 'NAPTR', 'CAA'] + TASK_STATUSES = ['ACTIVE', 'PENDING', 'DELETED', 'ERROR', 'COMPLETE'] TSIG_ALGORITHMS = ['hmac-md5', 'hmac-sha1', 'hmac-sha224', 'hmac-sha256', 'hmac-sha384', 'hmac-sha512'] diff --git a/designate/tests/resources/zonefiles/example.com.zone b/designate/tests/resources/zonefiles/example.com.zone index 06f076a7..7ec56363 100644 --- a/designate/tests/resources/zonefiles/example.com.zone +++ b/designate/tests/resources/zonefiles/example.com.zone @@ -7,15 +7,15 @@ example.com. 600 IN SOA ns1.example.com. nsadmin.example.com. ( 10800 ; minimum ) ipv4.example.com. 300 IN A 192.0.0.1 -ipv6.example.com. IN AAAA fd00::1 -cname.example.com. IN CNAME example.com. -example.com. IN MX 5 192.0.0.2 -example.com. IN MX 10 192.0.0.3 -_http._tcp.example.com. IN SRV 10 0 80 192.0.0.4 -_http._tcp.example.com. IN SRV 10 5 80 192.0.0.5 -example.com. IN TXT "abc" "def" -example.com. IN SPF "v=spf1 mx a" -example.com. IN NS ns1.example.com. -example.com. IN NS ns2.example.com. -delegation.example.com. IN NS ns1.example.com. -1.0.0.192.in-addr.arpa. IN PTR ipv4.example.com. +ipv6.example.com. 10800 IN AAAA fd00::1 +cname.example.com. 10800 IN CNAME example.com. +example.com. 10800 IN MX 5 192.0.0.2 +example.com. 10800 IN MX 10 192.0.0.3 +_http._tcp.example.com. 10800 IN SRV 10 0 80 192.0.0.4 +_http._tcp.example.com. 10800 IN SRV 10 5 80 192.0.0.5 +example.com. 10800 IN TXT "abc" "def" +example.com. 10800 IN SPF "v=spf1 mx a" +example.com. 10800 IN NS ns1.example.com. +example.com. 10800 IN NS ns2.example.com. +delegation.example.com. 10800 IN NS ns1.example.com. +1.0.0.192.in-addr.arpa. 10800 IN PTR ipv4.example.com. diff --git a/designate/tests/test_central/test_service.py b/designate/tests/test_central/test_service.py index 2b0e405c..c4f03762 100644 --- a/designate/tests/test_central/test_service.py +++ b/designate/tests/test_central/test_service.py @@ -1823,7 +1823,7 @@ class CentralServiceTest(CentralTestCase): def test_update_recordset_immutable_type(self): zone = self.create_zone() # ['A', 'AAAA', 'CNAME', 'MX', 'SRV', 'TXT', 'SPF', 'NS', 'PTR', - # 'SSHFP', 'SOA'] + # 'SSHFP', 'SOA', 'NAPTR', 'CAA'] # Create a recordset recordset = self.create_recordset(zone) cname_recordset = self.create_recordset(zone, type='CNAME') diff --git a/designate/tests/test_dnsutils.py b/designate/tests/test_dnsutils.py index d6205b69..21013787 100644 --- a/designate/tests/test_dnsutils.py +++ b/designate/tests/test_dnsutils.py @@ -24,9 +24,11 @@ from designate.tests import TestCase SAMPLES = { ("cname.example.com.", "CNAME"): { + "ttl": 10800, "records": ["example.com."], }, ("_http._tcp.example.com.", "SRV"): { + "ttl": 10800, "records": [ "10 0 80 192.0.0.4.example.com.", "10 5 80 192.0.0.5.example.com." @@ -37,9 +39,11 @@ SAMPLES = { "records": ["192.0.0.1"] }, ("delegation.example.com.", "NS"): { + "ttl": 10800, "records": ["ns1.example.com."] }, ("ipv6.example.com.", "AAAA"): { + "ttl": 10800, "records": ["fd00::1"], }, ("example.com.", "SOA"): { @@ -50,18 +54,22 @@ SAMPLES = { "ttl": 600 }, ("example.com.", "MX"): { + "ttl": 10800, "records": [ "5 192.0.0.2.example.com.", '10 192.0.0.3.example.com.' ] }, ("example.com.", "TXT"): { + "ttl": 10800, "records": ['"abc" "def"'] }, ("example.com.", "SPF"): { + "ttl": 10800, "records": ['"v=spf1 mx a"'] }, ("example.com.", "NS"): { + "ttl": 10800, "records": [ 'ns1.example.com.', 'ns2.example.com.' diff --git a/designate/tests/test_objects/test_caa_object.py b/designate/tests/test_objects/test_caa_object.py new file mode 100644 index 00000000..4fa20573 --- /dev/null +++ b/designate/tests/test_objects/test_caa_object.py @@ -0,0 +1,55 @@ +# Copyright 2018 Canonical Ltd. +# +# Author: Tytus Kurek <tytus.kurek@canonical.com> +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging +import oslotest.base + +from designate import objects + +LOG = logging.getLogger(__name__) + + +def debug(*a, **kw): + for v in a: + LOG.debug(repr(v)) + + for k in sorted(kw): + LOG.debug("%s: %s", k, repr(kw[k])) + + +class CAARecordTest(oslotest.base.BaseTestCase): + + def test_parse_caa_issue(self): + caa_record = objects.CAA() + caa_record._from_string('0 issue ca.example.net') + + self.assertEqual(0, caa_record.flags) + self.assertEqual('issue ca.example.net', caa_record.prpt) + + def test_parse_caa_issuewild(self): + caa_record = objects.CAA() + caa_record._from_string('1 issuewild ca.example.net; policy=ev') + + self.assertEqual(1, caa_record.flags) + self.assertEqual('issuewild ca.example.net; policy=ev', + caa_record.prpt) + + def test_parse_caa_iodef(self): + caa_record = objects.CAA() + caa_record._from_string('0 iodef https://example.net/') + + self.assertEqual(0, caa_record.flags) + self.assertEqual('iodef https://example.net/', caa_record.prpt) diff --git a/designate/tests/test_objects/test_naptr_object.py b/designate/tests/test_objects/test_naptr_object.py new file mode 100644 index 00000000..6266f93a --- /dev/null +++ b/designate/tests/test_objects/test_naptr_object.py @@ -0,0 +1,46 @@ +# Copyright 2018 Canonical Ltd. +# +# Author: Tytus Kurek <tytus.kurek@canonical.com> +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging +import oslotest.base + +from designate import objects + +LOG = logging.getLogger(__name__) + + +def debug(*a, **kw): + for v in a: + LOG.debug(repr(v)) + + for k in sorted(kw): + LOG.debug("%s: %s", k, repr(kw[k])) + + +class NAPTRRecordTest(oslotest.base.BaseTestCase): + + def test_parse_naptr(self): + naptr_record = objects.NAPTR() + naptr_record._from_string( + '0 0 S SIP+D2U !^.*$!sip:customer-service@example.com! _sip._udp.example.com.') # noqa + + self.assertEqual(0, naptr_record.order) + self.assertEqual(0, naptr_record.preference) + self.assertEqual('S', naptr_record.flags) + self.assertEqual('SIP+D2U', naptr_record.service) + self.assertEqual('!^.*$!sip:customer-service@example.com!', + naptr_record.regexp) + self.assertEqual('_sip._udp.example.com.', naptr_record.replacement) diff --git a/designate/tests/test_workers/test_service.py b/designate/tests/test_workers/test_service.py index f8d576ce..e41607a7 100644 --- a/designate/tests/test_workers/test_service.py +++ b/designate/tests/test_workers/test_service.py @@ -60,6 +60,8 @@ class TestService(TestCase): self.service._pool = mock.Mock() self.service.get_pool = mock.Mock() pool = mock.Mock() + pool.also_notifies = mock.MagicMock() + pool.also_notifies.__iter__.return_value = [] self.service.get_pool.return_value = pool self.service._do_zone_action(self.context, self.zone) @@ -72,7 +74,7 @@ class TestService(TestCase): self.zone.action ) - self.service._executor.run.assert_called_with(ZoneAction()) + self.service._executor.run.assert_called_with([ZoneAction()]) def test_get_pool(self): pool = mock.Mock() diff --git a/designate/worker/processing.py b/designate/worker/processing.py index cf43383a..a41564f1 100644 --- a/designate/worker/processing.py +++ b/designate/worker/processing.py @@ -15,7 +15,7 @@ # under the License. import time -from concurrent import futures +import futurist from oslo_log import log as logging from oslo_config import cfg @@ -32,7 +32,10 @@ def default_executor(): except Exception: pass - return futures.ThreadPoolExecutor(thread_count) + # TODO(mugsie): if (when) we move away from eventlet this may have to + # revert back to ThreadPoolExecutor - this is changing due to + # https://bugs.launchpad.net/bugs/1782647 (eventlet + py37 issues) + return futurist.GreenThreadPoolExecutor(thread_count) class Executor(object): diff --git a/designate/worker/service.py b/designate/worker/service.py index 624c3d56..213c5f36 100644 --- a/designate/worker/service.py +++ b/designate/worker/service.py @@ -33,6 +33,13 @@ LOG = logging.getLogger(__name__) CONF = cfg.CONF +class AlsoNotifyTask(object): + """ + Placeholder to define options for also_notify targets + """ + pass + + class Service(service.RPCService, service.Service): RPC_API_VERSION = '1.0' @@ -119,10 +126,20 @@ class Service(service.RPCService, service.Service): def _do_zone_action(self, context, zone): pool = self.get_pool(zone.pool_id) - task = zonetasks.ZoneAction( + all_tasks = [] + all_tasks.append(zonetasks.ZoneAction( self.executor, context, pool, zone, zone.action - ) - return self.executor.run(task) + )) + + # Send a NOTIFY to each also-notifies + for also_notify in pool.also_notifies: + notify_target = AlsoNotifyTask() + notify_target.options = {'host': also_notify.host, + 'port': also_notify.port} + all_tasks.append(zonetasks.SendNotify(self.executor, + zone, + notify_target)) + return self.executor.run(all_tasks) def create_zone(self, context, zone): """ diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 85ea2396..2264a937 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -11,17 +11,6 @@ if is_service_enabled designate && [[ -r $DESIGNATE_PLUGINS/backend-$DESIGNATE_B source $DESIGNATE_PLUGINS/backend-$DESIGNATE_BACKEND_DRIVER fi -# Helper Functions -# ---------------- -function setup_colorized_logging_designate { - local conf_file=$1 - local conf_section=$2 - local project_var=${3:-"project_name"} - local user_var=${4:-"user_name"} - - setup_colorized_logging $conf_file $conf_section $project_var $user_var -} - # DevStack Plugin # --------------- @@ -45,7 +34,7 @@ function configure_designate { # General Configuration iniset_rpc_backend designate $DESIGNATE_CONF DEFAULT - iniset $DESIGNATE_CONF DEFAULT rpc_response_timeout 5 + iniset $DESIGNATE_CONF DEFAULT rpc_response_timeout 60 iniset $DESIGNATE_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL iniset $DESIGNATE_CONF DEFAULT state_path $DESIGNATE_STATE_PATH @@ -139,7 +128,7 @@ function configure_designate { # Format logging if [ "$LOG_COLOR" == "True" ] && [ "$USE_SYSTEMD" == "False" ]; then - setup_colorized_logging_designate $DESIGNATE_CONF DEFAULT "tenant" "user" + setup_colorized_logging $DESIGNATE_CONF DEFAULT fi # Backend Plugin Configuation diff --git a/doc/ext/support_matrix.py b/doc/ext/support_matrix.py index b3e08076..e160689b 100644 --- a/doc/ext/support_matrix.py +++ b/doc/ext/support_matrix.py @@ -22,16 +22,19 @@ It is used via a single directive in the .rst file """ import os - -import six -import six.moves.configparser as config_parser import sys from docutils import nodes from docutils.parsers import rst +from sphinx.util import logging +from sphinx.util.osutil import copyfile +import six +import six.moves.configparser as config_parser + from designate.backend.base import Backend from designate.backend.agent_backend.base import AgentBackend -from sphinx.util.osutil import copyfile + +LOG = logging.getLogger(__name__) class SupportMatrix(object): @@ -434,7 +437,7 @@ def copy_assets(app, exception): assets = ['support-matrix.css', 'support-matrix.js'] if app.builder.name != 'html' or exception: return - app.info('Copying assets: %s' % ', '.join(assets)) + LOG.info('Copying assets: %s' % ', '.join(assets)) for asset in assets: dest = os.path.join(app.builder.outdir, '_static', asset) source = os.path.abspath(os.path.dirname(__file__)) diff --git a/doc/source/contributor/sourcedoc/objects.rst b/doc/source/contributor/sourcedoc/objects.rst index 884c748a..1934f463 100644 --- a/doc/source/contributor/sourcedoc/objects.rst +++ b/doc/source/contributor/sourcedoc/objects.rst @@ -180,3 +180,18 @@ Objects SSHFP Record :show-inheritance: +Objects NAPTR Record +==================== +.. automodule:: designate.objects.rrdata_naptr + :members: + :undoc-members: + :show-inheritance: + + +Objects CAA Record +==================== +.. automodule:: designate.objects.rrdata_caa + + :members: + :undoc-members: + :show-inheritance: diff --git a/doc/source/install/install-obs.rst b/doc/source/install/install-obs.rst index f45947c6..81324990 100644 --- a/doc/source/install/install-obs.rst +++ b/doc/source/install/install-obs.rst @@ -30,7 +30,7 @@ Install and configure components .. code-block:: console - # mysql -u root -p + # mysql MariaDB [(none)]> CREATE DATABASE designate CHARACTER SET utf8 COLLATE utf8_general_ci; MariaDB [(none)]> GRANT ALL PRIVILEGES ON designate.* TO 'designate'@'localhost' \ IDENTIFIED BY 'DESIGNATE_DBPASS'; diff --git a/doc/source/install/install-rdo.rst b/doc/source/install/install-rdo.rst index 1a60c440..99d9c0bf 100644 --- a/doc/source/install/install-rdo.rst +++ b/doc/source/install/install-rdo.rst @@ -30,7 +30,7 @@ Install and configure components .. code-block:: console - # mysql -u root -p + # mysql MariaDB [(none)]> CREATE DATABASE designate CHARACTER SET utf8 COLLATE utf8_general_ci; MariaDB [(none)]> GRANT ALL PRIVILEGES ON designate.* TO 'designate'@'localhost' \ IDENTIFIED BY 'DESIGNATE_DBPASS'; diff --git a/doc/source/install/install-ubuntu.rst b/doc/source/install/install-ubuntu.rst index 6d2ac5d4..6596e136 100644 --- a/doc/source/install/install-ubuntu.rst +++ b/doc/source/install/install-ubuntu.rst @@ -30,7 +30,7 @@ Install and configure components .. code-block:: console - # mysql -u root -p + # mysql mysql> CREATE DATABASE designate CHARACTER SET utf8 COLLATE utf8_general_ci; mysql> GRANT ALL PRIVILEGES ON designate.* TO 'designate'@'localhost' \ IDENTIFIED BY 'DESIGNATE_DBPASS'; diff --git a/lower-constraints.txt b/lower-constraints.txt index e91e2bdc..4cacf0ef 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -36,7 +36,7 @@ flake8==2.5.5 Flask==0.10 funcparserlib==0.3.6 future==0.16.0 -futurist==1.6.0 +futurist==1.2.0 gitdb2==2.0.3 GitPython==2.1.8 greenlet==0.4.10 diff --git a/releasenotes/notes/worker-executor-84d983c92dd13b49.yaml b/releasenotes/notes/worker-executor-84d983c92dd13b49.yaml new file mode 100644 index 00000000..288cb18d --- /dev/null +++ b/releasenotes/notes/worker-executor-84d983c92dd13b49.yaml @@ -0,0 +1,8 @@ +--- +other: + - | + To allow for python3.7 support, the `designate-worker` service was changed + from a `ThreadPoolExecutor` to a `GreenThreadPoolExecutor`. This should + no impact for most deployments, but in some cases it may cause performance + degredation. In these cases, tuning `[service:worker].workers` and + `[service:worker].threads` alleviate the issues. diff --git a/releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po b/releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po index 6f7faccb..bff6958a 100644 --- a/releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po +++ b/releasenotes/source/locale/en_GB/LC_MESSAGES/releasenotes.po @@ -4,11 +4,11 @@ msgid "" msgstr "" "Project-Id-Version: designate\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2018-09-06 02:51+0000\n" +"POT-Creation-Date: 2018-12-04 15:32+0000\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" -"PO-Revision-Date: 2018-09-07 11:23+0000\n" +"PO-Revision-Date: 2018-12-04 05:21+0000\n" "Last-Translator: Andi Chandler <andi@gowling.com>\n" "Language-Team: English (United Kingdom)\n" "Language: en_GB\n" @@ -42,8 +42,8 @@ msgstr "6.0.0" msgid "7.0.0" msgstr "7.0.0" -msgid "7.0.0-14" -msgstr "7.0.0-14" +msgid "7.0.0-40" +msgstr "7.0.0-40" msgid "" "A new recordset api ``/v2/recordsets`` is exposed with GET method allowed " @@ -451,6 +451,19 @@ msgstr "" "DNS Servers, and added scheduling across pools." msgid "" +"To allow for python3.7 support, the `designate-worker` service was changed " +"from a `ThreadPoolExecutor` to a `GreenThreadPoolExecutor`. This should no " +"impact for most deployments, but in some cases it may cause performance " +"degredation. In these cases, tuning `[service:worker].workers` and `[service:" +"worker].threads` alleviate the issues." +msgstr "" +"To allow for python3.7 support, the `designate-worker` service was changed " +"from a `ThreadPoolExecutor` to a `GreenThreadPoolExecutor`. This should no " +"impact for most deployments, but in some cases it may cause performance " +"degredation. In these cases, tuning `[service:worker].workers` and `[service:" +"worker].threads` alleviate the issues." + +msgid "" "To enable ``designate-worker`` and ``designate-producer`` add a section to " "your ``designate.conf`` called ``[service:worker]`` and add an option " "``enabled = True``. Then stop ``designate-pool-manager`` and ``designate-" diff --git a/requirements.txt b/requirements.txt index b2446fbf..0afcc00f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -49,3 +49,4 @@ tooz>=1.58.0 # Apache-2.0 debtcollector>=1.2.0 # Apache-2.0 os-win>=3.0.0 # Apache-2.0 monasca-statsd>=1.1.0 # Apache-2.0 +futurist>=1.2.0 # Apache-2.0 @@ -4,7 +4,7 @@ summary = DNS as a Service description-file = README.rst author = OpenStack -author-email = openstack-dev@lists.openstack.org +author-email = openstack-discuss@lists.openstack.org home-page = https://docs.openstack.org/designate/latest/ classifier = Environment :: OpenStack @@ -1,6 +1,6 @@ [tox] minversion = 2.0 -envlist = py35,py27,py36,flake8 +envlist = py35,py27,py36,py37,flake8 skipsdist = True [testenv] @@ -30,19 +30,25 @@ passenv = http_proxy [testenv:py27] commands = {[testenv]commands} - stestr run '{posargs}' + stestr run {posargs} stestr slowest [testenv:py35] basepython = python3 commands = {[testenv]commands} - stestr run '{posargs}' + stestr run {posargs} [testenv:py36] basepython = python3.6 commands = {[testenv]commands} + stestr run {posargs} + +[testenv:py37] +basepython = python3.7 +commands = + {[testenv]commands} stestr run '{posargs}' [testenv:docs] |