diff options
| -rw-r--r-- | playbooks/zuul-stream/fixtures/test-stream.yaml | 15 | ||||
| -rw-r--r-- | playbooks/zuul-stream/validate.yaml | 5 | ||||
| -rw-r--r-- | releasenotes/notes/fix-prune-database-a4873bd4dead7b5f.yaml | 16 | ||||
| -rw-r--r-- | releasenotes/notes/submit-requirements-1d61f88e54be1fde.yaml | 16 | ||||
| -rw-r--r-- | tests/base.py | 8 | ||||
| -rw-r--r-- | tests/unit/test_client.py | 136 | ||||
| -rw-r--r-- | tests/unit/test_gerrit.py | 86 | ||||
| -rw-r--r-- | web/src/pages/Labels.jsx | 57 | ||||
| -rw-r--r-- | web/src/pages/Nodes.jsx | 143 | ||||
| -rw-r--r-- | web/src/pages/Projects.jsx | 93 | ||||
| -rw-r--r-- | zuul/ansible/base/callback/zuul_stream.py | 5 | ||||
| -rwxr-xr-x | zuul/cmd/client.py | 6 | ||||
| -rw-r--r-- | zuul/driver/gerrit/gerritconnection.py | 40 | ||||
| -rw-r--r-- | zuul/driver/gerrit/gerritmodel.py | 4 | ||||
| -rw-r--r-- | zuul/driver/github/githubconnection.py | 5 | ||||
| -rw-r--r-- | zuul/driver/sql/sqlconnection.py | 39 |
16 files changed, 521 insertions, 153 deletions
diff --git a/playbooks/zuul-stream/fixtures/test-stream.yaml b/playbooks/zuul-stream/fixtures/test-stream.yaml index 488f8cb2f..49ceb092b 100644 --- a/playbooks/zuul-stream/fixtures/test-stream.yaml +++ b/playbooks/zuul-stream/fixtures/test-stream.yaml @@ -1,3 +1,16 @@ +# NOTE: We run this before starting the log streaming to validate that +# if we set zuul_console_disabled, we don't try to connect at all. If +# there is a log streamer running when we run this test, then we have +# no indication that we avoid the connection step. +- name: Run command to show skipping works without zuul_console running + vars: + zuul_console_disabled: true + hosts: node + tasks: + - name: Run quiet command + command: echo 'This command should not stream' + when: new_console | default(false) + - name: Start zuul stream daemon hosts: node tasks: @@ -11,7 +24,7 @@ port: 19887 when: new_console | default(false) -- name: Run command to show skipping works +- name: Run command to show skipping works with zuul_console running vars: zuul_console_disabled: true hosts: node diff --git a/playbooks/zuul-stream/validate.yaml b/playbooks/zuul-stream/validate.yaml index 81c613406..c7069f335 100644 --- a/playbooks/zuul-stream/validate.yaml +++ b/playbooks/zuul-stream/validate.yaml @@ -27,3 +27,8 @@ - name: Validate output - binary data shell: | egrep "^.*\| {{ item.node }} \| \\\\x80abc" {{ item.filename }} + +- name: Validate output - no waiting on logger + shell: | + egrep -v "Waiting on logger" {{ item.filename }} + egrep -v "Log Stream did not terminate" {{ item.filename }} diff --git a/releasenotes/notes/fix-prune-database-a4873bd4dead7b5f.yaml b/releasenotes/notes/fix-prune-database-a4873bd4dead7b5f.yaml new file mode 100644 index 000000000..6e036a754 --- /dev/null +++ b/releasenotes/notes/fix-prune-database-a4873bd4dead7b5f.yaml @@ -0,0 +1,16 @@ +--- +fixes: + - | + The `zuul-admin prune-database` command did not completely delete + expected data from the database. It may not have deleted all of + the buildsets older than the specified cutoff time, and it may + have left orphaned data in ancillary tables. This has been + corrected and it should now work as expected. Additionally, a + `--batch-size` argument has been added so that it may delete data + in multiple transactions which can facilitate smoother operation + when run while Zuul is operational. + + Users who have previously run the command may need to manually + delete rows from the `zuul_build`, `zuul_build_event`, + `zuul_artifact`, and `zuul_provides` tables which do not have + corresponding entries in the `zuul_buildset` table. diff --git a/releasenotes/notes/submit-requirements-1d61f88e54be1fde.yaml b/releasenotes/notes/submit-requirements-1d61f88e54be1fde.yaml new file mode 100644 index 000000000..14f037156 --- /dev/null +++ b/releasenotes/notes/submit-requirements-1d61f88e54be1fde.yaml @@ -0,0 +1,16 @@ +--- +fixes: + - | + Zuul will now attempt to honor Gerrit "submit requirements" when + determining whether to enqueue a change into a dependent (i.e., + "gate") pipeline. Zuul previously honored only Gerrit's older + "submit records" feature. The new checks will avoid enqueing + changes in "gate" pipelines in the cases where Zuul can + unambiguously determine that there is no possibility of merging, + but some non-mergable changes may still be enqueued if Zuul can + not be certain whether a rule should apply or be disregarded (in + these cases, Gerrit will fail to merge the change and Zuul will + report the buildset as a MERGE_FAILURE). + + This requires Gerrit version 3.5.0 or later, and Zuul to be + configured with HTTP access for Gerrit. diff --git a/tests/base.py b/tests/base.py index 290d51934..1cfcecde5 100644 --- a/tests/base.py +++ b/tests/base.py @@ -403,6 +403,7 @@ class FakeGerritChange(object): self.comments = [] self.checks = {} self.checks_history = [] + self.submit_requirements = [] self.data = { 'branch': branch, 'comments': self.comments, @@ -788,6 +789,12 @@ class FakeGerritChange(object): return [{'status': 'NOT_READY', 'labels': labels}] + def getSubmitRequirements(self): + return self.submit_requirements + + def setSubmitRequirements(self, reqs): + self.submit_requirements = reqs + def setDependsOn(self, other, patchset): self.depends_on_change = other self.depends_on_patchset = patchset @@ -894,6 +901,7 @@ class FakeGerritChange(object): data['parents'] = self.data['parents'] if 'topic' in self.data: data['topic'] = self.data['topic'] + data['submit_requirements'] = self.getSubmitRequirements() return json.loads(json.dumps(data)) def queryRevisionHTTP(self, revision): diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index f241147eb..2e90d3fb4 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -21,10 +21,12 @@ import time import configparser import datetime import dateutil.tz +import uuid import fixtures import jwt import testtools +import sqlalchemy from zuul.zk import ZooKeeperClient from zuul.zk.locks import SessionAwareLock @@ -499,27 +501,107 @@ class TestDBPruneParse(BaseTestCase): class DBPruneTestCase(ZuulTestCase): tenant_config_file = 'config/single-tenant/main.yaml' + # This should be larger than the limit size in sqlconnection + num_buildsets = 55 + + def _createBuildset(self, update_time): + connection = self.scheds.first.sched.sql.connection + buildset_uuid = uuid.uuid4().hex + event_id = uuid.uuid4().hex + with connection.getSession() as db: + start_time = update_time - datetime.timedelta(seconds=1) + end_time = update_time + db_buildset = db.createBuildSet( + uuid=buildset_uuid, + tenant='tenant-one', + pipeline='check', + project='org/project', + change='1', + patchset='1', + ref='refs/changes/1', + oldrev='', + newrev='', + branch='master', + zuul_ref='Zref', + ref_url='http://gerrit.example.com/1', + event_id=event_id, + event_timestamp=update_time, + updated=update_time, + first_build_start_time=start_time, + last_build_end_time=end_time, + result='SUCCESS', + ) + for build_num in range(2): + build_uuid = uuid.uuid4().hex + db_build = db_buildset.createBuild( + uuid=build_uuid, + job_name=f'job{build_num}', + start_time=start_time, + end_time=end_time, + result='SUCCESS', + voting=True, + ) + for art_num in range(2): + db_build.createArtifact( + name=f'artifact{art_num}', + url='http://example.com', + ) + for provides_num in range(2): + db_build.createProvides( + name=f'item{provides_num}', + ) + for event_num in range(2): + db_build.createBuildEvent( + event_type=f'event{event_num}', + event_time=start_time, + ) + + def _query(self, db, model): + table = model.__table__ + q = db.session().query(model).order_by(table.c.id.desc()) + try: + return q.all() + except sqlalchemy.orm.exc.NoResultFound: + return [] + + def _getBuildsets(self, db): + return self._query(db, db.connection.buildSetModel) + + def _getBuilds(self, db): + return self._query(db, db.connection.buildModel) + + def _getProvides(self, db): + return self._query(db, db.connection.providesModel) + + def _getArtifacts(self, db): + return self._query(db, db.connection.artifactModel) + + def _getBuildEvents(self, db): + return self._query(db, db.connection.buildEventModel) def _setup(self): config_file = os.path.join(self.test_root, 'zuul.conf') with open(config_file, 'w') as f: self.config.write(f) - A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') - self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1)) - self.waitUntilSettled() - - time.sleep(1) - - B = self.fake_gerrit.addFakeChange('org/project', 'master', 'B') - self.fake_gerrit.addEvent(B.getPatchsetCreatedEvent(1)) - self.waitUntilSettled() + update_time = (datetime.datetime.utcnow() - + datetime.timedelta(minutes=self.num_buildsets)) + for x in range(self.num_buildsets): + update_time = update_time + datetime.timedelta(minutes=1) + self._createBuildset(update_time) connection = self.scheds.first.sched.sql.connection - buildsets = connection.getBuildsets() - builds = connection.getBuilds() - self.assertEqual(len(buildsets), 2) - self.assertEqual(len(builds), 6) + with connection.getSession() as db: + buildsets = self._getBuildsets(db) + builds = self._getBuilds(db) + artifacts = self._getArtifacts(db) + provides = self._getProvides(db) + events = self._getBuildEvents(db) + self.assertEqual(len(buildsets), self.num_buildsets) + self.assertEqual(len(builds), 2 * self.num_buildsets) + self.assertEqual(len(artifacts), 4 * self.num_buildsets) + self.assertEqual(len(provides), 4 * self.num_buildsets) + self.assertEqual(len(events), 4 * self.num_buildsets) for build in builds: self.log.debug("Build %s %s %s", build, build.start_time, build.end_time) @@ -535,6 +617,7 @@ class DBPruneTestCase(ZuulTestCase): start_time = buildsets[0].first_build_start_time self.log.debug("Cutoff %s", start_time) + # Use the default batch size (omit --batch-size arg) p = subprocess.Popen( [os.path.join(sys.prefix, 'bin/zuul-admin'), '-c', config_file, @@ -545,13 +628,20 @@ class DBPruneTestCase(ZuulTestCase): out, _ = p.communicate() self.log.debug(out.decode('utf8')) - buildsets = connection.getBuildsets() - builds = connection.getBuilds() - self.assertEqual(len(buildsets), 1) - self.assertEqual(len(builds), 3) + with connection.getSession() as db: + buildsets = self._getBuildsets(db) + builds = self._getBuilds(db) + artifacts = self._getArtifacts(db) + provides = self._getProvides(db) + events = self._getBuildEvents(db) for build in builds: self.log.debug("Build %s %s %s", build, build.start_time, build.end_time) + self.assertEqual(len(buildsets), 1) + self.assertEqual(len(builds), 2) + self.assertEqual(len(artifacts), 4) + self.assertEqual(len(provides), 4) + self.assertEqual(len(events), 4) def test_db_prune_older_than(self): # Test pruning buildsets older than a relative time @@ -567,15 +657,23 @@ class DBPruneTestCase(ZuulTestCase): '-c', config_file, 'prune-database', '--older-than', '0d', + '--batch-size', '5', ], stdout=subprocess.PIPE) out, _ = p.communicate() self.log.debug(out.decode('utf8')) - buildsets = connection.getBuildsets() - builds = connection.getBuilds() + with connection.getSession() as db: + buildsets = self._getBuildsets(db) + builds = self._getBuilds(db) + artifacts = self._getArtifacts(db) + provides = self._getProvides(db) + events = self._getBuildEvents(db) self.assertEqual(len(buildsets), 0) self.assertEqual(len(builds), 0) + self.assertEqual(len(artifacts), 0) + self.assertEqual(len(provides), 0) + self.assertEqual(len(events), 0) class TestDBPruneMysql(DBPruneTestCase): diff --git a/tests/unit/test_gerrit.py b/tests/unit/test_gerrit.py index 2a63d5ef8..ac3dccf3b 100644 --- a/tests/unit/test_gerrit.py +++ b/tests/unit/test_gerrit.py @@ -958,6 +958,92 @@ class TestGerritConnection(ZuulTestCase): self.assertEqual(A.data['status'], 'MERGED') self.assertEqual(B.data['status'], 'MERGED') + def test_submit_requirements(self): + A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') + A.addApproval('Code-Review', 2) + # Set an unsatisfied submit requirement + A.setSubmitRequirements([ + { + "name": "Code-Review", + "description": "Disallow self-review", + "status": "UNSATISFIED", + "is_legacy": False, + "submittability_expression_result": { + "expression": "label:Code-Review=MAX,user=non_uploader " + "AND -label:Code-Review=MIN", + "fulfilled": False, + "passing_atoms": [], + "failing_atoms": [ + "label:Code-Review=MAX,user=non_uploader", + "label:Code-Review=MIN" + ] + } + }, + { + "name": "Verified", + "status": "UNSATISFIED", + "is_legacy": True, + "submittability_expression_result": { + "expression": "label:Verified=MAX -label:Verified=MIN", + "fulfilled": False, + "passing_atoms": [], + "failing_atoms": [ + "label:Verified=MAX", + "-label:Verified=MIN" + ] + } + }, + ]) + self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) + self.waitUntilSettled() + self.assertHistory([]) + self.assertEqual(A.queried, 1) + self.assertEqual(A.data['status'], 'NEW') + + # Mark the requirement satisfied + A.setSubmitRequirements([ + { + "name": "Code-Review", + "description": "Disallow self-review", + "status": "SATISFIED", + "is_legacy": False, + "submittability_expression_result": { + "expression": "label:Code-Review=MAX,user=non_uploader " + "AND -label:Code-Review=MIN", + "fulfilled": False, + "passing_atoms": [ + "label:Code-Review=MAX,user=non_uploader", + ], + "failing_atoms": [ + "label:Code-Review=MIN" + ] + } + }, + { + "name": "Verified", + "status": "UNSATISFIED", + "is_legacy": True, + "submittability_expression_result": { + "expression": "label:Verified=MAX -label:Verified=MIN", + "fulfilled": False, + "passing_atoms": [], + "failing_atoms": [ + "label:Verified=MAX", + "-label:Verified=MIN" + ] + } + }, + ]) + self.fake_gerrit.addEvent(A.addApproval('Approved', 1)) + self.waitUntilSettled() + self.assertHistory([ + dict(name="project-merge", result="SUCCESS", changes="1,1"), + dict(name="project-test1", result="SUCCESS", changes="1,1"), + dict(name="project-test2", result="SUCCESS", changes="1,1"), + ], ordered=False) + self.assertEqual(A.queried, 3) + self.assertEqual(A.data['status'], 'MERGED') + class TestGerritUnicodeRefs(ZuulTestCase): config_file = 'zuul-gerrit-web.conf' diff --git a/web/src/pages/Labels.jsx b/web/src/pages/Labels.jsx index 10decaa73..1c6e50db5 100644 --- a/web/src/pages/Labels.jsx +++ b/web/src/pages/Labels.jsx @@ -15,8 +15,15 @@ import * as React from 'react' import PropTypes from 'prop-types' import { connect } from 'react-redux' -import { Table } from 'patternfly-react' import { PageSection, PageSectionVariants } from '@patternfly/react-core' +import { + Table, + TableVariant, + TableHeader, + TableBody, +} from '@patternfly/react-table' +import { TagIcon } from '@patternfly/react-icons' +import { IconProperty } from '../Misc' import { fetchLabelsIfNeeded } from '../actions/labels' import { Fetchable, Fetching } from '../containers/Fetching' @@ -54,18 +61,22 @@ class LabelsPage extends React.Component { return <Fetching /> } - const headerFormat = value => <Table.Heading>{value}</Table.Heading> - const cellFormat = value => <Table.Cell>{value}</Table.Cell> - const columns = [] - const myColumns = ['name'] - myColumns.forEach(column => { - let formatter = cellFormat - let prop = column - columns.push({ - header: {label: column, formatters: [headerFormat]}, - property: prop, - cell: {formatters: [formatter]} - }) + const columns = [ + { + title: ( + <IconProperty icon={<TagIcon />} value="Name" /> + ), + dataLabel: 'name' + } + ] + let rows = [] + labels.forEach((label) => { + let r = { + cells: [ + {title: label.name, props: {column: 'Name'}} + ], + } + rows.push(r) }) return ( <PageSection variant={PageSectionVariants.light}> @@ -75,18 +86,16 @@ class LabelsPage extends React.Component { fetchCallback={this.updateData} /> </PageSection> - <Table.PfProvider - striped - bordered - hover - columns={columns} + <Table + aria-label="Labels Table" + variant={TableVariant.compact} + cells={columns} + rows={rows} + className="zuul-table" > - <Table.Header/> - <Table.Body - rows={labels} - rowKey="name" - /> - </Table.PfProvider> + <TableHeader /> + <TableBody /> + </Table> </PageSection> ) } diff --git a/web/src/pages/Nodes.jsx b/web/src/pages/Nodes.jsx index a7081ac59..b13878087 100644 --- a/web/src/pages/Nodes.jsx +++ b/web/src/pages/Nodes.jsx @@ -15,9 +15,29 @@ import * as React from 'react' import PropTypes from 'prop-types' import { connect } from 'react-redux' -import { Table } from 'patternfly-react' +import { + Table, + TableVariant, + TableHeader, + TableBody, +} from '@patternfly/react-table' import * as moment from 'moment' -import { PageSection, PageSectionVariants } from '@patternfly/react-core' +import { + PageSection, + PageSectionVariants, + ClipboardCopy, +} from '@patternfly/react-core' +import { + BuildIcon, + ClusterIcon, + ConnectedIcon, + OutlinedCalendarAltIcon, + TagIcon, + RunningIcon, + PencilAltIcon, + ZoneIcon, +} from '@patternfly/react-icons' +import { IconProperty } from '../Misc' import { fetchNodesIfNeeded } from '../actions/nodes' import { Fetchable } from '../containers/Fetching' @@ -51,43 +71,69 @@ class NodesPage extends React.Component { const { remoteData } = this.props const nodes = remoteData.nodes - const headerFormat = value => <Table.Heading>{value}</Table.Heading> - const cellFormat = value => <Table.Cell>{value}</Table.Cell> - const cellLabelsFormat = value => <Table.Cell>{value.join(',')}</Table.Cell> - const cellPreFormat = value => ( - <Table.Cell style={{fontFamily: 'Menlo,Monaco,Consolas,monospace'}}> - {value} - </Table.Cell>) - const cellAgeFormat = value => ( - <Table.Cell style={{fontFamily: 'Menlo,Monaco,Consolas,monospace'}}> - {moment.unix(value).fromNow()} - </Table.Cell>) - - const columns = [] - const myColumns = [ - 'id', 'labels', 'connection', 'server', 'provider', 'state', - 'age', 'comment' - ] - myColumns.forEach(column => { - let formatter = cellFormat - let prop = column - if (column === 'labels') { - prop = 'type' - formatter = cellLabelsFormat - } else if (column === 'connection') { - prop = 'connection_type' - } else if (column === 'server') { - prop = 'external_id' - formatter = cellPreFormat - } else if (column === 'age') { - prop = 'state_time' - formatter = cellAgeFormat + const columns = [ + { + title: ( + <IconProperty icon={<BuildIcon />} value="ID" /> + ), + dataLabel: 'id', + }, + { + title: ( + <IconProperty icon={<TagIcon />} value="Labels" /> + ), + dataLabel: 'labels', + }, + { + title: ( + <IconProperty icon={<ConnectedIcon />} value="Connection" /> + ), + dataLabel: 'connection', + }, + { + title: ( + <IconProperty icon={<ClusterIcon />} value="Server" /> + ), + dataLabel: 'server', + }, + { + title: ( + <IconProperty icon={<ZoneIcon />} value="Provider" /> + ), + dataLabel: 'provider', + }, + { + title: ( + <IconProperty icon={<RunningIcon />} value="State" /> + ), + dataLabel: 'state', + }, + { + title: ( + <IconProperty icon={<OutlinedCalendarAltIcon />} value="Age" /> + ), + dataLabel: 'age', + }, + { + title: ( + <IconProperty icon={<PencilAltIcon />} value="Comment" /> + ), + dataLabel: 'comment', } - columns.push({ - header: {label: column, formatters: [headerFormat]}, - property: prop, - cell: {formatters: [formatter]} - }) + ] + let rows = [] + nodes.forEach((node) => { + let r = [ + {title: node.id, props: {column: 'ID'}}, + {title: node.type.join(','), props: {column: 'Label' }}, + {title: node.connection_type, props: {column: 'Connection'}}, + {title: <ClipboardCopy hoverTip="Copy" clickTip="Copied" variant="inline-compact">{node.external_id}</ClipboardCopy>, props: {column: 'Server'}}, + {title: node.provider, props: {column: 'Provider'}}, + {title: node.state, props: {column: 'State'}}, + {title: moment.unix(node.state_time).fromNow(), props: {column: 'Age'}}, + {title: node.comment, props: {column: 'Comment'}}, + ] + rows.push({cells: r}) }) return ( <PageSection variant={PageSectionVariants.light}> @@ -97,18 +143,17 @@ class NodesPage extends React.Component { fetchCallback={this.updateData} /> </PageSection> - <Table.PfProvider - striped - bordered - hover - columns={columns} + + <Table + aria-label="Nodes Table" + variant={TableVariant.compact} + cells={columns} + rows={rows} + className="zuul-table" > - <Table.Header/> - <Table.Body - rows={nodes} - rowKey="id" - /> - </Table.PfProvider> + <TableHeader /> + <TableBody /> + </Table> </PageSection> ) } diff --git a/web/src/pages/Projects.jsx b/web/src/pages/Projects.jsx index 13ee81bd8..598133384 100644 --- a/web/src/pages/Projects.jsx +++ b/web/src/pages/Projects.jsx @@ -16,8 +16,18 @@ import * as React from 'react' import PropTypes from 'prop-types' import { connect } from 'react-redux' import { Link } from 'react-router-dom' -import { Table } from 'patternfly-react' import { PageSection, PageSectionVariants } from '@patternfly/react-core' +import { + Table, + TableVariant, + TableHeader, + TableBody, +} from '@patternfly/react-table' +import { + CubeIcon, + ConnectedIcon, +} from '@patternfly/react-icons' +import { IconProperty } from '../Misc' import { fetchProjectsIfNeeded } from '../actions/projects' import { Fetchable, Fetching } from '../containers/Fetching' @@ -59,42 +69,35 @@ class ProjectsPage extends React.Component { return <Fetching /> } - const headerFormat = value => <Table.Heading>{value}</Table.Heading> - const cellFormat = (value) => ( - <Table.Cell>{value}</Table.Cell>) - const cellProjectFormat = (value, row) => ( - <Table.Cell> - <Link to={this.props.tenant.linkPrefix + '/project/' + row.rowData.canonical_name}> - {value} - </Link> - </Table.Cell>) - const cellBuildFormat = (value) => ( - <Table.Cell> - <Link to={this.props.tenant.linkPrefix + '/builds?project=' + value}> - builds - </Link> - </Table.Cell>) - const columns = [] - const myColumns = ['name', 'connection', 'type', 'last builds'] - myColumns.forEach(column => { - let formatter = cellFormat - let prop = column - if (column === 'name') { - formatter = cellProjectFormat + const columns = [ + { + title: <IconProperty icon={<CubeIcon />} value="Name" />, + dataLabel: 'name', + }, + { + title: <IconProperty icon={<ConnectedIcon />} value="Connection" />, + dataLabel: 'connection', + }, + { + title: 'Type', + dataLabel: 'type', + }, + { + title: 'Last builds', + dataLabel: 'last-builds', } - if (column === 'connection') { - prop = 'connection_name' + ] + let rows = [] + projects.forEach((project) => { + let r = { + cells: [ + {title: <Link to={this.props.tenant.linkPrefix + '/project/' + project.canonical_name}>{project.name}</Link>, props: {column: 'Name'}}, + {title: project.connection_name, props: {column: 'Connection'}}, + {title: project.type, props: {column: 'Type'}}, + {title: <Link to={this.props.tenant.linkPrefix + '/builds?project=' + project.name}>Builds</Link>, props: {column: 'Last builds'}}, + ] } - if (column === 'last builds') { - prop = 'name' - formatter = cellBuildFormat - } - columns.push({ - header: {label: column, - formatters: [headerFormat]}, - property: prop, - cell: {formatters: [formatter]} - }) + rows.push(r) }) return ( <PageSection variant={PageSectionVariants.light}> @@ -104,18 +107,16 @@ class ProjectsPage extends React.Component { fetchCallback={this.updateData} /> </PageSection> - <Table.PfProvider - striped - bordered - hover - columns={columns} + <Table + aria-label="Projects Table" + variant={TableVariant.compact} + cells={columns} + rows={rows} + className="zuul-table" > - <Table.Header/> - <Table.Body - rows={projects} - rowKey="name" - /> - </Table.PfProvider> + <TableHeader /> + <TableBody /> + </Table> </PageSection> ) } diff --git a/zuul/ansible/base/callback/zuul_stream.py b/zuul/ansible/base/callback/zuul_stream.py index b5c14691b..3f886c797 100644 --- a/zuul/ansible/base/callback/zuul_stream.py +++ b/zuul/ansible/base/callback/zuul_stream.py @@ -44,6 +44,7 @@ import time from ansible.plugins.callback import default from ansible.module_utils._text import to_text +from ansible.module_utils.parsing.convert_bool import boolean from zuul.ansible import paths from zuul.ansible import logconfig @@ -333,6 +334,10 @@ class CallbackModule(default.CallbackModule): if (ip in ('localhost', '127.0.0.1')): # Don't try to stream from localhost continue + if boolean(play_vars[host].get( + 'zuul_console_disabled', False)): + # The user has told us not to even try + continue if play_vars[host].get('ansible_connection') in ('winrm',): # The winrm connections don't support streaming for now continue diff --git a/zuul/cmd/client.py b/zuul/cmd/client.py index 62e51ac3f..6fa20c7c4 100755 --- a/zuul/cmd/client.py +++ b/zuul/cmd/client.py @@ -540,6 +540,10 @@ class Client(zuul.cmd.ZuulApp): cmd_prune_database.add_argument( '--older-than', help='relative time (e.g., "24h" or "180d")') + cmd_prune_database.add_argument( + '--batch-size', + default=10000, + help='transaction batch size') cmd_prune_database.set_defaults(func=self.prune_database) return parser @@ -1049,7 +1053,7 @@ class Client(zuul.cmd.ZuulApp): cutoff = parse_cutoff(now, args.before, args.older_than) self.configure_connections(source_only=False, require_sql=True) connection = self.connections.getSqlConnection() - connection.deleteBuildsets(cutoff) + connection.deleteBuildsets(cutoff, args.batch_size) sys.exit(0) diff --git a/zuul/driver/gerrit/gerritconnection.py b/zuul/driver/gerrit/gerritconnection.py index f871671aa..6efca17c5 100644 --- a/zuul/driver/gerrit/gerritconnection.py +++ b/zuul/driver/gerrit/gerritconnection.py @@ -1182,9 +1182,34 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection): return True if change.wip: return False - if change.missing_labels <= set(allow_needs): - return True - return False + if change.missing_labels > set(allow_needs): + self.log.debug("Unable to merge due to " + "missing labels: %s", change.missing_labels) + return False + for sr in change.submit_requirements: + if sr.get('status') == 'UNSATISFIED': + # Otherwise, we don't care and should skip. + + # We're going to look at each unsatisfied submit + # requirement, and if one of the involved labels is an + # "allow_needs" label, we will assume that Zuul may be + # able to take an action which can cause the + # requirement to be satisfied, and we will ignore it. + # Otherwise, it is likely a requirement that Zuul can + # not alter in which case the requirement should stand + # and block merging. + result = sr.get("submittability_expression_result", {}) + expression = result.get("expression", '') + expr_contains_allow = False + for allow in allow_needs: + if f'label:{allow}' in expression: + expr_contains_allow = True + break + if not expr_contains_allow: + self.log.debug("Unable to merge due to " + "submit requirement: %s", sr) + return False + return True def getProjectOpenChanges(self, project: Project) -> List[GerritChange]: # This is a best-effort function in case Gerrit is unable to return @@ -1443,9 +1468,12 @@ class GerritConnection(ZKChangeCacheMixin, ZKBranchCacheMixin, BaseConnection): return data def queryChangeHTTP(self, number, event=None): - data = self.get('changes/%s?o=DETAILED_ACCOUNTS&o=CURRENT_REVISION&' - 'o=CURRENT_COMMIT&o=CURRENT_FILES&o=LABELS&' - 'o=DETAILED_LABELS' % (number,)) + query = ('changes/%s?o=DETAILED_ACCOUNTS&o=CURRENT_REVISION&' + 'o=CURRENT_COMMIT&o=CURRENT_FILES&o=LABELS&' + 'o=DETAILED_LABELS' % (number,)) + if self.version >= (3, 5, 0): + query += '&o=SUBMIT_REQUIREMENTS' + data = self.get(query) related = self.get('changes/%s/revisions/%s/related' % ( number, data['current_revision'])) files = self.get('changes/%s/revisions/%s/files?parent=1' % ( diff --git a/zuul/driver/gerrit/gerritmodel.py b/zuul/driver/gerrit/gerritmodel.py index 0ac3e7f9d..4ac291f2b 100644 --- a/zuul/driver/gerrit/gerritmodel.py +++ b/zuul/driver/gerrit/gerritmodel.py @@ -34,6 +34,7 @@ class GerritChange(Change): self.wip = None self.approvals = [] self.missing_labels = set() + self.submit_requirements = [] self.commit = None self.zuul_query_ltime = None @@ -52,6 +53,7 @@ class GerritChange(Change): "wip": self.wip, "approvals": self.approvals, "missing_labels": list(self.missing_labels), + "submit_requirements": self.submit_requirements, "commit": self.commit, "zuul_query_ltime": self.zuul_query_ltime, }) @@ -64,6 +66,7 @@ class GerritChange(Change): self.wip = data["wip"] self.approvals = data["approvals"] self.missing_labels = set(data["missing_labels"]) + self.submit_requirements = data.get("submit_requirements", []) self.commit = data.get("commit") self.zuul_query_ltime = data.get("zuul_query_ltime") @@ -189,6 +192,7 @@ class GerritChange(Change): if 'approved' in label_data: continue self.missing_labels.add(label_name) + self.submit_requirements = data.get('submit_requirements', []) self.open = data['status'] == 'NEW' self.status = data['status'] self.wip = data.get('work_in_progress', False) diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py index 0155e6963..a1353cb4d 100644 --- a/zuul/driver/github/githubconnection.py +++ b/zuul/driver/github/githubconnection.py @@ -684,6 +684,11 @@ class GithubEventProcessor(object): branch, project_name) events.append( self._branch_protection_rule_to_event(project_name, branch)) + + for event in events: + # Make sure every event has a branch cache ltime + self.connection.clearConnectionCacheOnBranchEvent(event) + return events def _branch_protection_rule_to_event(self, project_name, branch): diff --git a/zuul/driver/sql/sqlconnection.py b/zuul/driver/sql/sqlconnection.py index 2d5c39ec3..7a4aea626 100644 --- a/zuul/driver/sql/sqlconnection.py +++ b/zuul/driver/sql/sqlconnection.py @@ -247,12 +247,25 @@ class DatabaseSession(object): except sqlalchemy.orm.exc.MultipleResultsFound: raise Exception("Multiple buildset found with uuid %s", uuid) - def deleteBuildsets(self, cutoff): + def deleteBuildsets(self, cutoff, batch_size): """Delete buildsets before the cutoff""" # delete buildsets updated before the cutoff - for buildset in self.getBuildsets(updated_max=cutoff): - self.session().delete(buildset) + deleted = True + while deleted: + deleted = False + oldest = None + for buildset in self.getBuildsets( + updated_max=cutoff, limit=batch_size): + deleted = True + if oldest is None: + oldest = buildset.updated + else: + oldest = min(oldest, buildset.updated) + self.session().delete(buildset) + self.session().commit() + if deleted: + self.log.info("Deleted from %s to %s", oldest, cutoff) class SQLConnection(BaseConnection): @@ -409,7 +422,10 @@ class SQLConnection(BaseConnection): final = sa.Column(sa.Boolean) held = sa.Column(sa.Boolean) nodeset = sa.Column(sa.String(255)) - buildset = orm.relationship(BuildSetModel, backref="builds") + buildset = orm.relationship(BuildSetModel, + backref=orm.backref( + "builds", + cascade="all, delete-orphan")) sa.Index(self.table_prefix + 'job_name_buildset_id_idx', job_name, buildset_id) @@ -468,7 +484,10 @@ class SQLConnection(BaseConnection): name = sa.Column(sa.String(255)) url = sa.Column(sa.TEXT()) meta = sa.Column('metadata', sa.TEXT()) - build = orm.relationship(BuildModel, backref="artifacts") + build = orm.relationship(BuildModel, + backref=orm.backref( + "artifacts", + cascade="all, delete-orphan")) class ProvidesModel(Base): __tablename__ = self.table_prefix + PROVIDES_TABLE @@ -476,7 +495,10 @@ class SQLConnection(BaseConnection): build_id = sa.Column(sa.Integer, sa.ForeignKey( self.table_prefix + BUILD_TABLE + ".id")) name = sa.Column(sa.String(255)) - build = orm.relationship(BuildModel, backref="provides") + build = orm.relationship(BuildModel, + backref=orm.backref( + "provides", + cascade="all, delete-orphan")) class BuildEventModel(Base): __tablename__ = self.table_prefix + BUILD_EVENTS_TABLE @@ -486,7 +508,10 @@ class SQLConnection(BaseConnection): event_time = sa.Column(sa.DateTime) event_type = sa.Column(sa.String(255)) description = sa.Column(sa.TEXT()) - build = orm.relationship(BuildModel, backref="build_events") + build = orm.relationship(BuildModel, + backref=orm.backref( + "build_events", + cascade="all, delete-orphan")) self.buildEventModel = BuildEventModel self.zuul_build_event_table = self.buildEventModel.__table__ |