Fix unpickle for source paths missing run_dir (#863)

On the datasource class, we require the use of paths.run_dir to
perform some operations. On older cloud-init version, the
Paths class does not have the run_dir attribute. To fix that,
we are now manually adding that attribute in the Paths
object if doesn't exist in the unpickle operation.

LP: #1899299

3 files changed, 523 insertions, 1 deletions

diff --git a/cloudinit/helpers.py b/cloudinit/helpers.py
index fc5011ec..b8f9d2c3 100644
--- a/cloudinit/helpers.py
+++ b/cloudinit/helpers.py
@@ -20,6 +20,7 @@ from cloudinit.settings import (PER_INSTANCE, PER_ALWAYS, PER_ONCE,
 
 from cloudinit import log as logging
 from cloudinit import type_utils
+from cloudinit import persistence
 from cloudinit import util
 
 LOG = logging.getLogger(__name__)
@@ -317,7 +318,9 @@ class ContentHandlers(object):
         return list(self.registered.items())
 
 
-class Paths(object):
+class Paths(persistence.CloudInitPickleMixin):
+    _ci_pkl_version = 1
+
     def __init__(self, path_cfgs, ds=None):
         self.cfgs = path_cfgs
         # Populate all the initial paths
@@ -354,6 +357,18 @@ class Paths(object):
         # Set when a datasource becomes active
         self.datasource = ds
 
+    def _unpickle(self, ci_pkl_version: int) -> None:
+        """Perform deserialization fixes for Paths."""
+        if not hasattr(self, "run_dir"):
+            # On older versions of cloud-init the Paths class do not
+            # have the run_dir attribute. This is problematic because
+            # when loading the pickle object on newer versions of cloud-init
+            # we will rely on this attribute. To fix that, we are now
+            # manually adding that attribute here.
+            self.run_dir = Paths(
+                path_cfgs=self.cfgs,
+                ds=self.datasource).run_dir
+
     # get_ipath_cur: get the current instance path for an item
     def get_ipath_cur(self, name=None):
         return self._get_path(self.instance_link, name)
diff --git a/cloudinit/tests/test_upgrade.py b/cloudinit/tests/test_upgrade.py
index f79a2536..05eefd2a 100644
--- a/cloudinit/tests/test_upgrade.py
+++ b/cloudinit/tests/test_upgrade.py
@@ -43,3 +43,6 @@ class TestUpgrade:
     def test_blacklist_drivers_set_on_networking(self, previous_obj_pkl):
         """We always expect Networking.blacklist_drivers to be initialised."""
         assert previous_obj_pkl.distro.networking.blacklist_drivers is None
+
+    def test_paths_has_run_dir_attribute(self, previous_obj_pkl):
+        assert previous_obj_pkl.paths.run_dir is not None
diff --git a/tests/data/old_pickles/trusty-14.04.1-0.7.5.pkl b/tests/data/old_pickles/trusty-14.04.1-0.7.5.pkl
new file mode 100644
index 00000000..c7d7844b
--- /dev/null
+++ b/tests/data/old_pickles/trusty-14.04.1-0.7.5.pkl
@@ -0,0 +1,504 @@
+ccopy_reg
+_reconstructor
+p1
+(ccloudinit.sources.DataSourceNoCloud
+DataSourceNoCloudNet
+p2
+c__builtin__
+object
+p3
+NtRp4
+(dp5
+S'paths'
+p6
+g1
+(ccloudinit.helpers
+Paths
+p7
+g3
+NtRp8
+(dp9
+S'lookups'
+p10
+(dp11
+S'cloud_config'
+p12
+S'cloud-config.txt'
+p13
+sS'userdata'
+p14
+S'user-data.txt.i'
+p15
+sS'vendordata'
+p16
+S'vendor-data.txt.i'
+p17
+sS'userdata_raw'
+p18
+S'user-data.txt'
+p19
+sS'boothooks'
+p20
+g20
+sS'scripts'
+p21
+g21
+sS'sem'
+p22
+g22
+sS'data'
+p23
+g23
+sS'vendor_scripts'
+p24
+S'scripts/vendor'
+p25
+sS'handlers'
+p26
+g26
+sS'obj_pkl'
+p27
+S'obj.pkl'
+p28
+sS'vendordata_raw'
+p29
+S'vendor-data.txt'
+p30
+sS'vendor_cloud_config'
+p31
+S'vendor-cloud-config.txt'
+p32
+ssS'template_tpl'
+p33
+S'/etc/cloud/templates/%s.tmpl'
+p34
+sS'cfgs'
+p35
+(dp36
+S'cloud_dir'
+p37
+S'/var/lib/cloud/'
+p38
+sS'templates_dir'
+p39
+S'/etc/cloud/templates/'
+p40
+sS'upstart_dir'
+p41
+S'/etc/init/'
+p42
+ssS'cloud_dir'
+p43
+g38
+sS'datasource'
+p44
+NsS'upstart_conf_d'
+p45
+g42
+sS'boot_finished'
+p46
+S'/var/lib/cloud/instance/boot-finished'
+p47
+sS'instance_link'
+p48
+S'/var/lib/cloud/instance'
+p49
+sS'seed_dir'
+p50
+S'/var/lib/cloud/seed'
+p51
+sbsS'supported_seed_starts'
+p52
+(S'http://'
+S'https://'
+S'ftp://'
+tp53
+sS'sys_cfg'
+p54
+(dp55
+S'output'
+p56
+(dp57
+S'all'
+p58
+S'| tee -a /var/log/cloud-init-output.log'
+p59
+ssS'users'
+p60
+(lp61
+S'default'
+p62
+asS'def_log_file'
+p63
+S'/var/log/cloud-init.log'
+p64
+sS'cloud_final_modules'
+p65
+(lp66
+S'rightscale_userdata'
+p67
+aS'scripts-vendor'
+p68
+aS'scripts-per-once'
+p69
+aS'scripts-per-boot'
+p70
+aS'scripts-per-instance'
+p71
+aS'scripts-user'
+p72
+aS'ssh-authkey-fingerprints'
+p73
+aS'keys-to-console'
+p74
+aS'phone-home'
+p75
+aS'final-message'
+p76
+aS'power-state-change'
+p77
+asS'disable_root'
+p78
+I01
+sS'syslog_fix_perms'
+p79
+S'syslog:adm'
+p80
+sS'log_cfgs'
+p81
+(lp82
+(lp83
+S'[loggers]\nkeys=root,cloudinit\n\n[handlers]\nkeys=consoleHandler,cloudLogHandler\n\n[formatters]\nkeys=simpleFormatter,arg0Formatter\n\n[logger_root]\nlevel=DEBUG\nhandlers=consoleHandler,cloudLogHandler\n\n[logger_cloudinit]\nlevel=DEBUG\nqualname=cloudinit\nhandlers=\npropagate=1\n\n[handler_consoleHandler]\nclass=StreamHandler\nlevel=WARNING\nformatter=arg0Formatter\nargs=(sys.stderr,)\n\n[formatter_arg0Formatter]\nformat=%(asctime)s - %(filename)s[%(levelname)s]: %(message)s\n\n[formatter_simpleFormatter]\nformat=[CLOUDINIT] %(filename)s[%(levelname)s]: %(message)s\n'
+p84
+aS'[handler_cloudLogHandler]\nclass=handlers.SysLogHandler\nlevel=DEBUG\nformatter=simpleFormatter\nargs=("/dev/log", handlers.SysLogHandler.LOG_USER)\n'
+p85
+aa(lp86
+g84
+aS"[handler_cloudLogHandler]\nclass=FileHandler\nlevel=DEBUG\nformatter=arg0Formatter\nargs=('/var/log/cloud-init.log',)\n"
+p87
+aasS'cloud_init_modules'
+p88
+(lp89
+S'migrator'
+p90
+aS'seed_random'
+p91
+aS'bootcmd'
+p92
+aS'write-files'
+p93
+aS'growpart'
+p94
+aS'resizefs'
+p95
+aS'set_hostname'
+p96
+aS'update_hostname'
+p97
+aS'update_etc_hosts'
+p98
+aS'ca-certs'
+p99
+aS'rsyslog'
+p100
+aS'users-groups'
+p101
+aS'ssh'
+p102
+asS'preserve_hostname'
+p103
+I00
+sS'_log'
+p104
+(lp105
+g84
+ag87
+ag85
+asS'datasource_list'
+p106
+(lp107
+S'NoCloud'
+p108
+aS'ConfigDrive'
+p109
+aS'OpenNebula'
+p110
+aS'Azure'
+p111
+aS'AltCloud'
+p112
+aS'OVF'
+p113
+aS'MAAS'
+p114
+aS'GCE'
+p115
+aS'OpenStack'
+p116
+aS'CloudSigma'
+p117
+aS'Ec2'
+p118
+aS'CloudStack'
+p119
+aS'SmartOS'
+p120
+aS'None'
+p121
+asS'vendor_data'
+p122
+(dp123
+S'prefix'
+p124
+(lp125
+sS'enabled'
+p126
+I01
+ssS'cloud_config_modules'
+p127
+(lp128
+S'emit_upstart'
+p129
+aS'disk_setup'
+p130
+aS'mounts'
+p131
+aS'ssh-import-id'
+p132
+aS'locale'
+p133
+aS'set-passwords'
+p134
+aS'grub-dpkg'
+p135
+aS'apt-pipelining'
+p136
+aS'apt-configure'
+p137
+aS'package-update-upgrade-install'
+p138
+aS'landscape'
+p139
+aS'timezone'
+p140
+aS'puppet'
+p141
+aS'chef'
+p142
+aS'salt-minion'
+p143
+aS'mcollective'
+p144
+aS'disable-ec2-metadata'
+p145
+aS'runcmd'
+p146
+aS'byobu'
+p147
+assg14
+Nsg16
+Nsg18
+S'#cloud-config\n{}\n\n'
+p148
+sg29
+S'#cloud-config\n{}\n\n'
+p149
+sS'dsmode'
+p150
+S'net'
+p151
+sS'seed'
+p152
+S'/var/lib/cloud/seed/nocloud-net'
+p153
+sS'cmdline_id'
+p154
+S'ds=nocloud-net'
+p155
+sS'ud_proc'
+p156
+g1
+(ccloudinit.user_data
+UserDataProcessor
+p157
+g3
+NtRp158
+(dp159
+g6
+g8
+sS'ssl_details'
+p160
+(dp161
+sbsg50
+g153
+sS'ds_cfg'
+p162
+(dp163
+sS'distro'
+p164
+g1
+(ccloudinit.distros.ubuntu
+Distro
+p165
+g3
+NtRp166
+(dp167
+S'osfamily'
+p168
+S'debian'
+p169
+sS'_paths'
+p170
+g8
+sS'name'
+p171
+S'ubuntu'
+p172
+sS'_runner'
+p173
+g1
+(ccloudinit.helpers
+Runners
+p174
+g3
+NtRp175
+(dp176
+g6
+g8
+sS'sems'
+p177
+(dp178
+sbsS'_cfg'
+p179
+(dp180
+S'paths'
+p181
+(dp182
+g37
+g38
+sg39
+g40
+sg41
+g42
+ssS'default_user'
+p183
+(dp184
+S'shell'
+p185
+S'/bin/bash'
+p186
+sS'name'
+p187
+S'ubuntu'
+p188
+sS'sudo'
+p189
+(lp190
+S'ALL=(ALL) NOPASSWD:ALL'
+p191
+asS'lock_passwd'
+p192
+I01
+sS'gecos'
+p193
+S'Ubuntu'
+p194
+sS'groups'
+p195
+(lp196
+S'adm'
+p197
+aS'audio'
+p198
+aS'cdrom'
+p199
+aS'dialout'
+p200
+aS'dip'
+p201
+aS'floppy'
+p202
+aS'netdev'
+p203
+aS'plugdev'
+p204
+aS'sudo'
+p205
+aS'video'
+p206
+assS'package_mirrors'
+p207
+(lp208
+(dp209
+S'arches'
+p210
+(lp211
+S'i386'
+p212
+aS'amd64'
+p213
+asS'failsafe'
+p214
+(dp215
+S'security'
+p216
+S'http://security.ubuntu.com/ubuntu'
+p217
+sS'primary'
+p218
+S'http://archive.ubuntu.com/ubuntu'
+p219
+ssS'search'
+p220
+(dp221
+S'security'
+p222
+(lp223
+sS'primary'
+p224
+(lp225
+S'http://%(ec2_region)s.ec2.archive.ubuntu.com/ubuntu/'
+p226
+aS'http://%(availability_zone)s.clouds.archive.ubuntu.com/ubuntu/'
+p227
+aS'http://%(region)s.clouds.archive.ubuntu.com/ubuntu/'
+p228
+assa(dp229
+S'arches'
+p230
+(lp231
+S'armhf'
+p232
+aS'armel'
+p233
+aS'default'
+p234
+asS'failsafe'
+p235
+(dp236
+S'security'
+p237
+S'http://ports.ubuntu.com/ubuntu-ports'
+p238
+sS'primary'
+p239
+S'http://ports.ubuntu.com/ubuntu-ports'
+p240
+ssasS'ssh_svcname'
+p241
+S'ssh'
+p242
+ssbsS'metadata'
+p243
+(dp244
+g150
+g151
+sS'local-hostname'
+p245
+S'trusty-upgrade2'
+p246
+sS'instance-id'
+p247
+S'trusty-upgrade2'
+p248
+ssb.
\ No newline at end of file