Class | BoxGrinder::S3Plugin |
In: |
lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb
lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb |
Parent: | BasePlugin |
REGION_OPTIONS | = | { 'eu-west-1' => { :endpoint => 's3.amazonaws.com', :location => 'EU', :kernel => { 'i386' => {:aki => 'aki-4deec439'}, 'x86_64' => {:aki => 'aki-4feec43b'} |
REGION_OPTIONS | = | { 'eu-west-1' => { :endpoint => 's3.amazonaws.com', :location => 'EU', :kernel => { 'i386' => {:aki => 'aki-4deec439'}, 'x86_64' => {:aki => 'aki-4feec43b'} |
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 70 70: def after_init 71: set_default_config_value('overwrite', false) 72: set_default_config_value('path', '/') 73: set_default_config_value('region', 'us-east-1') 74: 75: register_supported_os("fedora", ['13', '14', '15']) 76: register_supported_os("centos", ['5']) 77: register_supported_os("rhel", ['5', '6']) 78: 79: @ami_build_dir = "#{@dir.base}/ami" 80: @ami_manifest = "#{@ami_build_dir}/#{@appliance_config.name}.ec2.manifest.xml" 81: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 70 70: def after_init 71: set_default_config_value('overwrite', false) 72: set_default_config_value('path', '/') 73: set_default_config_value('region', 'us-east-1') 74: 75: register_supported_os("fedora", ['13', '14', '15']) 76: register_supported_os("centos", ['5']) 77: register_supported_os("rhel", ['5', '6']) 78: 79: @ami_build_dir = "#{@dir.base}/ami" 80: @ami_manifest = "#{@ami_build_dir}/#{@appliance_config.name}.ec2.manifest.xml" 81: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 188 188: def ami_info(ami_manifest_key) 189: ami_info = nil 190: 191: images = @ec2.describe_images(:owner_id => @plugin_config['account_number']).imagesSet 192: 193: return nil if images.nil? 194: 195: for image in images.item do 196: ami_info = image if (image.imageLocation.eql?("#{@plugin_config['bucket']}/#{ami_manifest_key}")) 197: end 198: 199: ami_info 200: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 188 188: def ami_info(ami_manifest_key) 189: ami_info = nil 190: 191: images = @ec2.describe_images(:owner_id => @plugin_config['account_number']).imagesSet 192: 193: return nil if images.nil? 194: 195: for image in images.item do 196: ami_info = image if (image.imageLocation.eql?("#{@plugin_config['bucket']}/#{ami_manifest_key}")) 197: end 198: 199: ami_info 200: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 208 208: def ami_key(appliance_name, path) 209: base_path = "#{s3_path(path)}#{appliance_name}/#{@appliance_config.os.name}/#{@appliance_config.os.version}/#{@appliance_config.version}.#{@appliance_config.release}" 210: 211: return "#{base_path}/#{@appliance_config.hardware.arch}" unless @plugin_config['snapshot'] 212: 213: snapshot = 1 214: 215: while s3_object_exists?("#{base_path}-SNAPSHOT-#{snapshot}/#{@appliance_config.hardware.arch}/") 216: snapshot += 1 217: end 218: 219: "#{base_path}-SNAPSHOT-#{snapshot}/#{@appliance_config.hardware.arch}" 220: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 208 208: def ami_key(appliance_name, path) 209: base_path = "#{s3_path(path)}#{appliance_name}/#{@appliance_config.os.name}/#{@appliance_config.os.version}/#{@appliance_config.version}.#{@appliance_config.release}" 210: 211: return "#{base_path}/#{@appliance_config.hardware.arch}" unless @plugin_config['snapshot'] 212: 213: snapshot = 1 214: 215: while s3_object_exists?("#{base_path}-SNAPSHOT-#{snapshot}/#{@appliance_config.hardware.arch}/") 216: snapshot += 1 217: end 218: 219: "#{base_path}-SNAPSHOT-#{snapshot}/#{@appliance_config.hardware.arch}" 220: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 148 148: def bucket(create_if_missing = true, permissions = 'private') 149: @s3 ||= Aws::S3.new(@plugin_config['access_key'], @plugin_config['secret_access_key'], :connection_mode => :single, :logger => @log, :server => REGION_OPTIONS[@plugin_config['region']][:endpoint]) 150: @s3.bucket(@plugin_config['bucket'], create_if_missing, permissions, :location => REGION_OPTIONS[@plugin_config['region']][:location]) 151: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 148 148: def bucket(create_if_missing = true, permissions = 'private') 149: @s3 ||= Aws::S3.new(@plugin_config['access_key'], @plugin_config['secret_access_key'], :connection_mode => :single, :logger => @log, :server => REGION_OPTIONS[@plugin_config['region']][:endpoint]) 150: @s3.bucket(@plugin_config['bucket'], create_if_missing, permissions, :location => REGION_OPTIONS[@plugin_config['region']][:location]) 151: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 153 153: def bundle_image(deliverables) 154: if @plugin_config['snapshot'] 155: @log.debug "Removing bundled image from local disk..." 156: FileUtils.rm_rf(@ami_build_dir) 157: end 158: 159: return if File.exists?(@ami_build_dir) 160: 161: @log.info "Bundling AMI..." 162: 163: FileUtils.mkdir_p(@ami_build_dir) 164: 165: @exec_helper.execute("euca-bundle-image --ec2cert #{File.dirname(__FILE__)}/src/cert-ec2.pem -i #{deliverables[:disk]} --kernel #{REGION_OPTIONS[@plugin_config['region']][:kernel][@appliance_config.hardware.base_arch][:aki]} -c #{@plugin_config['cert_file']} -k #{@plugin_config['key_file']} -u #{@plugin_config['account_number']} -r #{@appliance_config.hardware.base_arch} -d #{@ami_build_dir}", :redacted => [@plugin_config['account_number'], @plugin_config['key_file'], @plugin_config['cert_file']]) 166: 167: @log.info "Bundling AMI finished." 168: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 153 153: def bundle_image(deliverables) 154: if @plugin_config['snapshot'] 155: @log.debug "Removing bundled image from local disk..." 156: FileUtils.rm_rf(@ami_build_dir) 157: end 158: 159: return if File.exists?(@ami_build_dir) 160: 161: @log.info "Bundling AMI..." 162: 163: FileUtils.mkdir_p(@ami_build_dir) 164: 165: @exec_helper.execute("euca-bundle-image --ec2cert #{File.dirname(__FILE__)}/src/cert-ec2.pem -i #{deliverables[:disk]} --kernel #{REGION_OPTIONS[@plugin_config['region']][:kernel][@appliance_config.hardware.base_arch][:aki]} -c #{@plugin_config['cert_file']} -k #{@plugin_config['key_file']} -u #{@plugin_config['account_number']} -r #{@appliance_config.hardware.base_arch} -d #{@ami_build_dir}", :redacted => [@plugin_config['account_number'], @plugin_config['key_file'], @plugin_config['cert_file']]) 166: 167: @log.info "Bundling AMI finished." 168: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 83 83: def execute(type = :ami) 84: validate_plugin_config(['bucket', 'access_key', 'secret_access_key'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin') 85: 86: case type 87: when :s3 88: upload_to_bucket(@previous_deliverables) 89: when :cloudfront 90: upload_to_bucket(@previous_deliverables, 'public-read') 91: when :ami 92: set_default_config_value('snapshot', false) 93: validate_plugin_config(['cert_file', 'key_file', 'account_number'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin') 94: 95: @plugin_config['account_number'] = @plugin_config['account_number'].to_s.gsub(/-/, '') 96: 97: @ec2 = AWS::EC2::Base.new(:access_key_id => @plugin_config['access_key'], :secret_access_key => @plugin_config['secret_access_key'], :server => "ec2.#{@plugin_config['region']}.amazonaws.com") 98: 99: ami_dir = ami_key(@appliance_config.name, @plugin_config['path']) 100: ami_manifest_key = "#{ami_dir}/#{@appliance_config.name}.ec2.manifest.xml" 101: 102: if !s3_object_exists?(ami_manifest_key) or @plugin_config['snapshot'] 103: bundle_image(@previous_deliverables) 104: fix_sha1_sum 105: upload_image(ami_dir) 106: else 107: @log.debug "AMI for #{@appliance_config.name} appliance already uploaded, skipping..." 108: end 109: 110: register_image(ami_manifest_key) 111: end 112: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 83 83: def execute(type = :ami) 84: validate_plugin_config(['bucket', 'access_key', 'secret_access_key'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin') 85: 86: case type 87: when :s3 88: upload_to_bucket(@previous_deliverables) 89: when :cloudfront 90: upload_to_bucket(@previous_deliverables, 'public-read') 91: when :ami 92: set_default_config_value('snapshot', false) 93: validate_plugin_config(['cert_file', 'key_file', 'account_number'], 'http://boxgrinder.org/tutorials/boxgrinder-build-plugins/#S3_Delivery_Plugin') 94: 95: @plugin_config['account_number'] = @plugin_config['account_number'].to_s.gsub(/-/, '') 96: 97: @ec2 = AWS::EC2::Base.new(:access_key_id => @plugin_config['access_key'], :secret_access_key => @plugin_config['secret_access_key'], :server => "ec2.#{@plugin_config['region']}.amazonaws.com") 98: 99: ami_dir = ami_key(@appliance_config.name, @plugin_config['path']) 100: ami_manifest_key = "#{ami_dir}/#{@appliance_config.name}.ec2.manifest.xml" 101: 102: if !s3_object_exists?(ami_manifest_key) or @plugin_config['snapshot'] 103: bundle_image(@previous_deliverables) 104: fix_sha1_sum 105: upload_image(ami_dir) 106: else 107: @log.debug "AMI for #{@appliance_config.name} appliance already uploaded, skipping..." 108: end 109: 110: register_image(ami_manifest_key) 111: end 112: end
jira.jboss.org/browse/BGBUILD-34
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 115 115: def fix_sha1_sum 116: ami_manifest = File.open(@ami_manifest).read 117: ami_manifest.gsub!('(stdin)= ', '') 118: 119: File.open(@ami_manifest, "w") { |f| f.write(ami_manifest) } 120: end
jira.jboss.org/browse/BGBUILD-34
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 115 115: def fix_sha1_sum 116: ami_manifest = File.open(@ami_manifest).read 117: ami_manifest.gsub!('(stdin)= ', '') 118: 119: File.open(@ami_manifest, "w") { |f| f.write(ami_manifest) } 120: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 177 177: def register_image(ami_manifest_key) 178: info = ami_info(ami_manifest_key) 179: 180: if info 181: @log.info "Image for #{@appliance_config.name} is registered under id: #{info.imageId} (region: #{@plugin_config['region']})." 182: else 183: info = @ec2.register_image(:image_location => "#{@plugin_config['bucket']}/#{ami_manifest_key}") 184: @log.info "Image for #{@appliance_config.name} successfully registered under id: #{info.imageId} (region: #{@plugin_config['region']})." 185: end 186: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 177 177: def register_image(ami_manifest_key) 178: info = ami_info(ami_manifest_key) 179: 180: if info 181: @log.info "Image for #{@appliance_config.name} is registered under id: #{info.imageId} (region: #{@plugin_config['region']})." 182: else 183: info = @ec2.register_image(:image_location => "#{@plugin_config['bucket']}/#{ami_manifest_key}") 184: @log.info "Image for #{@appliance_config.name} successfully registered under id: #{info.imageId} (region: #{@plugin_config['region']})." 185: end 186: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 222 222: def s3_object_exists?(path) 223: @log.trace "Checking if '#{path}' path exists in #{@plugin_config['bucket']}..." 224: 225: begin 226: b = bucket(false) 227: # Retrieve only one or no keys (if bucket is empty), throw an exception if bucket doesn't exists 228: b.keys('max-keys' => 1) 229: 230: if b.key(path).exists? 231: @log.trace "Path exists!" 232: return true 233: end 234: rescue 235: end 236: @log.trace "Path doesn't exist!" 237: false 238: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 222 222: def s3_object_exists?(path) 223: @log.trace "Checking if '#{path}' path exists in #{@plugin_config['bucket']}..." 224: 225: begin 226: b = bucket(false) 227: # Retrieve only one or no keys (if bucket is empty), throw an exception if bucket doesn't exists 228: b.keys('max-keys' => 1) 229: 230: if b.key(path).exists? 231: @log.trace "Path exists!" 232: return true 233: end 234: rescue 235: end 236: @log.trace "Path doesn't exist!" 237: false 238: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 202 202: def s3_path(path) 203: return '' if path == '/' 204: 205: "#{path.gsub(/^(\/)*/, '').gsub(/(\/)*$/, '')}/" 206: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 202 202: def s3_path(path) 203: return '' if path == '/' 204: 205: "#{path.gsub(/^(\/)*/, '').gsub(/(\/)*$/, '')}/" 206: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 170 170: def upload_image(ami_dir) 171: bucket # this will create the bucket if needed 172: @log.info "Uploading #{@appliance_config.name} AMI to bucket '#{@plugin_config['bucket']}'..." 173: 174: @exec_helper.execute("euca-upload-bundle -U #{@plugin_config['url'].nil? ? "http://#{REGION_OPTIONS[@plugin_config['region']][:endpoint]}" : @plugin_config['url']} -b #{@plugin_config['bucket']}/#{ami_dir} -m #{@ami_manifest} -a #{@plugin_config['access_key']} -s #{@plugin_config['secret_access_key']}", :redacted => [@plugin_config['access_key'], @plugin_config['secret_access_key']]) 175: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 170 170: def upload_image(ami_dir) 171: bucket # this will create the bucket if needed 172: @log.info "Uploading #{@appliance_config.name} AMI to bucket '#{@plugin_config['bucket']}'..." 173: 174: @exec_helper.execute("euca-upload-bundle -U #{@plugin_config['url'].nil? ? "http://#{REGION_OPTIONS[@plugin_config['region']][:endpoint]}" : @plugin_config['url']} -b #{@plugin_config['bucket']}/#{ami_dir} -m #{@ami_manifest} -a #{@plugin_config['access_key']} -s #{@plugin_config['secret_access_key']}", :redacted => [@plugin_config['access_key'], @plugin_config['secret_access_key']]) 175: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 122 122: def upload_to_bucket(previous_deliverables, permissions = 'private') 123: register_deliverable( 124: :package => "#{@appliance_config.name}-#{@appliance_config.version}.#{@appliance_config.release}-#{@appliance_config.os.name}-#{@appliance_config.os.version}-#{@appliance_config.hardware.arch}-#{current_platform}.tgz" 125: ) 126: 127: # quick and dirty workaround to use @deliverables[:package] later in code 128: FileUtils.mv(@target_deliverables[:package], @deliverables[:package]) if File.exists?(@target_deliverables[:package]) 129: 130: PackageHelper.new(@config, @appliance_config, :log => @log, :exec_helper => @exec_helper).package(File.dirname(previous_deliverables[:disk]), @deliverables[:package]) 131: 132: remote_path = "#{s3_path(@plugin_config['path'])}#{File.basename(@deliverables[:package])}" 133: size_b = File.size(@deliverables[:package]) 134: 135: key = bucket(true, permissions).key(remote_path.gsub(/^\//, '').gsub(/\/\//, '')) 136: 137: unless key.exists? or @plugin_config['overwrite'] 138: @log.info "Uploading #{File.basename(@deliverables[:package])} (#{size_b/1024/1024}MB) to '#{@plugin_config['bucket']}#{remote_path}' path..." 139: key.put(open(@deliverables[:package]), permissions, :server => REGION_OPTIONS[@plugin_config['region']][:endpoint]) 140: @log.info "Appliance #{@appliance_config.name} uploaded to S3." 141: else 142: @log.info "File '#{@plugin_config['bucket']}#{remote_path}' already uploaded, skipping." 143: end 144: 145: @s3.close_connection 146: end
# File lib/boxgrinder-build/plugins/delivery/s3/s3-plugin.rb, line 122 122: def upload_to_bucket(previous_deliverables, permissions = 'private') 123: register_deliverable( 124: :package => "#{@appliance_config.name}-#{@appliance_config.version}.#{@appliance_config.release}-#{@appliance_config.os.name}-#{@appliance_config.os.version}-#{@appliance_config.hardware.arch}-#{current_platform}.tgz" 125: ) 126: 127: # quick and dirty workaround to use @deliverables[:package] later in code 128: FileUtils.mv(@target_deliverables[:package], @deliverables[:package]) if File.exists?(@target_deliverables[:package]) 129: 130: PackageHelper.new(@config, @appliance_config, :log => @log, :exec_helper => @exec_helper).package(File.dirname(previous_deliverables[:disk]), @deliverables[:package]) 131: 132: remote_path = "#{s3_path(@plugin_config['path'])}#{File.basename(@deliverables[:package])}" 133: size_b = File.size(@deliverables[:package]) 134: 135: key = bucket(true, permissions).key(remote_path.gsub(/^\//, '').gsub(/\/\//, '')) 136: 137: unless key.exists? or @plugin_config['overwrite'] 138: @log.info "Uploading #{File.basename(@deliverables[:package])} (#{size_b/1024/1024}MB) to '#{@plugin_config['bucket']}#{remote_path}' path..." 139: key.put(open(@deliverables[:package]), permissions, :server => REGION_OPTIONS[@plugin_config['region']][:endpoint]) 140: @log.info "Appliance #{@appliance_config.name} uploaded to S3." 141: else 142: @log.info "File '#{@plugin_config['bucket']}#{remote_path}' already uploaded, skipping." 143: end 144: 145: @s3.close_connection 146: end