From 6be1032f41f1eb8cf321320fb8bcd26bcd86e873 Mon Sep 17 00:00:00 2001 From: Slawosz Slawinski Date: Tue, 1 Sep 2026 10:28:21 +0100 Subject: [PATCH] Remove airbyte config json from gem config, it is now generated dynamically during airbyte rake tasks --- config/locales/en.yml | 4 -- lib/dfe/analytics.rb | 6 -- lib/dfe/analytics/airbyte_stream_config.rb | 4 -- lib/dfe/analytics/config.rb | 3 +- lib/dfe/analytics/fields.rb | 14 ---- lib/dfe/analytics/tasks/fields.rake | 8 --- .../dfe/analytics/install_generator.rb | 4 -- .../analytics/airbyte_stream_config_spec.rb | 9 --- spec/dfe/analytics/config_spec.rb | 4 +- .../generators/install_generator_spec.rb | 8 --- spec/dfe/analytics_spec.rb | 66 ------------------- 11 files changed, 3 insertions(+), 127 deletions(-) diff --git a/config/locales/en.yml b/config/locales/en.yml index 287acc4b..eb94de5c 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -116,10 +116,6 @@ en: description: | Perform airbyte checks on startup and allow airbyte config generation default: false - airbyte_stream_config_path: - description: | - Path of airbyte stream config file relative to the App root (Rails.root) - default: /terraform/aks/workspace-variables/airbyte_stream_config.json airbyte_internal_dataset: description: | Airbyte internal dataset for service diff --git a/lib/dfe/analytics.rb b/lib/dfe/analytics.rb index 32272588..414d5d10 100644 --- a/lib/dfe/analytics.rb +++ b/lib/dfe/analytics.rb @@ -120,12 +120,6 @@ def self.event_debug_filters {} end - def self.airbyte_stream_config - JSON.parse(File.read(DfE::Analytics.config.airbyte_stream_config_path)).deep_symbolize_keys - rescue RuntimeError - {} - end - def self.environment config.environment end diff --git a/lib/dfe/analytics/airbyte_stream_config.rb b/lib/dfe/analytics/airbyte_stream_config.rb index 49031215..1954ce36 100644 --- a/lib/dfe/analytics/airbyte_stream_config.rb +++ b/lib/dfe/analytics/airbyte_stream_config.rb @@ -13,10 +13,6 @@ class AirbyteStreamConfig AIRBYTE_HEARTBEAT_ATTRIBUTES = %w[id last_heartbeat].freeze AIRBYTE_HEARTBEAT_ENTITY_ATTRIBUTES = { AIRBYTE_HEARTBEAT_ENTITY.to_sym => AIRBYTE_HEARTBEAT_ATTRIBUTES }.freeze - def self.generate_pretty_json_for(table_attributes) - JSON.pretty_generate(generate_for(table_attributes)) - end - def self.generate_for(table_attributes) { configurations: { streams: streams_for(table_attributes) } } end diff --git a/lib/dfe/analytics/config.rb b/lib/dfe/analytics/config.rb index 834ed61c..bc321e87 100644 --- a/lib/dfe/analytics/config.rb +++ b/lib/dfe/analytics/config.rb @@ -76,11 +76,10 @@ def self.configure(config) config.airbyte_server_url ||= ENV.fetch('AIRBYTE_SERVER_URL', nil) config.airbyte_configuration ||= JSON.parse(ENV.fetch('AIRBYTE_CONFIGURATION', '{}')).transform_keys(&:underscore).symbolize_keys + config.airbyte_stream_config_path ||= nil config.async = true if config.async.nil? - config.airbyte_stream_config_path = File.join(Rails.root, config.airbyte_stream_config_path) if config.airbyte_stream_config_path.present? - return unless config.azure_federated_auth config.azure_client_id ||= ENV.fetch('AZURE_CLIENT_ID', nil) diff --git a/lib/dfe/analytics/fields.rb b/lib/dfe/analytics/fields.rb index 3e885063..52443bf4 100644 --- a/lib/dfe/analytics/fields.rb +++ b/lib/dfe/analytics/fields.rb @@ -45,20 +45,6 @@ def self.check! HEREDOC end - if DfE::Analytics.airbyte_enabled? && airbyte_conflicting_fields.any? - errors << <<~HEREDOC - Differences detected between analytics.yml and #{File.basename(DfE::Analytics.config.airbyte_stream_config_path)}! - - The following field differences exist. To upgrade the airbyte stream config, run: - - SUPPRESS_DFE_ANALYTICS_INIT=1 bundle exec rails dfe:analytics:regenerate_airbyte_stream_config - - Field Differences: - - #{airbyte_conflicting_fields.to_yaml} - HEREDOC - end - configuration_errors = errors.join("\n\n----------------\n\n") raise(ConfigurationError, configuration_errors) if errors.any? diff --git a/lib/dfe/analytics/tasks/fields.rake b/lib/dfe/analytics/tasks/fields.rake index 5df1bd3b..997c6990 100644 --- a/lib/dfe/analytics/tasks/fields.rake +++ b/lib/dfe/analytics/tasks/fields.rake @@ -12,13 +12,5 @@ namespace :dfe do { shared: DfE::Analytics::Fields.generate_blocklist }.to_yaml ) end - - desc 'Generate a new airbyte stream config listing all fields to be send to Bigquery from analytics.yml' - task regenerate_airbyte_stream_config: :environment do - File.write( - DfE::Analytics.config.airbyte_stream_config_path, - DfE::Analytics::AirbyteStreamConfig.generate_pretty_json_for(DfE::Analytics.allowlist) - ) - end end end diff --git a/lib/generators/dfe/analytics/install_generator.rb b/lib/generators/dfe/analytics/install_generator.rb index a0e1644b..5b9c3697 100644 --- a/lib/generators/dfe/analytics/install_generator.rb +++ b/lib/generators/dfe/analytics/install_generator.rb @@ -13,10 +13,6 @@ def install create_file 'config/analytics.yml', { 'shared' => {} }.to_yaml create_file 'config/analytics_hidden_pii.yml', { 'shared' => {} }.to_yaml create_file 'config/analytics_blocklist.yml', { 'shared' => {} }.to_yaml - create_file( - DfE::Analytics.config.airbyte_stream_config_path, - DfE::Analytics::AirbyteStreamConfig.generate_pretty_json_for(table1: %w[id field1 field2]) - ) end private diff --git a/spec/dfe/analytics/airbyte_stream_config_spec.rb b/spec/dfe/analytics/airbyte_stream_config_spec.rb index 3aacf642..b4c0e47b 100644 --- a/spec/dfe/analytics/airbyte_stream_config_spec.rb +++ b/spec/dfe/analytics/airbyte_stream_config_spec.rb @@ -56,15 +56,6 @@ end end - describe '.generate_pretty_json_for' do - let(:entity_attributes) { { users: %w[id name] } } - - it 'returns a pretty JSON version of generate_for' do - expect(described_class.generate_pretty_json_for(entity_attributes)) - .to eq(JSON.pretty_generate(described_class.generate_for(entity_attributes))) - end - end - describe '.entity_attributes' do context 'when config is empty' do before { allow(DfE::Analytics).to receive(:airbyte_stream_config).and_return({}) } diff --git a/spec/dfe/analytics/config_spec.rb b/spec/dfe/analytics/config_spec.rb index e48638c2..9adb83aa 100644 --- a/spec/dfe/analytics/config_spec.rb +++ b/spec/dfe/analytics/config_spec.rb @@ -73,8 +73,8 @@ described_class.configure(config) end - it 'resolves the full path from Rails.root' do - expect(config.airbyte_stream_config_path).to eq(Rails.root.join('config/airbyte.json').to_s) + it 'still stores the value' do + expect(config.airbyte_stream_config_path).to eq('config/airbyte.json') end end end diff --git a/spec/dfe/analytics/generators/install_generator_spec.rb b/spec/dfe/analytics/generators/install_generator_spec.rb index 786b5c6d..676295c1 100644 --- a/spec/dfe/analytics/generators/install_generator_spec.rb +++ b/spec/dfe/analytics/generators/install_generator_spec.rb @@ -64,14 +64,6 @@ { 'shared' => {} }.to_yaml ) end - it 'creates the airbyte stream config file' do - generator.install - - expect(generator).to have_received(:create_file).with( - airbyte_stream_config_path, - generated_stream_config - ) - end it 'uses the generated config options in the initializer' do generator.install diff --git a/spec/dfe/analytics_spec.rb b/spec/dfe/analytics_spec.rb index db375db6..9ce77b5f 100644 --- a/spec/dfe/analytics_spec.rb +++ b/spec/dfe/analytics_spec.rb @@ -427,70 +427,4 @@ end end end - - describe '.airbyte_stream_config' do - let(:mock_path) { '/fake/path/airbyte_stream_config.json' } - - let(:config_double) do - instance_double( - 'DfE::Analytics.config', - airbyte_stream_config_path: mock_path - ) - end - - before do - allow(DfE::Analytics).to receive(:config).and_return(config_double) - end - - context 'when the JSON file is valid' do - let(:json_data) do - { - configurations: { - streams: [ - { - name: 'teachers', - syncMode: 'incremental_append', - selectedFields: [{ fieldPath: ['id'] }] - } - ] - } - }.to_json - end - - before do - allow(File).to receive(:read).with(mock_path).and_return(json_data) - end - - it 'returns the parsed JSON with symbolized keys' do - result = described_class.airbyte_stream_config - - expect(result).to eq( - configurations: { - streams: [ - { - name: 'teachers', - syncMode: 'incremental_append', - selectedFields: [{ fieldPath: ['id'] }] - } - ] - } - ) - - # verify keys are deep symbolized - expect(result.keys).to all(be_a(Symbol)) - expect(result[:configurations][:streams].first[:name]).to eq('teachers') - expect(result[:configurations][:streams].first[:selectedFields].first[:fieldPath]).to eq(['id']) - end - end - - context 'when File.read raises a RuntimeError' do - before do - allow(File).to receive(:read).with(mock_path).and_raise(RuntimeError) - end - - it 'returns an empty hash' do - expect(described_class.airbyte_stream_config).to eq({}) - end - end - end end