Remove airbyte config json from gem config, it is now generated dynam…

config/locales/en.yml

@@ -116,10 +116,6 @@ en:
           description: |
             Perform airbyte checks on startup and allow airbyte config generation
           default: false
-        airbyte_stream_config_path:
-          description: |
-            Path of airbyte stream config file relative to the App root (Rails.root)
-          default: /terraform/aks/workspace-variables/airbyte_stream_config.json
         airbyte_internal_dataset:
           description: |
             Airbyte internal dataset for service

lib/dfe/analytics.rb

@@ -120,12 +120,6 @@ def self.event_debug_filters
       {}
     end
 
-    def self.airbyte_stream_config
-      JSON.parse(File.read(DfE::Analytics.config.airbyte_stream_config_path)).deep_symbolize_keys
-    rescue RuntimeError
-      {}
-    end
-
     def self.environment
       config.environment
     end

lib/dfe/analytics/airbyte_stream_config.rb

@@ -13,10 +13,6 @@ class AirbyteStreamConfig
       AIRBYTE_HEARTBEAT_ATTRIBUTES = %w[id last_heartbeat].freeze
       AIRBYTE_HEARTBEAT_ENTITY_ATTRIBUTES = { AIRBYTE_HEARTBEAT_ENTITY.to_sym => AIRBYTE_HEARTBEAT_ATTRIBUTES }.freeze
 
-      def self.generate_pretty_json_for(table_attributes)
-        JSON.pretty_generate(generate_for(table_attributes))
-      end
-
       def self.generate_for(table_attributes)
         { configurations: { streams: streams_for(table_attributes) } }
       end

lib/dfe/analytics/config.rb

@@ -76,11 +76,10 @@ def self.configure(config)
         config.airbyte_server_url               ||= ENV.fetch('AIRBYTE_SERVER_URL', nil)
         config.airbyte_configuration            ||=
           JSON.parse(ENV.fetch('AIRBYTE_CONFIGURATION', '{}')).transform_keys(&:underscore).symbolize_keys
+        config.airbyte_stream_config_path       ||= nil
 
         config.async = true if config.async.nil?
 
-        config.airbyte_stream_config_path = File.join(Rails.root, config.airbyte_stream_config_path) if config.airbyte_stream_config_path.present?
-
         return unless config.azure_federated_auth
 
         config.azure_client_id          ||= ENV.fetch('AZURE_CLIENT_ID', nil)

lib/dfe/analytics/fields.rb

@@ -45,20 +45,6 @@ def self.check!
           HEREDOC
         end
 
-        if DfE::Analytics.airbyte_enabled? && airbyte_conflicting_fields.any?
-          errors << <<~HEREDOC
-            Differences detected between analytics.yml and #{File.basename(DfE::Analytics.config.airbyte_stream_config_path)}!
-
-            The following field differences exist. To upgrade the airbyte stream config, run:
-
-            SUPPRESS_DFE_ANALYTICS_INIT=1 bundle exec rails dfe:analytics:regenerate_airbyte_stream_config
-
-            Field Differences:
-
-            #{airbyte_conflicting_fields.to_yaml}
-          HEREDOC
-        end
-
         configuration_errors = errors.join("\n\n----------------\n\n")
 
         raise(ConfigurationError, configuration_errors) if errors.any?

lib/dfe/analytics/tasks/fields.rake

@@ -12,13 +12,5 @@ namespace :dfe do
         { shared: DfE::Analytics::Fields.generate_blocklist }.to_yaml
       )
     end
-
-    desc 'Generate a new airbyte stream config listing all fields to be send to Bigquery from analytics.yml'
-    task regenerate_airbyte_stream_config: :environment do
-      File.write(
-        DfE::Analytics.config.airbyte_stream_config_path,
-        DfE::Analytics::AirbyteStreamConfig.generate_pretty_json_for(DfE::Analytics.allowlist)
-      )
-    end
   end
 end

lib/generators/dfe/analytics/install_generator.rb

@@ -13,10 +13,6 @@ def install
         create_file 'config/analytics.yml', { 'shared' => {} }.to_yaml
         create_file 'config/analytics_hidden_pii.yml', { 'shared' => {} }.to_yaml
         create_file 'config/analytics_blocklist.yml', { 'shared' => {} }.to_yaml
-        create_file(
-          DfE::Analytics.config.airbyte_stream_config_path,
-          DfE::Analytics::AirbyteStreamConfig.generate_pretty_json_for(table1: %w[id field1 field2])
-        )
       end
 
       private

spec/dfe/analytics/airbyte_stream_config_spec.rb

@@ -56,15 +56,6 @@
     end
   end
 
-  describe '.generate_pretty_json_for' do
-    let(:entity_attributes) { { users: %w[id name] } }
-
-    it 'returns a pretty JSON version of generate_for' do
-      expect(described_class.generate_pretty_json_for(entity_attributes))
-        .to eq(JSON.pretty_generate(described_class.generate_for(entity_attributes)))
-    end
-  end
-
   describe '.entity_attributes' do
     context 'when config is empty' do
       before { allow(DfE::Analytics).to receive(:airbyte_stream_config).and_return({}) }

spec/dfe/analytics/config_spec.rb

@@ -73,8 +73,8 @@
         described_class.configure(config)
       end
 
-      it 'resolves the full path from Rails.root' do
-        expect(config.airbyte_stream_config_path).to eq(Rails.root.join('config/airbyte.json').to_s)
+      it 'still stores the value' do
+        expect(config.airbyte_stream_config_path).to eq('config/airbyte.json')
       end
     end
   end

spec/dfe/analytics/generators/install_generator_spec.rb

@@ -64,14 +64,6 @@
         { 'shared' => {} }.to_yaml
       )
     end
-    it 'creates the airbyte stream config file' do
-      generator.install
-
-      expect(generator).to have_received(:create_file).with(
-        airbyte_stream_config_path,
-        generated_stream_config
-      )
-    end
 
     it 'uses the generated config options in the initializer' do
       generator.install

spec/dfe/analytics_spec.rb

@@ -427,70 +427,4 @@
       end
     end
   end
-
-  describe '.airbyte_stream_config' do
-    let(:mock_path) { '/fake/path/airbyte_stream_config.json' }
-
-    let(:config_double) do
-      instance_double(
-        'DfE::Analytics.config',
-        airbyte_stream_config_path: mock_path
-      )
-    end
-
-    before do
-      allow(DfE::Analytics).to receive(:config).and_return(config_double)
-    end
-
-    context 'when the JSON file is valid' do
-      let(:json_data) do
-        {
-          configurations: {
-            streams: [
-              {
-                name: 'teachers',
-                syncMode: 'incremental_append',
-                selectedFields: [{ fieldPath: ['id'] }]
-              }
-            ]
-          }
-        }.to_json
-      end
-
-      before do
-        allow(File).to receive(:read).with(mock_path).and_return(json_data)
-      end
-
-      it 'returns the parsed JSON with symbolized keys' do
-        result = described_class.airbyte_stream_config
-
-        expect(result).to eq(
-          configurations: {
-            streams: [
-              {
-                name: 'teachers',
-                syncMode: 'incremental_append',
-                selectedFields: [{ fieldPath: ['id'] }]
-              }
-            ]
-          }
-        )
-
-        # verify keys are deep symbolized
-        expect(result.keys).to all(be_a(Symbol))
-        expect(result[:configurations][:streams].first[:name]).to eq('teachers')
-        expect(result[:configurations][:streams].first[:selectedFields].first[:fieldPath]).to eq(['id'])
-      end
-    end
-
-    context 'when File.read raises a RuntimeError' do
-      before do
-        allow(File).to receive(:read).with(mock_path).and_raise(RuntimeError)
-      end
-
-      it 'returns an empty hash' do
-        expect(described_class.airbyte_stream_config).to eq({})
-      end
-    end
-  end
 end