training federated model by bert and resnet pre-train model

Question

I want to train a multi-modal model in the federated learning environment.
this is my model definition.

num_classes=5
def image_text_model():
  # Define the image input
  image_input = Input(shape=(224, 224, 3), name="image_input")
  # text_input = Input(shape=(tokenizer.model_max_length, ), dtype=tf.int32, name="text_input")

  input_ids = Input(shape=(96,), dtype=tf.int32, name="input_ids")
  token_type_ids = Input(shape=(96,), dtype=tf.int32, name="token_type_ids")
  attention_mask = Input(shape=(96,), dtype=tf.int32, name="attention_mask")


  # Use the base_model to extract features from the image input
  # image_features = img_model()

  resnet_model = ResNet50(weights="imagenet", include_top=False)
  image_output = resnet_model(image_input)
  image_output = GlobalAveragePooling2D()(image_output)

  # Use the BERT model to extract features from the text input
  text_features = bert_model(input_ids, token_type_ids, attention_mask).pooler_output

  # Concatenate the image features and text features
  concatenated = concatenate([image_output,text_features])

  # Add a Dense layer
  x = Dense(units=128, activation='relu')(concatenated)

  # Add the output Dense layer with softmax activation for classification
  output = Dense(num_classes, activation='softmax')(x)

  # Define the model
  model = tf.keras.Model(inputs=[image_input, input_ids, token_type_ids, attention_mask], outputs=[output])

  return model

def federated_model():
  keras_model = image_text_model()
  return tff.learning.models.from_keras_model(
      keras_model,
      input_spec=[tf.TensorSpec(shape=(224, 224, 3), dtype=tf.float32),
          tf.TensorSpec(shape=(96,), dtype=tf.int32),
          tf.TensorSpec(shape=(96,), dtype=tf.int32),
          tf.TensorSpec(shape=(96,), dtype=tf.int32)
          ],

      loss=tf.keras.losses.SparseCategoricalCrossentropy(),
      metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]

  )

# Create a tff learning process
iterative_process =  tff.learning.algorithms.build_weighted_fed_avg(
    model_fn=federated_model,
    client_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=0.02),
    server_optimizer_fn=lambda: tf.keras.optimizers.SGD(learning_rate=1.0),
    # metrics=[tf.keras.metrics.SparseCategoricalAccuracy()]
    )

# Initialize the Federated Averaging algorithm to get the initial model state.
state = iterative_process.initialize()

# Run one round of Federated Averaging.
state, metrics = iterative_process.next(state, next(train_generator))
print('round  1, metrics={}'.format(metrics))

# You can continue training with additional rounds as needed.
# state, metrics = iterative_process.next(state, clients_data)
# print('round  2, metrics={}'.format(metrics))

this is my bert_model code bert_model = TFBertForSequenceClassification.from_pretrained("bert-base-uncased", num_labels=5).
When I want to train the model I got this error. enter image description here
I load my data by using the generator.
and this is my value error

ValueError: Your Layer or Model is in an invalid state. This can happen for the following cases:
 1. You might be interleaving estimator/non-estimator models or interleaving models/layers made in tf.compat.v1.Graph.as_default() with models/layers created outside of it. Converting a model to an estimator (via model_to_estimator) invalidates all models/layers made before the conversion (even if they were not the model converted to an estimator). Similarly, making a layer or a model inside a a tf.compat.v1.Graph invalidates all layers/models you previously made outside of the graph.
2. You might be using a custom keras layer implementation with custom __init__ which didn't call super().__init__.  Please check the implementation of <class 'transformers.models.bert.modeling_tf_bert.TFBertForSequenceClassification'> and its bases.

Leave a Comment Cancel reply