change config to paper

.idea/workspace.xml

@@ -3,6 +3,11 @@
   <component name="ChangeListManager">
     <list default="true" id="b4fb7f33-5387-4628-bcdb-b1b79dd926d0" name="Default Changelist" comment="">
       <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/config/config.json" beforeDir="false" afterPath="$PROJECT_DIR$/config/config.json" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/config/config_layer_3.json" beforeDir="false" afterPath="$PROJECT_DIR$/config/config_layer_3.json" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/modeling_bert.py" beforeDir="false" afterPath="$PROJECT_DIR$/modeling_bert.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/run_interaction.py" beforeDir="false" afterPath="$PROJECT_DIR$/run_interaction.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/train.sh" beforeDir="false" afterPath="$PROJECT_DIR$/train.sh" afterDir="false" />
     </list>
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
@@ -21,13 +26,44 @@
     <property name="RunOnceActivity.OpenProjectViewOnStart" value="true" />
     <property name="RunOnceActivity.ShowReadmeOnStart" value="true" />
     <property name="WebServerToolWindowFactoryState" value="false" />
+    <property name="restartRequiresConfirmation" value="false" />
   </component>
   <component name="RecentsManager">
     <key name="MoveFile.RECENT_KEYS">
-      <recent name="E:\CPI\project\CPI\log\lr-5e-6-batch-30-layer3" />
       <recent name="E:\CPI\project\CPI" />
+      <recent name="E:\CPI\project\CPI\config" />
+      <recent name="E:\CPI\project\CPI\log\lr-5e-6-batch-30-layer3" />
     </key>
   </component>
+  <component name="RunManager">
+    <configuration name="run_interaction" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
+      <module name="CPI" />
+      <option name="INTERPRETER_OPTIONS" value="" />
+      <option name="PARENT_ENVS" value="true" />
+      <envs>
+        <env name="PYTHONUNBUFFERED" value="1" />
+      </envs>
+      <option name="SDK_HOME" value="" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+      <option name="IS_MODULE_SDK" value="true" />
+      <option name="ADD_CONTENT_ROOTS" value="true" />
+      <option name="ADD_SOURCE_ROOTS" value="true" />
+      <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/run_interaction.py" />
+      <option name="PARAMETERS" value="" />
+      <option name="SHOW_COMMAND_LINE" value="false" />
+      <option name="EMULATE_TERMINAL" value="false" />
+      <option name="MODULE_MODE" value="false" />
+      <option name="REDIRECT_INPUT" value="false" />
+      <option name="INPUT_FILE" value="" />
+      <method v="2" />
+    </configuration>
+    <recent_temporary>
+      <list>
+        <item itemvalue="Python.run_interaction" />
+      </list>
+    </recent_temporary>
+  </component>
   <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
   <component name="TaskManager">
     <task active="true" id="Default" summary="Default task">
@@ -36,11 +72,36 @@
       <option name="number" value="Default" />
       <option name="presentableId" value="Default" />
       <updated>1617112323231</updated>
-      <workItem from="1617112324809" duration="4170000" />
+      <workItem from="1617112324809" duration="10450000" />
     </task>
     <servers />
   </component>
   <component name="TypeScriptGeneratedFilesManager">
     <option name="version" value="3" />
   </component>
+  <component name="Vcs.Log.Tabs.Properties">
+    <option name="TAB_STATES">
+      <map>
+        <entry key="MAIN">
+          <value>
+            <State />
+          </value>
+        </entry>
+      </map>
+    </option>
+  </component>
+  <component name="XDebuggerManager">
+    <breakpoint-manager>
+      <breakpoints>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/modeling_bert.py</url>
+          <line>1989</line>
+          <option name="timeStamp" value="1" />
+        </line-breakpoint>
+      </breakpoints>
+    </breakpoint-manager>
+  </component>
+  <component name="com.intellij.coverage.CoverageDataManagerImpl">
+    <SUITE FILE_PATH="coverage/CPI$run_interaction.coverage" NAME="run_interaction Coverage Results" MODIFIED="1617283608264" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="true" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+  </component>
 </project>
\ No newline at end of file

config/config.json

@@ -7,12 +7,12 @@
   "hidden_dropout_prob": 0.1,
   "hidden_size": 768,
   "initializer_range": 0.02,
-  "intermediate_size": 3072,
+  "intermediate_size": 1536,
   "layer_norm_eps": 1e-12,
   "max_position_embeddings": 512,
   "model_type": "bert",
   "num_attention_heads": 12,
-  "num_hidden_layers": 12,
+  "num_hidden_layers": 3,
   "pad_token_id": 0,
   "type_vocab_size": 2,
   "vocab_size": 23615

config/config_layer_3.json

@@ -5,9 +5,9 @@
   "attention_probs_dropout_prob": 0.1,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
-  "hidden_size": 768,
+  "hidden_size": 384,
   "initializer_range": 0.02,
-  "intermediate_size": 3072,
+  "intermediate_size": 1536,
   "layer_norm_eps": 1e-12,
   "max_position_embeddings": 512,
   "model_type": "bert",

modeling_bert.py

@@ -1986,6 +1986,7 @@ class BertAffinityModel(BertPreTrainedModel):
             inputs_embeds=inputs_embeds,
             past_key_values_length=past_key_values_length,
         )
+        print(embedding_output.size())
         encoder_outputs = self.encoder(
             embedding_output,
             attention_mask=extended_attention_mask,

run_interaction.py

@@ -64,12 +64,16 @@ def main(args):
         for epoch in range(args.epochs):
             for i, (input, affinity) in enumerate(data_generator):
                 # use cuda
-#                 if torch.cuda.is_available():
-#                     input.cuda()
-#                     affinity.cuda()
+                # if torch.cuda.is_available():
+                #     input.cuda()
+                #     affinity.cuda()
                 # input model
-                pred_affinity = model(input.cuda().long())
-                loss = loss_fct(pred_affinity, affinity.cuda().unsqueeze(-1))
+                if torch.cuda.is_available():
+                    pred_affinity = model(input.cuda().long())
+                    loss = loss_fct(pred_affinity, affinity.cuda().unsqueeze(-1))
+                else:
+                    pred_affinity = model(input.long())
+                    loss = loss_fct(pred_affinity, affinity.unsqueeze(-1))
                 writer.add_scalar('loss', loss, num_step)
                 # Update gradient
                 opt.zero_grad()

train.sh

-CUDA_VISIBLE_DEVICES=3 python run_interaction.py --b=30 --task=train --epochs=10 --lr=5e-5 --savedir=lr-5e-5-batch-30-layer3 --config=./config/config_layer_3.json
\ No newline at end of file
+CUDA_VISIBLE_DEVICES=1 python run_interaction.py --b=64 --task=train --epochs=10 --lr=1e-7 --savedir=lr-1e-7-batch-64-layer3 --config=./config/config_layer_3.json
\ No newline at end of file