Skip to content

Commit c1e4bb2

Browse files
softestsophsoftestsoph
authored
Merge pull request #30 from softestsoph/dancecontent/recording
Recording to DanceData form Kinect or File.
2 parents 210c629 + 83cabfc commit c1e4bb2

File tree

8 files changed

+6075
-7
lines changed

8 files changed

+6075
-7
lines changed

‎unity/Assets/RecordingScene.unity

Lines changed: 5654 additions & 0 deletions
Large diffs are not rendered by default.

‎unity/Assets/Scripts/DanceManager.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -190,4 +190,4 @@ public void QuitToMenu()
190190
SceneManager.LoadScene("StartMenu", LoadSceneMode.Single);
191191
}
192192
}
193-
}
193+
}

‎unity/Assets/Scripts/Data/DanceData.cs

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ public static DanceData LoadFromJSON(string fileName)
6262
string jsonString = File.ReadAllText("jsondata/" + fileName);
6363
return JsonUtility.FromJson<DanceData>(jsonString);
6464
}
65+
66+
6567
}
6668

6769
public enum DanceDifficulty
@@ -108,6 +110,43 @@ public PoseData toPoseData()
108110
}
109111
return poseData;
110112
}
113+
114+
public static DancePose fromPoseData(PoseData poseData, float timeStamp = 0)
115+
{
116+
DancePose dancePose = new DancePose();
117+
118+
for(int i = 0; i < (int)JointId.Count; ++i)
119+
{
120+
dancePose.positions[i] = poseData.data[i].Position;
121+
dancePose.orientations[i] = poseData.data[i].Orientation;
122+
dancePose.timestamp = timeStamp;
123+
}
124+
125+
return dancePose;
126+
}
127+
128+
public static DancePose Body2DancePose(Body body, float ts)
129+
{
130+
DancePose dancePose = new DancePose();
131+
dancePose.timestamp = ts;
132+
for (int i = 0; i < (int)JointId.Count; ++i)
133+
{
134+
// write recorded poses to file
135+
Microsoft.Azure.Kinect.BodyTracking.Joint joint = body.Skeleton.GetJoint(i);
136+
var pos = joint.Position;
137+
var orientation = joint.Quaternion;
138+
// save raw data
139+
var v = new Vector3(pos.X, pos.Y, pos.Z);
140+
var r = new Quaternion(orientation.X, orientation.Y, orientation.Z, orientation.W);
141+
142+
dancePose.positions[i] = v;
143+
dancePose.orientations[i] = r;
144+
}
145+
146+
return dancePose;
147+
}
148+
111149
}
112150

151+
113152
}

‎unity/Assets/Scripts/PoseGetting/FilePoseGetter.cs

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,34 @@
22
using System.Collections.Generic;
33
using UnityEngine;
44
using System.IO;
5+
using System;
56

67
namespace PoseTeacher
78
{
89
public class FilePoseGetter : PoseGetter
910
{
11+
12+
private bool recording;
13+
public override bool Recording
14+
{
15+
get
16+
{
17+
return recording;
18+
}
19+
set
20+
{
21+
if (value)
22+
{
23+
RecordingStartTicks = DateTime.Now.Ticks;
24+
}
25+
else
26+
{
27+
LastTimeStamp = CurrentTimeStamp;
28+
}
29+
recording = value;
30+
}
31+
}
32+
1033
bool loop = false;
1134

1235
IEnumerator<string> SequenceEnum;
@@ -37,9 +60,41 @@ public override PoseData GetNextPose()
3760
string frame_json = SequenceEnum.Current;
3861
PoseData fake_live_data = PoseDataUtils.JSONstring2PoseData(frame_json);
3962
CurrentPose = fake_live_data;
63+
64+
// also save as dance pose
65+
CurrentTicks = System.DateTime.Now.Ticks;
66+
CurrentDancePose = DancePose.fromPoseData(CurrentPose, GetTimeStamp());
67+
if (Recording)
68+
{
69+
recordedDanceData.poses.Add(CurrentDancePose);
70+
}
71+
4072
return CurrentPose;
4173
}
4274

75+
public override DancePose GetNextDancePose()
76+
{
77+
if (!SequenceEnum.MoveNext())
78+
{
79+
// Quick and dirty way to loop (by reloading file)
80+
if (SequenceEnum == null || loop)
81+
{
82+
LoadData();
83+
SequenceEnum.MoveNext();
84+
}
85+
}
86+
87+
string frame_json = SequenceEnum.Current;
88+
PoseData fake_live_data = PoseDataUtils.JSONstring2PoseData(frame_json);
89+
CurrentTicks = System.DateTime.Now.Ticks;
90+
CurrentDancePose = DancePose.fromPoseData(CurrentPose, GetTimeStamp());
91+
if (Recording)
92+
{
93+
recordedDanceData.poses.Add(CurrentDancePose);
94+
}
95+
return CurrentDancePose;
96+
}
97+
4398
public override void Dispose(){}
4499

45100
public void RestartFile()
@@ -51,6 +106,20 @@ void LoadData()
51106
{
52107
SequenceEnum = File.ReadLines(ReadDataPath).GetEnumerator();
53108
}
109+
110+
public override void SaveDanceData()
111+
{
112+
string timestamp = DateTime.Now.ToString("yyyy_MM_dd-HH_mm_ss");
113+
string recordingName = "Recordings/recording-" + timestamp;
114+
DanceDataScriptableObject.SaveDanceDataToScriptableObject(recordedDanceData, recordingName, true);
115+
116+
// After Saving reset recorded data to have space for a new one
117+
recordedDanceData = new DanceData();
118+
}
119+
120+
54121
}
122+
123+
55124
}
56125

‎unity/Assets/Scripts/PoseGetting/KinectPoseGetter.cs

Lines changed: 107 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,21 @@ namespace PoseTeacher
1010
public class KinectPoseGetter : PoseGetter
1111
{
1212
private bool recording;
13-
public bool Recording {
13+
public override bool Recording {
1414
get
1515
{
1616
return recording;
1717
}
1818
set {
19-
if (value) StartRecording();
19+
if (value)
20+
{
21+
StartRecording();
22+
RecordingStartTicks = DateTime.Now.Ticks;
23+
}
24+
else
25+
{
26+
LastTimeStamp = CurrentTimeStamp;
27+
}
2028
recording = value;
2129
}
2230
}
@@ -46,6 +54,8 @@ public override PoseData GetNextPose()
4654
{
4755
if (device != null)
4856
{
57+
//Debug.Log("device: " + device.GetCapture(new System.TimeSpan(0, 0, 1)));
58+
4959
using (Capture capture = device.GetCapture())
5060
{
5161
// Make tracker estimate body
@@ -104,12 +114,86 @@ public override PoseData GetNextPose()
104114
}
105115
}
106116
}
117+
107118
else
108119
{
109120
Debug.Log("device is null!");
110121
}
122+
111123
return CurrentPose;
124+
125+
}
126+
127+
128+
public override DancePose GetNextDancePose()
129+
{
130+
if (device != null)
131+
{
132+
using (Capture capture = device.GetCapture())
133+
{
134+
// Make tracker estimate body
135+
tracker.EnqueueCapture(capture);
136+
137+
// Code for getting RGB image from camera
138+
Microsoft.Azure.Kinect.Sensor.Image color = capture.Color;
139+
if (color != null && color.WidthPixels > 0 && (streamCanvas != null || videoRenderer != null))
140+
{
141+
UnityEngine.Object.Destroy(tex);
142+
tex = new Texture2D(color.WidthPixels, color.HeightPixels, TextureFormat.BGRA32, false);
143+
tex.LoadRawTextureData(color.Memory.ToArray());
144+
tex.Apply();
145+
146+
//Fetch the RawImage component from the GameObject
147+
if (tex != null)
148+
{
149+
if (streamCanvas != null)
150+
{
151+
streamCanvas.GetComponent<RawImage>().texture = tex;
152+
}
153+
if (videoRenderer != null)
154+
{
155+
videoRenderer.material.mainTexture = tex;
156+
}
157+
}
158+
}
159+
160+
}
161+
162+
// Get pose estimate from tracker
163+
using (Frame frame = tracker.PopResult())
164+
{
165+
// At least one body found by Body Tracking
166+
if (frame.NumberOfBodies > 0)
167+
{
168+
// Use first estimated person, if mutiple are in the image
169+
// !!! There are (probably) no guarantees on consisitent ordering between estimates
170+
//var bodies = frame.Bodies;
171+
var body = frame.GetBody(0);
172+
TimeSpan ts = frame.DeviceTimestamp;
173+
174+
// Apply pose to user avatar(s)
175+
CurrentTicks = ts.Ticks;
176+
DancePose live_data = DancePose.Body2DancePose(body, GetTimeStamp());
177+
178+
if (Recording) // recording
179+
{
180+
recordedDanceData.poses.Add(live_data);
181+
}
182+
CurrentDancePose = live_data;
183+
}
184+
}
185+
}
186+
187+
else
188+
{
189+
Debug.Log("device is null!");
190+
}
191+
192+
return CurrentDancePose;
193+
112194
}
195+
196+
113197
public override void Dispose()
114198
{
115199
if (tracker != null)
@@ -135,17 +219,24 @@ void StartAzureKinect()
135219
WiredSyncMode = WiredSyncMode.Standalone,
136220
};
137221
device.StartCameras(config);
138-
Debug.Log("Open K4A device successful. sn:" + device.SerialNum);
222+
Debug.Log("Open K4A device successful. Serial Nr: " + device.SerialNum);
139223

140224
//var calibration = device.GetCalibration(config.DepthMode, config.ColorResolution);
141225
var calibration = device.GetCalibration();
142226

227+
int GpuID = SystemInfo.graphicsDeviceID;
228+
//Debug.Log("GPU ID: " + GpuID);
229+
230+
var trackerDefaultConfiguration = TrackerConfiguration.Default;
143231
var trackerConfiguration = new TrackerConfiguration
144232
{
145-
ProcessingMode = TrackerProcessingMode.Gpu,
146-
SensorOrientation = SensorOrientation.Default
233+
ProcessingMode = TrackerProcessingMode.Cuda, // Set to Cpu if it doesn't run
234+
SensorOrientation = SensorOrientation.Default,
235+
//GpuDeviceId = GpuID
147236
};
148237

238+
239+
Debug.Log("Creatting the Tracker");
149240
this.tracker = Tracker.Create(calibration, trackerConfiguration);
150241
Debug.Log("Body tracker created.");
151242
}
@@ -163,13 +254,23 @@ public void ResetRecording()
163254
File.WriteAllText(WriteDataPath, "");
164255
Debug.Log("Reset recording file");
165256
}
166-
257+
167258
void StartRecording()
168259
{
169260
string timestamp = DateTime.Now.ToString("yyyy_MM_dd-HH_mm_ss");
170261
WriteDataPath = "jsondata/" + timestamp + ".txt";
171262
}
172263

264+
public override void SaveDanceData()
265+
{
266+
string timestamp = DateTime.Now.ToString("yyyy_MM_dd-HH_mm_ss");
267+
string recordingName = "Recordings/recording-" + timestamp;
268+
DanceDataScriptableObject.SaveDanceDataToScriptableObject(recordedDanceData, recordingName, true);
269+
270+
// After Saving reset recorded data to have space for a new one
271+
recordedDanceData = new DanceData();
272+
}
273+
173274
}
174275
}
175276

‎unity/Assets/Scripts/PoseGetting/PoseGetter.cs

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,29 @@
11
using System.Collections;
22
using System.Collections.Generic;
33
using UnityEngine;
4+
using System;
45

56
namespace PoseTeacher
67
{
78
public enum InputSource { KINECT, FILE }
89
public abstract class PoseGetter
910
{
11+
public abstract bool Recording { get; set; }
12+
public float LastTimeStamp = 0;
13+
public float CurrentTimeStamp;
14+
public long RecordingStartTicks;
15+
public long CurrentTicks;
16+
public float GetTimeStamp()
17+
{
18+
CurrentTimeStamp = LastTimeStamp + (RecordingStartTicks - CurrentTicks) / 10_000_000;
19+
return CurrentTimeStamp;
20+
}
1021
public PoseData CurrentPose { get; protected set; }
22+
public DancePose CurrentDancePose { get; protected set; }
23+
public DanceData recordedDanceData = new DanceData();
1124
public abstract PoseData GetNextPose();
25+
public abstract DancePose GetNextDancePose();
26+
public abstract void SaveDanceData();
1227
public abstract void Dispose();
1328
}
1429

0 commit comments

Comments
 (0)