After a couple weeks of on and off work, this is what I have:
I used the mido python library to get the input keys, then made the keys control driver objects with bpy that controlled the face with shape keys.
The audio is from Garage Band, and I used the internal viewport renderer with matcap
Here is a code framework for anyone who wants to use this method (you will need mido and rtmidi in your Blender python):
import mido
import bpy
from time import sleep
mido.set_backend('mido.backends.rtmidi')
port = mido.open_input() #<-- insert name of MIDI input (get list of names of MIDI inputs with mido.get_input_names())
frames = 20000 #duration of running code
beingPressed = [] #contains the keys being pressed in each frame
for frame in range(frames): #for each frame
for msg in port.iter_pending(): #Update beingPressed
if msg.velocity == 0: #if velocity is 0, that means the key has just stopped being pressed
beingPressed.remove(msg.note)
else: #if the velocity isn't 0, then that means the key just started being pressed
beingPressed.append(msg.note)
#put whatever code you want here to do something each frame using beingPressed
#draw the frame
bpy.ops.wm.redraw_timer(type='DRAW_WIN_SWAP', iterations=1)
sleep(#insert 1/(framerate))
I got the mido and rtmidi files into Blender by using pip3 and brew in terminal to install them onto my system python and then transferred the files from /Library/Frameworks/Python.framework/Versions/3.5/lib/python3.5/site-packages to blender.app/Contents/Resources/2.77/python/lib/python3.5/site-packages
I’m sure there’s a better way to get python libraries into blender, but this worked for me
mido documentation: https://mido.readthedocs.io/en/latest/ports.html
Here’s my code, but it is very specific to my setup (here’s my blend file: MIDI24.blend (2.22 MB)), so it’s probably not very useful:
import mido
import bpy
from time import sleep
mido.set_backend('mido.backends.rtmidi')
port = mido.open_input(u'KeyRig 49')
frames = 20000
controlSpeed = .08
beingPressed = [] #contains the keys being pressed each frame
#Face Drivers change the character
faceDrivers = [bpy.data.objects['Face1'], bpy.data.objects['Face2'], bpy.data.objects['Face3'], bpy.data.objects['Face4'], bpy.data.objects['Face5'], bpy.data.objects['Face6'], bpy.data.objects['Face7'], bpy.data.objects['Face8']]
currentFace = 0
#Driver information format: [Driver obj, increase key, decrease key]
JawUD = [bpy.data.objects['JawUD'],36,38]
CheekPuffUD = [bpy.data.objects['CheekPuffUD'],37,39]
JawLR = [bpy.data.objects['JawLR'],41,40]
TopLipUD = [bpy.data.objects['TopLipUD'],43,45]
BotLipUD = [bpy.data.objects['BotLipUD'],47,48]
LipPuckerUD = [bpy.data.objects['LipPuckerUD'],49,51]
LeftLipUD = [bpy.data.objects['LeftLipUD'],50,52]
LeftLipLR = [bpy.data.objects['LeftLipLR'],55,53]
RightLipUD = [bpy.data.objects['RightLipUD'],57,59]
RightLipLR = [bpy.data.objects['RightLipLR'],62,60]
NoseSniffUD = [bpy.data.objects['NoseSniffUD'],61,63]
LeftBrowInUD = [bpy.data.objects['LeftBrowInUD'],64,65]
LeftBrowOutUD = [bpy.data.objects['LeftBrowOutUD'],67,69]
RightBrowInUD = [bpy.data.objects['RightBrowInUD'],71,72]
EyeDilateUD = [bpy.data.objects['EyeDilateUD'],73,75]
RightBrowOutUD = [bpy.data.objects['RightBrowOutUD'],74,76]
LeftEyeLidUD = [bpy.data.objects['LeftEyeLidUD'],77,79]
RightEyeLidUD = [bpy.data.objects['RightEyeLidUD'],81,83]
drivers = [JawUD, CheekPuffUD, JawLR, TopLipUD, BotLipUD, LipPuckerUD, LeftLipUD, LeftLipLR, RightLipUD, RightLipUD, RightLipLR, NoseSniffUD, LeftBrowInUD, LeftBrowOutUD, RightBrowInUD, EyeDilateUD, RightBrowOutUD, LeftEyeLidUD, RightEyeLidUD]
#pose keys set the face back to a preset face pose
#pose keys format: note number:
[list of locations for each driver]
poseKeys = {
'82':[1, 36, 12, -27, 25, 0, 19, -4, 36, 36, -13, 0, 12, -1, 15, 0, -11, -8, -19],
'80':[-36, -6, -2, 37, -36, 2, 36, -31, 33, 33, 35, -19, 44, 5, 33, -2, 10, 9, 14],
'78':[11, 0, 2, -38, 2, 0, 8, 7, 3, 3, -33, 0, 20, -4, 16, 0, 2, 9, 6],
'70':[-33, -16, 22, 26, -35, -2, -8, 40, -15, -15, -39, -36, -1, -34, -19, -11, -34, -3, 0],
'68':[41, -9, -16, 2, 45, 13, 4, 36, -5, -5, 18, -30, 32, -6, 12, -1, -41, 2, -4],
'66':[-32, -11, -3, 37, -44, 1, -13, 34, -41, -41, -38, -28, 39, 6, 33, 0, 17, 34, 44],
'58':[-10, 0, -4, -32, 0, 0, 44, -8, 35, 35, 31, 0, -31, 23, -36, 0, 33, -3, -2],
'56':[-33, 0, -18, 40, -2, 0, 9, 32, 31, 31, 36, 0, -27, 3, -32, 0, 16, 10, 13],
'54':[46, 0, 15, 13, 32, 0, -4, 7, 0, 0, -17, 0, -32, 39, -38, 0, 16, 9, 7],
'46':[-33, 7, 1, 37, -33, -34, -33, -33, -32, -32, 37, -32, 37, 36, 35, 0, 34, 12, 8],
'44':[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
'42':[33, 33, 0, 6, 33, 20, 7, 20, 4, 4, 0, 33, -35, 3, -35, -18, 10, -10, -9],
}
for key in bpy.data.groups['Keys'].objects: #reset keys to off mode
key.location.z = 0
for driver in drivers: #resets drivers their default positions
driver[0].location.z = 0
for frame in range(frames): #for each frame
for msg in port.iter_pending(): #Update beingPressed
if hasattr(msg, 'velocity'): #if it is a note (rather than modulation wheel for example)
if msg.velocity == 0: #if velocity is 0, that means the key has just stopped being pressed
beingPressed.remove(msg.note)
key = bpy.data.objects[str(msg.note)]
key.location.z = 0
else:
beingPressed.append(msg.note)
key = bpy.data.objects[str(msg.note)]
key.location.z = msg.velocity
if msg.note == 84 and msg.velocity > 0: #84 is the key that switches the character
faceDrivers[currentFace%len(faceDrivers)].location.z = 0 #reset previous face shape key driver
currentFace += 1
faceDrivers[currentFace%len(faceDrivers)].location.z = 1 #turn on new shape key driver
for note in beingPressed: #stuff for keys that set the face to a default pose
if str(note) in poseKeys.keys():
for i, driver in enumerate(drivers, start=0):
driver[0].location.z = poseKeys[str(note)][i]
for driver in drivers: #stuff for keys that affect muscle drivers
if beingPressed.__contains__(driver[1]):
if driver[0].location.z < 30:
hitStrength = bpy.data.objects[str(driver[1])].location.z
driver[0].location.z += 3 + controlSpeed*hitStrength
if beingPressed.__contains__(driver[2]):
if driver[0].location.z > -30:
hitStrength = bpy.data.objects[str(driver[2])].location.z
driver[0].location.z -= 3 + controlSpeed*hitStrength
#next frame
bpy.ops.wm.redraw_timer(type='DRAW_WIN_SWAP', iterations=1)
sleep(.025)
print((frame+1)/frames)
#reset positions of keys and drivers
for driver in faceDrivers:
driver.location.z = 0
faceDrivers[0].location.z = 1
for key in bpy.data.groups['Keys'].objects:
key.location.z = 0
for driver in drivers:
driver[0].location.z = 0