===============================================================================
VFIO DEVICE PASSTHROUGH with DPAA2 devices
(Direct device assignment in VM)
===============================================================================
Note: Please first read nxp/README.

1.Preparations
  - The default dpdk binaries in SDK are built with dynamic IOVA selection.
  - if you are building with "arm64-dpaa2-linuxapp-gcc", Compile the DPDK
    code using CONFIG_RTE_LIBRTE_DPAA2_USE_PHYS_IOVA=y

2.Run the dynamic DPL with sample parent config file.

     Check (/usr/local/dpdk/dpaa2). It is also part of the DPDK
     source code in `nxp` folder.
     export DPDK_SCRIPTS=/usr/local/dpdk/dpaa2

    #create dpni on host (for communication/transfer with VM)
        ls-addni -n
	>> --- Created interface: eth0 (object:dpni.1, endpoint: )
    #Create VM - Linux M/C Container
	source $DPDK_SCRIPTS/dynamic_dpl.sh -c $DPDK_SCRIPTS/vm_linux.conf dpni.1
    #Create VM - Userspace DPDK container
	source $DPDK_SCRIPTS/dynamic_dpl.sh -c $DPDK_SCRIPTS/vm_dpdk.conf <dpmac.1> <dpmac.2>
    #Note: Make sure to enter the created parent DPRC into vm_dpdk.conf

Assuming parent container as dprc.2 and child container as dprc.3

3. Create an ethernet connect between host and VM for communication/transfer
	This was already created and passed during vm-linux container.
	#assign IP to host interface created to communicate with VM (dprc.2, eth0)
	ifconfig eth0 192.168.2.2

4. Bind the dprc to vfio and create hugepages.

	echo hugetlbfs /hugetlbfs hugetlbfs defaults,mode=0777 0 0 >> /etc/fstab
	mkdir /hugetlbfs
	mount /hugetlbfs

5. Build QEMU 2.9.x (if not already built)
	- Refer to SDK usermanual.
	git clone https://source.codeaurora.org/external/qoriq/qoriq-components/qemu
	cd qemu
	git checkout qemu-2.9
	git submodule update --init dtc

	#update your ubuntu m/c with required packages.
	apt-get install pkg-config
	apt-get install libglib2.0-dev
	apt-get install libpixman-1-dev
	apt-get install libaio-dev
	apt-get install libusb-1.0-0-dev

	./configure --prefix=/root/qemu-2.9 --target-list=aarch64-softmmu --enable-fdt --enable-kvm --with-system-pixman
	make
	make install

6. launch qemu (Version : 2.9.0) using following command:

	export ROOTFS_IMG=/ubuntu_xenial_arm64_rootfs.ext4.img
	export GUEST_CONSOLE_TELNET_PORT=4446
	export KERNEL_IMG=/root/Image-4.14
	export VM_DPRC=dprc.2

	# Telnet port to be used for accessing this instance of virtual machine

	# single core VM launch
        /root/qemu-2.9/bin/qemu-system-aarch64  -smp 1 -m 4096 -mem-path /hugetlbfs -cpu host -machine type=virt,gic-version=3 -kernel $KERNEL_IMG -enable-kvm -display none -serial tcp::$GUEST_CONSOLE_TELNET_PORT,server,telnet -drive if=none,file=$ROOTFS_IMG,id=foo,format=raw -device virtio-blk-device,drive=foo   -append 'root=/dev/vda rw console=ttyAMA0 rootwait earlyprintk' -monitor stdio -device vfio-fsl-mc,host=$VM_DPRC -S

	# Two core VM launch (check the isolcpus for core #1 in bootargs)
        /root/qemu-2.9/bin/qemu-system-aarch64  -smp 2 -m 4096 -mem-path /hugetlbfs -cpu host -machine type=virt,gic-version=3 -kernel $KERNEL_IMG -enable-kvm -display none -serial tcp::$GUEST_CONSOLE_TELNET_PORT,server,telnet -drive if=none,file=$ROOTFS_IMG,id=foo,format=raw -device virtio-blk-device,drive=foo   -append 'root=/dev/vda rw console=ttyAMA0 rootwait earlyprintk isolcpus=1' -monitor stdio -device vfio-fsl-mc,host=$VM_DPRC -S

	The userspace core shall be using "isolcpus=x" in the bootargs for guest
	kernel. i.e. add it to qemu cmdline argument e.g.
	 -append 'root=/dev/vda rw console=ttyAMA0 rootwait earlyprintk isolcpus=1'

	Note: Make sure to assign each vcpu thread to one physical CPU only?
	Start QEMU with -S option (the vcpu threads are not yet started).

7. Launch VM using : telnet <Board ip addr> <GUEST_CONSOLE_TELNET_PORT>
	eg: or if you are aleady on board, you can do:
	telnet localhost 4446

	#come back to qemu and affine the CPUs to core
	qemu> info cpus
                * CPU #0: thread_id=7211
                  CPU #1: (halted) thread_id=7212

	#on host set the pids to processors (e.g for 2 core mode)
	#note that threads shall be affined to consequtive cores.
	taskset -p 1 7211
	taskset -p 2 7212

	#continue on qemu
	qemu> c

8. Use "root" as login & password and run the following commands:

	echo 1000 > /proc/sys/vm/nr_hugepages
	# child DPRC container
	export DPRC=dprc.3

	# Start CPU core as per taskset (w.r.t host) to run DPDK app
	export HOST_START_CPU=1

	echo 1 > /sys/module/vfio/parameters/enable_unsafe_noiommu_mode
	echo vfio-fsl-mc > /sys/bus/fsl-mc/devices/dprc.3/driver_override
	echo dprc.3 > /sys/bus/fsl-mc/drivers/vfio-fsl-mc/bind
	echo hugetlbfs /hugetlbfs hugetlbfs defaults,mode=0777 0 0 >> /etc/fstab
	mkdir /hugetlbfs
	mount /hugetlbfs
	#configure the host connection for SCP, ssh and filetransfer
	ifconfig eth1 192.168.2.1

9. Run the DPDK application
	#one core VM (core #0 for dpdk)
        ./l3fwd -c 0x1 -n 1 --log-level=bus.fslmc,8 -- -p 0x1 -P --config="(0,0,0)"
        ./l2fwd-crypto -c 0x1 -n 1  --log-level=bus.fslmc,8 -- -p 0x1 -q 1 --chain HASH_ONLY --auth_algo sha2-256-hmac --auth_op GENERATE --auth_key_random_size 64

        #two core VM (core #1 for DPDK)
	./l3fwd -c 0x2 -n 1  -- -p 0x1 -P --config="(0,0,1)"
	./l3fwd -c 0x2 -n 1  -- -p 0x3 -P --config="(0,0,1),(1,0,1)"

        ./testpmd  -c 0x3 -n 1 -- -i --portmask=0x3 --nb-cores=1  --forward-mode=txonly

	./l2fwd-crypto -c 0x2 -n 1  -- -p 0x1 -q 1 --chain HASH_ONLY --auth_algo sha2-256-hmac --auth_op GENERATE --auth_key_random_size 64
	./l2fwd-crypto -c 0x2 -n 1  -- -p 0x1 --chain CIPHER_ONLY --cipher_op ENCRYPT --cipher_algo aes-cbc --cipher_key 00:01:02:03:04:05:06:07:08:09:0a:0b:0c:0d:0e:0f
