780 static void gcm_walk_start(struct gcm_sg_walk *gw, struct scatterlist *sg,  in gcm_walk_start()  argument
783 	memset(gw, 0, sizeof(*gw));  in gcm_walk_start()
784 	gw->walk_bytes_remain = len;  in gcm_walk_start()
785 	scatterwalk_start(&gw->walk, sg);  in gcm_walk_start()
788 static inline unsigned int _gcm_sg_clamp_and_map(struct gcm_sg_walk *gw)  in _gcm_sg_clamp_and_map()  argument
792 	gw->walk_bytes = scatterwalk_clamp(&gw->walk, gw->walk_bytes_remain);  in _gcm_sg_clamp_and_map()
793 	while (!gw->walk_bytes) {  in _gcm_sg_clamp_and_map()
794 		nextsg = sg_next(gw->walk.sg);  in _gcm_sg_clamp_and_map()
797 		scatterwalk_start(&gw->walk, nextsg);  in _gcm_sg_clamp_and_map()
798 		gw->walk_bytes = scatterwalk_clamp(&gw->walk,  in _gcm_sg_clamp_and_map()
799 						   gw->walk_bytes_remain);  in _gcm_sg_clamp_and_map()
801 	gw->walk_ptr = scatterwalk_map(&gw->walk);  in _gcm_sg_clamp_and_map()
802 	return gw->walk_bytes;  in _gcm_sg_clamp_and_map()
805 static inline void _gcm_sg_unmap_and_advance(struct gcm_sg_walk *gw,  in _gcm_sg_unmap_and_advance()  argument
808 	gw->walk_bytes_remain -= nbytes;  in _gcm_sg_unmap_and_advance()
809 	scatterwalk_unmap(gw->walk_ptr);  in _gcm_sg_unmap_and_advance()
810 	scatterwalk_advance(&gw->walk, nbytes);  in _gcm_sg_unmap_and_advance()
811 	scatterwalk_done(&gw->walk, 0, gw->walk_bytes_remain);  in _gcm_sg_unmap_and_advance()
812 	gw->walk_ptr = NULL;  in _gcm_sg_unmap_and_advance()
815 static int gcm_in_walk_go(struct gcm_sg_walk *gw, unsigned int minbytesneeded)  in gcm_in_walk_go()  argument
819 	if (gw->buf_bytes && gw->buf_bytes >= minbytesneeded) {  in gcm_in_walk_go()
820 		gw->ptr = gw->buf;  in gcm_in_walk_go()
821 		gw->nbytes = gw->buf_bytes;  in gcm_in_walk_go()
825 	if (gw->walk_bytes_remain == 0) {  in gcm_in_walk_go()
826 		gw->ptr = NULL;  in gcm_in_walk_go()
827 		gw->nbytes = 0;  in gcm_in_walk_go()
831 	if (!_gcm_sg_clamp_and_map(gw)) {  in gcm_in_walk_go()
832 		gw->ptr = NULL;  in gcm_in_walk_go()
833 		gw->nbytes = 0;  in gcm_in_walk_go()
837 	if (!gw->buf_bytes && gw->walk_bytes >= minbytesneeded) {  in gcm_in_walk_go()
838 		gw->ptr = gw->walk_ptr;  in gcm_in_walk_go()
839 		gw->nbytes = gw->walk_bytes;  in gcm_in_walk_go()
844 		n = min(gw->walk_bytes, AES_BLOCK_SIZE - gw->buf_bytes);  in gcm_in_walk_go()
845 		memcpy(gw->buf + gw->buf_bytes, gw->walk_ptr, n);  in gcm_in_walk_go()
846 		gw->buf_bytes += n;  in gcm_in_walk_go()
847 		_gcm_sg_unmap_and_advance(gw, n);  in gcm_in_walk_go()
848 		if (gw->buf_bytes >= minbytesneeded) {  in gcm_in_walk_go()
849 			gw->ptr = gw->buf;  in gcm_in_walk_go()
850 			gw->nbytes = gw->buf_bytes;  in gcm_in_walk_go()
853 		if (!_gcm_sg_clamp_and_map(gw)) {  in gcm_in_walk_go()
854 			gw->ptr = NULL;  in gcm_in_walk_go()
855 			gw->nbytes = 0;  in gcm_in_walk_go()
861 	return gw->nbytes;  in gcm_in_walk_go()
864 static int gcm_out_walk_go(struct gcm_sg_walk *gw, unsigned int minbytesneeded)  in gcm_out_walk_go()  argument
866 	if (gw->walk_bytes_remain == 0) {  in gcm_out_walk_go()
867 		gw->ptr = NULL;  in gcm_out_walk_go()
868 		gw->nbytes = 0;  in gcm_out_walk_go()
872 	if (!_gcm_sg_clamp_and_map(gw)) {  in gcm_out_walk_go()
873 		gw->ptr = NULL;  in gcm_out_walk_go()
874 		gw->nbytes = 0;  in gcm_out_walk_go()
878 	if (gw->walk_bytes >= minbytesneeded) {  in gcm_out_walk_go()
879 		gw->ptr = gw->walk_ptr;  in gcm_out_walk_go()
880 		gw->nbytes = gw->walk_bytes;  in gcm_out_walk_go()
884 	scatterwalk_unmap(gw->walk_ptr);  in gcm_out_walk_go()
885 	gw->walk_ptr = NULL;  in gcm_out_walk_go()
887 	gw->ptr = gw->buf;  in gcm_out_walk_go()
888 	gw->nbytes = sizeof(gw->buf);  in gcm_out_walk_go()
891 	return gw->nbytes;  in gcm_out_walk_go()
894 static int gcm_in_walk_done(struct gcm_sg_walk *gw, unsigned int bytesdone)  in gcm_in_walk_done()  argument
896 	if (gw->ptr == NULL)  in gcm_in_walk_done()
899 	if (gw->ptr == gw->buf) {  in gcm_in_walk_done()
900 		int n = gw->buf_bytes - bytesdone;  in gcm_in_walk_done()
902 			memmove(gw->buf, gw->buf + bytesdone, n);  in gcm_in_walk_done()
903 			gw->buf_bytes = n;  in gcm_in_walk_done()
905 			gw->buf_bytes = 0;  in gcm_in_walk_done()
907 		_gcm_sg_unmap_and_advance(gw, bytesdone);  in gcm_in_walk_done()
912 static int gcm_out_walk_done(struct gcm_sg_walk *gw, unsigned int bytesdone)  in gcm_out_walk_done()  argument
916 	if (gw->ptr == NULL)  in gcm_out_walk_done()
919 	if (gw->ptr == gw->buf) {  in gcm_out_walk_done()
921 			if (!_gcm_sg_clamp_and_map(gw))  in gcm_out_walk_done()
923 			n = min(gw->walk_bytes, bytesdone - i);  in gcm_out_walk_done()
924 			memcpy(gw->walk_ptr, gw->buf + i, n);  in gcm_out_walk_done()
925 			_gcm_sg_unmap_and_advance(gw, n);  in gcm_out_walk_done()
928 		_gcm_sg_unmap_and_advance(gw, bytesdone);  in gcm_out_walk_done()